bes Updated for version 3.20.13
HDF5GMCF.cc
Go to the documentation of this file.
1// This file is part of the hdf5_handler implementing for the CF-compliant
2// Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
3//
4// This is free software; you can redistribute it and/or modify it under the
5// terms of the GNU Lesser General Public License as published by the Free
6// Software Foundation; either version 2.1 of the License, or (at your
7// option) any later version.
8//
9// This software is distributed in the hope that it will be useful, but
10// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
12// License for more details.
13//
14// You should have received a copy of the GNU Lesser General Public
15// License along with this library; if not, write to the Free Software
16// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17//
18// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
19// You can contact The HDF Group, Inc. at 1800 South Oak Street,
20// Suite 203, Champaign, IL 61820
21
36
37#include "HDF5CF.h"
38#include "HDF5RequestHandler.h"
39#include "h5apicompatible.h"
40#include <BESDebug.h>
41#include <algorithm>
42
43using namespace std;
44using namespace libdap;
45using namespace HDF5CF;
46
47// Copier function.
48GMCVar::GMCVar(Var*var) {
49
50 BESDEBUG("h5", "Coming to GMCVar()"<<endl);
51 newname = var->newname;
52 name = var->name;
53 fullpath = var->fullpath;
54 rank = var->rank;
55 total_elems = var->total_elems;
56 zero_storage_size = var->zero_storage_size;
57 dtype = var->dtype;
58 unsupported_attr_dtype = var->unsupported_attr_dtype;
59 unsupported_dspace = var->unsupported_dspace;
60 coord_attr_add_path = false;
61
62 for (auto ira = var->attrs.begin();
63 ira!=var->attrs.end(); ++ira) {
64 auto attr= new Attribute();
65 attr->name = (*ira)->name;
66 attr->newname = (*ira)->newname;
67 attr->dtype =(*ira)->dtype;
68 attr->count =(*ira)->count;
69 attr->strsize = (*ira)->strsize;
70 attr->fstrsize = (*ira)->fstrsize;
71 attr->value =(*ira)->value;
72 attrs.push_back(attr);
73 }
74
75 for (auto ird = var->dims.begin();
76 ird!=var->dims.end(); ++ird) {
77 auto dim = new Dimension((*ird)->size);
78 dim->name = (*ird)->name;
79 dim->newname = (*ird)->newname;
80 dim->unlimited_dim = (*ird)->unlimited_dim;
81 dims.push_back(dim);
82 }
83
84 product_type = General_Product;
85
86}
87#if 0
88GMCVar::GMCVar(GMCVar*cvar) {
89
90 newname = cvar->newname;
91 name = cvar->name;
92 fullpath = cvar->fullpath;
93 rank = cvar->rank;
94 dtype = cvar->dtype;
95 unsupported_attr_dtype = cvar->unsupported_attr_dtype;
96 unsupported_dspace = cvar->unsupported_dspace;
97
98 for (vector<Attribute*>::iterator ira = cvar->attrs.begin();
99 ira!=cvar->attrs.end(); ++ira) {
100 Attribute* attr= new Attribute();
101 attr->name = (*ira)->name;
102 attr->newname = (*ira)->newname;
103 attr->dtype =(*ira)->dtype;
104 attr->count =(*ira)->count;
105 attr->strsize = (*ira)->strsize;
106 attr->fstrsize = (*ira)->fstrsize;
107 attr->value =(*ira)->value;
108 attrs.push_back(attr);
109 }
110
111 for (vector<Dimension*>::iterator ird = cvar->dims.begin();
112 ird!=cvar->dims.end(); ++ird) {
113 Dimension *dim = new Dimension((*ird)->size);
114//"h5","dim->name "<< (*ird)->name <<endl;
115//"h5","dim->newname "<< (*ird)->newname <<endl;
116 dim->name = (*ird)->name;
117 dim->newname = (*ird)->newname;
118 dims.push_back(dim);
119 }
120
121 GMcvar->cfdimname = latdim0;
122 GMcvar->cvartype = CV_EXIST;
123 GMcvar->product_type = product_type;
124
125
126}
127#endif
128
129//Copier function of a special variable.
130GMSPVar::GMSPVar(Var*var) {
131
132 BESDEBUG("h5", "Coming to GMSPVar()"<<endl);
133 fullpath = var->fullpath;
134 rank = var->rank;
135 total_elems = var->total_elems;
136 zero_storage_size = var->zero_storage_size;
137 unsupported_attr_dtype = var->unsupported_attr_dtype;
138 unsupported_dspace = var->unsupported_dspace;
139 coord_attr_add_path = var->coord_attr_add_path;
140 // The caller of this function should change the following fields.
141 // This is just to make data coverity happy.
142 otype = H5UNSUPTYPE;
143 sdbit = -1;
144 numofdbits = -1;
145
146 for (auto ira = var->attrs.begin();
147 ira!=var->attrs.end(); ++ira) {
148 auto attr= new Attribute();
149 attr->name = (*ira)->name;
150 attr->newname = (*ira)->newname;
151 attr->dtype =(*ira)->dtype;
152 attr->count =(*ira)->count;
153 attr->strsize = (*ira)->strsize;
154 attr->fstrsize = (*ira)->fstrsize;
155 attr->value =(*ira)->value;
156 attrs.push_back(attr);
157 }
158
159 for (auto ird = var->dims.begin();
160 ird!=var->dims.end(); ++ird) {
161 auto dim = new Dimension((*ird)->size);
162 dim->name = (*ird)->name;
163 dim->newname = (*ird)->newname;
164 dim->unlimited_dim = (*ird)->unlimited_dim;
165 dims.push_back(dim);
166 }
167}
168
169
170GMFile::GMFile(const char*file_fullpath, hid_t file_id, H5GCFProduct product_type, GMPattern gproduct_pattern):
171File(file_fullpath,file_id), product_type(product_type),gproduct_pattern(gproduct_pattern)
172{
173
174}
175
176// destructor
177GMFile::~GMFile()
178{
179
180 if (!this->cvars.empty()){
181 for (auto i= this->cvars.begin(); i!=this->cvars.end(); ++i) {
182 delete *i;
183 }
184 }
185
186 if (!this->spvars.empty()){
187 for (auto i= this->spvars.begin(); i!=this->spvars.end(); ++i) {
188 delete *i;
189 }
190 }
191
192}
193
194// Get CF string
195string GMFile::get_CF_string(string s) {
196
197 // HDF5 group or variable path always starts with '/'. When CF naming rule is applied,
198 // the first '/' is always changes to "_", this is not necessary. However,
199 // to keep the backward compatiablity, I use a BES key for people to go back with the original name.
200
201 if(s[0] !='/')
202 return File::get_CF_string(s);
203 else if (General_Product == product_type && OTHERGMS == gproduct_pattern) {
204
205 if(true == HDF5RequestHandler::get_keep_var_leading_underscore())
206 return File::get_CF_string(s);
207 else {
208 s.erase(0,1);
209 return File::get_CF_string(s);
210 }
211 }
212 else {
213 // The leading underscore should be removed from all supported products
214 s.erase(0,1);
215 return File::get_CF_string(s);
216 }
217}
218
219// Retrieve all the HDF5 information.
220void GMFile::Retrieve_H5_Info(const char *file_fullpath,
221 hid_t file_id, bool include_attr) {
222
223 BESDEBUG("h5", "Coming to Retrieve_H5_Info()"<<endl);
224 // GPM needs the attribute info. to obtain the lat/lon.
225 // So set the include_attr to be true for these products.
226 if (product_type == Mea_SeaWiFS_L2 || product_type == Mea_SeaWiFS_L3
227 || GPMS_L3 == product_type || GPMM_L3 == product_type || GPM_L1 == product_type || OBPG_L3 == product_type
228 || Mea_Ozone == product_type || General_Product == product_type)
229 File::Retrieve_H5_Info(file_fullpath,file_id,true);
230 else
231 File::Retrieve_H5_Info(file_fullpath,file_id,include_attr);
232
233}
234
235// Update the product type. This is because the file structure may change across different versions of products
236// I need to handle them differently and still support different versions. The goal is to support two versions in a row.
237// Currently GPM level 3 is changed.
238// This routine should be called right after Retrieve_H5_Info.
240
241 BESDEBUG("h5", "Coming to Update_Product_Type()"<<endl);
242 if(GPMS_L3 == this->product_type || GPMM_L3 == this->product_type) {
243
244 // Check Dimscale attributes
245#if 0
246 //Check_General_Product_Pattern();
247#endif
248 Check_Dimscale_General_Product_Pattern();
249 if(GENERAL_DIMSCALE == this->gproduct_pattern){
250 if(GPMS_L3 == this->product_type) {
251 for (auto irv = this->vars.begin();
252 irv != this->vars.end(); ++irv)
253 (*irv)->newname = (*irv)->name;
254 }
255 this->product_type = General_Product;
256 }
257 }
258 else if(General_Product == this->product_type)
259 Check_General_Product_Pattern();
260}
261
263
264 BESDEBUG("h5", "Coming to Remove_Unneeded_Objects()"<<endl);
265 if(General_Product == this->product_type) {
266 string file_path = this->path;
267 if(HDF5CFUtil::obtain_string_after_lastslash(file_path).find("OMPS-NPP")==0)
268 Remove_OMPSNPP_InputPointers();
269 }
270 if((General_Product == this->product_type) && (GENERAL_DIMSCALE == this->gproduct_pattern)) {
271 set<string> nc4_non_coord_set;
272 string nc4_non_coord="_nc4_non_coord_";
273 size_t nc4_non_coord_size= nc4_non_coord.size();
274 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
275 if((*irv)->name.find(nc4_non_coord)==0)
276 nc4_non_coord_set.insert((*irv)->name.substr(nc4_non_coord_size,(*irv)->name.size()-nc4_non_coord_size));
277
278 }
279
280 for (auto irv = this->vars.begin(); irv != this->vars.end();) {
281 if(nc4_non_coord_set.find((*irv)->name)!=nc4_non_coord_set.end()){
282 delete(*irv);
283 irv=this->vars.erase(irv);
284 }
285 else
286 ++irv;
287 }
288
289 if(nc4_non_coord_set.empty()==false)
290 this->have_nc4_non_coord = true;
291 }
292 else if(GPM_L3_New == this->product_type) {
293 for (auto irg = this->groups.begin(); irg != this->groups.end(); ) {
294 if((*irg)->attrs.empty()) {
295 delete(*irg);
296 irg = this->groups.erase(irg);
297
298 }
299 else
300 ++irg;
301 }
302 }
303}
304
305void GMFile::Remove_OMPSNPP_InputPointers() {
306 // Here I don't check whether this is a netCDF file by
307 // using Check_Dimscale_General_Product_Pattern() to see if it returns true.
308 // We will see if we need this.
309 for (auto irg = this->groups.begin(); irg != this->groups.end(); ) {
310 if((*irg)->path.find("/InputPointers")==0) {
311 delete(*irg);
312 irg = this->groups.erase(irg);
313
314 }
315 else
316 ++irg;
317 }
318
319 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
320 if((*irv)->fullpath.find("/InputPointers")==0) {
321 delete(*irv);
322 irv = this->vars.erase(irv);
323
324 }
325 else
326 ++irv;
327 }
328}
330
331 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
332
333 if ((*ircv)->cvartype != CV_NONLATLON_MISS){
334 for (auto ira = (*ircv)->attrs.begin();
335 ira != (*ircv)->attrs.end(); ++ira) {
336 Retrieve_H5_Attr_Value(*ira,(*ircv)->fullpath);
337 }
338 }
339 }
340}
341
342// Retrieve HDF5 supported attribute values.
344
345 BESDEBUG("h5", "Coming to Retrieve_H5_Supported_Attr_Values()"<<endl);
346
347 // General attributes
349
350 //Coordinate variable attributes
351 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
352 if ((*ircv)->cvartype != CV_NONLATLON_MISS){
353 for (auto ira = (*ircv)->attrs.begin();
354 ira != (*ircv)->attrs.end(); ++ira) {
355 Retrieve_H5_Attr_Value(*ira,(*ircv)->fullpath);
356 }
357 }
358 }
359
360 // Special variable attributes
361 for (auto irspv = this->spvars.begin(); irspv != this->spvars.end(); ++irspv) {
362 for (auto ira = (*irspv)->attrs.begin();
363 ira != (*irspv)->attrs.end(); ++ira) {
364 Retrieve_H5_Attr_Value(*ira,(*irspv)->fullpath);
366 }
367 }
368}
369
370// Adjust attribute values. Currently this is only for ACOS and OCO2.
371// Reason: DAP2 doesn't support 64-bit integer and they have 64-bit integer data
372// in these files. Chop them to two 32-bit integers following the data producer's information.
374
375 BESDEBUG("h5", "Coming to Adjust_H5_Attr_Value()"<<endl);
376 if (product_type == ACOS_L2S_OR_OCO2_L1B) {
377 if (("Type" == attr->name) && (H5VSTRING == attr->dtype)) {
378 string orig_attrvalues(attr->value.begin(),attr->value.end());
379 if (orig_attrvalues != "Signed64") return;
380 string new_attrvalues = "Signed32";
381 // Since the new_attrvalues size is the same as the orig_attrvalues size
382 // No need to adjust the strsize and fstrsize. KY 2011-2-1
383 attr->value.clear();
384 attr->value.resize(new_attrvalues.size());
385 copy(new_attrvalues.begin(),new_attrvalues.end(),attr->value.begin());
386 }
387 }
388}
389
390// Unsupported datatype
391void GMFile:: Handle_Unsupported_Dtype(bool include_attr) {
392
393 BESDEBUG("h5", "Coming to Handle_Unsupported_Dtype()"<<endl);
394 if(true == check_ignored) {
395 Gen_Unsupported_Dtype_Info(include_attr);
396 }
397 File::Handle_Unsupported_Dtype(include_attr);
398 Handle_GM_Unsupported_Dtype(include_attr);
399}
400
401// Unsupported datatype for general data products.
402void GMFile:: Handle_GM_Unsupported_Dtype(bool include_attr) {
403
404 BESDEBUG("h5", "Coming to Handle_GM_Unsupported_Dtype()"<<endl);
405 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ) {
406 if (true == include_attr) {
407 for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ) {
408 H5DataType temp_dtype = (*ira)->getType();
409 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
410 delete (*ira);
411 ira = (*ircv)->attrs.erase(ira);
412 }
413 else {
414 ++ira;
415 }
416 }
417 }
418 H5DataType temp_dtype = (*ircv)->getType();
419 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
420
421 // This may need to be checked carefully in the future,
422 // My current understanding is that the coordinate variable can
423 // be ignored if the corresponding variable is ignored.
424 // Currently we don't find any NASA files in this category.
425 // KY 2012-5-21
426 delete (*ircv);
427 ircv = this->cvars.erase(ircv);
428 }
429 else {
430 ++ircv;
431 }
432
433 }
434
435 for (auto ircv = this->spvars.begin(); ircv != this->spvars.end(); ) {
436 if (true == include_attr) {
437 for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ) {
438 H5DataType temp_dtype = (*ira)->getType();
439 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
440 delete (*ira);
441 ira = (*ircv)->attrs.erase(ira);
442 }
443 else {
444 ++ira;
445 }
446 }
447 }
448 H5DataType temp_dtype = (*ircv)->getType();
449 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
450 delete (*ircv);
451 ircv = this->spvars.erase(ircv);
452 }
453 else {
454 ++ircv;
455 }
456
457 }
458}
459
460// Datatype ignore information.
461void GMFile:: Gen_Unsupported_Dtype_Info(bool include_attr) {
462
463 BESDEBUG("h5", "GMFile::Coming to Gen_Unsupported_Dtype_Info()"<<endl);
464 if(true == include_attr) {
465
466 File::Gen_Group_Unsupported_Dtype_Info();
467 File::Gen_Var_Unsupported_Dtype_Info();
468 Gen_VarAttr_Unsupported_Dtype_Info();
469 }
470
471}
472
473// Datatype ignored information for variable ttributes
474void GMFile:: Gen_VarAttr_Unsupported_Dtype_Info() {
475
476 BESDEBUG("h5", "GMFile::Coming to Gen_Unsupported_Dtype_Info()"<<endl);
477 // First general variables(non-CV and non-special variable) that use dimension scales.
478 if((General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)
479 || (Mea_Ozone == this->product_type) || (Mea_SeaWiFS_L2 == this->product_type) || (Mea_SeaWiFS_L3 == this->product_type)
480 || (OBPG_L3 == this->product_type)) {
481 Gen_DimScale_VarAttr_Unsupported_Dtype_Info();
482 }
483
484 else
485 File::Gen_VarAttr_Unsupported_Dtype_Info();
486
487 // CV and special variables
488 Gen_GM_VarAttr_Unsupported_Dtype_Info();
489
490}
491
492// Generate ignored object,attribute information for the CVs and special variables of general supported products.
493void GMFile:: Gen_GM_VarAttr_Unsupported_Dtype_Info(){
494
495 BESDEBUG("h5", "GMFile::Coming to Gen_GM_VarAttr_Unsupported_Dtype_Info()"<<endl);
496 if((General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)
497 || (Mea_Ozone == this->product_type) || (Mea_SeaWiFS_L2 == this->product_type) || (Mea_SeaWiFS_L3 == this->product_type)
498 || (OBPG_L3 == this->product_type)) {
499
500 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
501 // If the attribute REFERENCE_LIST comes with the attribut CLASS, the
502 // attribute REFERENCE_LIST is okay to ignore. No need to report.
503 bool is_ignored = ignored_dimscale_ref_list((*irv));
504 if (false == (*irv)->attrs.empty()) {
505 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
506 H5DataType temp_dtype = (*ira)->getType();
507 // TODO: check why 64-bit integer is included here.
508 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
509 // "DIMENSION_LIST" is okay to ignore and "REFERENCE_LIST"
510 // is okay to ignore if the variable has another attribute
511 // CLASS="DIMENSION_SCALE"
512 if (("DIMENSION_LIST" !=(*ira)->name) &&
513 ("REFERENCE_LIST" != (*ira)->name || true == is_ignored))
514 this->add_ignored_info_attrs(false,(*irv)->fullpath,(*ira)->name);
515 }
516 }
517 }
518 }
519
520 for (auto irv = this->spvars.begin(); irv != this->spvars.end(); ++irv) {
521 // If the attribute REFERENCE_LIST comes with the attribut CLASS, the
522 // attribute REFERENCE_LIST is okay to ignore. No need to report.
523 bool is_ignored = ignored_dimscale_ref_list((*irv));
524 if (false == (*irv)->attrs.empty()) {
525#if 0
526 //if (true == (*irv)->unsupported_attr_dtype)
527#endif
528 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
529 H5DataType temp_dtype = (*ira)->getType();
530 //TODO; check why 64-bit integer is included here.
531 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
532 // "DIMENSION_LIST" is okay to ignore and "REFERENCE_LIST"
533 // is okay to ignore if the variable has another attribute
534 // CLASS="DIMENSION_SCALE"
535 if (("DIMENSION_LIST" !=(*ira)->name) &&
536 ("REFERENCE_LIST" != (*ira)->name || true == is_ignored))
537 this->add_ignored_info_attrs(false,(*irv)->fullpath,(*ira)->name);
538 }
539 }
540 }
541 }
542 }// "if((General_Product == ......)"
543 else {
544 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
545 if (false == (*irv)->attrs.empty()) {
546#if 0
547 //if (true == (*irv)->unsupported_attr_dtype)
548#endif
549 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
550 H5DataType temp_dtype = (*ira)->getType();
551 // TODO: check why 64-bit integer is included here.
552 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4) || temp_dtype == H5INT64 || temp_dtype == H5UINT64)
553 this->add_ignored_info_attrs(false,(*irv)->fullpath,(*ira)->name);
554 }
555 }
556 }
557
558 for (auto irv = this->spvars.begin(); irv != this->spvars.end(); ++irv) {
559 if (false == (*irv)->attrs.empty()) {
560#if 0
561 //if (true == (*irv)->unsupported_attr_dtype)
562#endif
563 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
564 H5DataType temp_dtype = (*ira)->getType();
565 //TODO: check why 64-bit integer is included here.
566 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
567 this->add_ignored_info_attrs(false,(*irv)->fullpath,(*ira)->name);
568 }
569 }
570 }
571 }
572
573 }// else
574
575}
576
577// Unsupported data space
578void GMFile:: Handle_Unsupported_Dspace(bool include_attr) {
579
580 BESDEBUG("h5", "Coming to GMFile:Handle_Unsupported_Dspace()"<<endl);
581 if(true == check_ignored)
582 Gen_Unsupported_Dspace_Info();
583
585 Handle_GM_Unsupported_Dspace(include_attr);
586
587}
588
589// Unsupported data space for coordinate variables and special variables of general products
590void GMFile:: Handle_GM_Unsupported_Dspace(bool include_attr) {
591
592 BESDEBUG("h5", "Coming to GMFile:Handle_GM_Unsupported_Dspace()"<<endl);
593 if(true == this->unsupported_var_dspace) {
594 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ) {
595 if (true == (*ircv)->unsupported_dspace ) {
596
597 // This may need to be checked carefully in the future,
598 // My current understanding is that the coordinate variable can
599 // be ignored if the corresponding variable is ignored.
600 // Currently we don't find any NASA files in this category.
601 // KY 2012-5-21
602 delete (*ircv);
603 ircv = this->cvars.erase(ircv);
604 }
605 else {
606 ++ircv;
607 }
608 }
609
610 for (auto ircv = this->spvars.begin(); ircv != this->spvars.end(); ) {
611 if (true == (*ircv)->unsupported_dspace) {
612 delete (*ircv);
613 ircv = this->spvars.erase(ircv);
614 }
615 else {
616 ++ircv;
617 }
618
619 }
620 }// if
621
622 if(true == include_attr) {
623 if(true == this->unsupported_var_attr_dspace) {
624 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
625 if (false == (*ircv)->attrs.empty()) {
626 if (true == (*ircv)->unsupported_attr_dspace) {
627 for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ) {
628 if (0 == (*ira)->count) {
629 delete (*ira);
630 ira = (*ircv)->attrs.erase(ira);
631 }
632 else {
633 ++ira;
634 }
635 }
636 }
637 }
638 }
639
640 for (auto ircv = this->spvars.begin(); ircv != this->spvars.end(); ++ircv) {
641 if (false == (*ircv)->attrs.empty()) {
642 if (true == (*ircv)->unsupported_attr_dspace) {
643 for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ) {
644 if (0 == (*ira)->count) {
645 delete (*ira);
646 ira = (*ircv)->attrs.erase(ira);
647 }
648 else {
649 ++ira;
650 }
651 }
652 }
653 }
654 } // for
655 }// if
656 }// if
657
658}
659
660// Generate unsupported data space information
661void GMFile:: Gen_Unsupported_Dspace_Info() {
662
663 File::Gen_Unsupported_Dspace_Info();
664
665}
666
667// Handle other unsupported objects
668void GMFile:: Handle_Unsupported_Others(bool include_attr) {
669
670 BESDEBUG("h5", "Coming to GMFile:Handle_Unsupported_Others()"<<endl);
672
673 // Add the removal of CLASS=DIM_SCALE attribute if this is a netCDF-4-like attribute.
674 //
675 if(General_Product != this->product_type
676 || (General_Product == this->product_type && OTHERGMS != this->gproduct_pattern)){
677 //
678#if 0
679 if((General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)
680 || (Mea_Ozone == this->product_type) || (Mea_SeaWiFS_L2 == this->product_type)
681 || (Mea_SeaWiFS_L3 == this->product_type)
682 || (OBPG_L3 == this->product_type))
683#endif
684 remove_netCDF_internal_attributes(include_attr);
685 if(include_attr == true) {
686 // We also need to remove the _nc3_strict from the root attributes
687 for (auto ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
688
689 if((*ira)->name == "_nc3_strict") {
690 delete(*ira);
691 ira =this->root_attrs.erase(ira);
692 //If we have other root attributes to remove, remove the break statement.
693 }
694 else if((*ira)->name == "_NCProperties") {
695 delete(*ira);
696 ira =this->root_attrs.erase(ira);
697 }
698 else if((*ira)->name == "_Netcdf4Coordinates") {
699 delete(*ira);
700 ira =this->root_attrs.erase(ira);
701 }
702
703 else {
704 ++ira;
705 }
706 }
707 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
708 for(auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
709 if((*ira)->name == "CLASS") {
710 string class_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
711
712 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
713 // "DIMENSION_SCALE", which is 15.
714 if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
715 delete(*ira);
716 ira = (*irv)->attrs.erase(ira);
717 // Add another block to set a key
718 }
719 else {
720 ++ira;
721 }
722 }
723 else if((*ira)->name == "NAME") {// Add a BES Key later
724 delete(*ira);
725 ira =(*irv)->attrs.erase(ira);
726 //"NAME" attribute causes the file netCDF-4 failed.
727#if 0
728
729 string name_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
730 if( 0 == name_value.compare(0,(*irv)->name.size(),(*irv)->name)) {
731 delete(*ira);
732 ira =(*irv)->attrs.erase(ira);
733 }
734 else {
735 string netcdf_dim_mark= "This is a netCDF dimension but not a netCDF variable";
736 if( 0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark)) {
737 delete((*ira));
738 ira =(*irv)->attrs.erase(ira);
739 }
740 else {
741 ++ira;
742 }
743 }
744#endif
745 }
746 else if((*ira)->name == "_Netcdf4Dimid") {
747 delete(*ira);
748 ira =(*irv)->attrs.erase(ira);
749 }
750 else if((*ira)->name == "_Netcdf4Coordinates") {
751 delete(*ira);
752 ira =(*irv)->attrs.erase(ira);
753 }
754
755#if 0
756 else if((*ira)->name == "_nc3_strict") {
757 delete((*ira));
758 ira =(*irv)->attrs.erase(ira);
759 }
760#endif
761 else {
762 ++ira;
763 }
764 }
765 }
766 }
767 }
768 // netCDF Java lifts the string size limitation. All the string attributes can be
769 // represented by netCDF Java. So comment out the code. KY 2018/08/10
770#if 0
771 if(true == this->check_ignored && true == include_attr) {
772 if(true == HDF5RequestHandler::get_drop_long_string()){
773 for (vector<GMCVar *>::iterator irv = this->cvars.begin();
774 irv != this->cvars.end(); ++irv) {
775 for(vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
776 ira != (*irv)->attrs.end();++ira) {
777 if(true == Check_DropLongStr((*irv),(*ira))) {
778 this->add_ignored_droplongstr_hdr();
779 this->add_ignored_var_longstr_info((*irv),(*ira));
780 }
781 }
782 }
783
784 for (vector<GMSPVar *>::iterator irv = this->spvars.begin();
785 irv != this->spvars.end(); ++irv) {
786 for(vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
787 ira != (*irv)->attrs.end();++ira) {
788 if(true == Check_DropLongStr((*irv),(*ira))) {
789 this->add_ignored_droplongstr_hdr();
790 this->add_ignored_var_longstr_info((*irv),(*ira));
791 }
792 }
793
794 }
795 }
796 }
797#endif
798
799 if(false == this->have_ignored)
800 this->add_no_ignored_info();
801
802}
803
804// Add dimension names
806
807 BESDEBUG("h5", "Coming to GMFile:Add_Dim_Name()"<<endl);
808 switch(product_type) {
809 case Mea_SeaWiFS_L2:
810 case Mea_SeaWiFS_L3:
811 Add_Dim_Name_Mea_SeaWiFS();
812 break;
813 case Aqu_L3:
814 Add_Dim_Name_Aqu_L3();
815 break;
816 case OSMAPL2S:
817 Add_Dim_Name_OSMAPL2S();
818 break;
819 case ACOS_L2S_OR_OCO2_L1B:
820 Add_Dim_Name_ACOS_L2S_OCO2_L1B();
821 break;
822 case Mea_Ozone:
823 Add_Dim_Name_Mea_Ozonel3z();
824 break;
825 case GPMS_L3:
826 case GPMM_L3:
827 case GPM_L1:
828 case GPM_L3_New:
829 Add_Dim_Name_GPM();
830 break;
831 case OBPG_L3:
832 Add_Dim_Name_OBPG_L3();
833 break;
834 case General_Product:
835 Add_Dim_Name_General_Product();
836 break;
837 default:
838 throw1("Cannot generate dim. names for unsupported datatype");
839 }
840
841// Just for debugging
842#if 0
843for (vector<Var*>::iterator irv2 = this->vars.begin();
844 irv2 != this->vars.end(); irv2++) {
845 for (vector<Dimension *>::iterator ird = (*irv2)->dims.begin();
846 ird !=(*irv2)->dims.end(); ird++) {
847 cerr<<"Dimension name afet Add_Dim_Name "<<(*ird)->newname <<endl;
848 }
849}
850#endif
851
852}
853
854//Add Dim. Names for OBPG level 3 product
855void GMFile::Add_Dim_Name_OBPG_L3() {
856
857 BESDEBUG("h5", "Coming to Add_Dim_Name_OBPG_L3()"<<endl);
858 // netCDF-4 like structure
859 // Note: We need to change the product type to netCDF-4 like product type and pattern.
860 Check_General_Product_Pattern();
861 Add_Dim_Name_General_Product();
862}
863
864//Add Dim. Names for MeaSures SeaWiFS. Future: May combine with the handling of netCDF-4 products
865void GMFile::Add_Dim_Name_Mea_SeaWiFS() {
866
867 BESDEBUG("h5", "Coming to Add_Dim_Name_Mea_SeaWiFS()"<<endl);
868 pair<set<string>::iterator,bool> setret;
869 if (Mea_SeaWiFS_L3 == product_type)
870 iscoard = true;
871 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
872 Handle_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone((*irv));
873 for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
874 setret = dimnamelist.insert((*ird)->name);
875 if (true == setret.second)
876 Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size,(*ird)->unlimited_dim);
877 }
878 }
879
880 if (true == dimnamelist.empty())
881 throw1("This product should have the dimension names, but no dimension names are found");
882}
883
884// Handle Dimension scales for MEasUREs SeaWiFS and OZone.
885void GMFile::Handle_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone(Var* var)
886{
887
888 BESDEBUG("h5", "Coming to Handle_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone()"<<endl);
889 Attribute* dimlistattr = nullptr;
890 bool has_dimlist = false;
891 bool has_class = false;
892 bool has_reflist = false;
893
894 for(auto ira = var->attrs.begin(); ira != var->attrs.end();ira++) {
895 if ("DIMENSION_LIST" == (*ira)->name) {
896 dimlistattr = *ira;
897 has_dimlist = true;
898 }
899 if ("CLASS" == (*ira)->name)
900 has_class = true;
901 if ("REFERENCE_LIST" == (*ira)->name)
902 has_reflist = true;
903
904 if (true == has_dimlist)
905 break;
906 if (true == has_class && true == has_reflist)
907 break;
908 }
909
910 if (true == has_dimlist)
911 Add_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone(var,dimlistattr);
912
913 // Dim name is the same as the variable name for dimscale variable
914 else if(true == has_class && true == has_reflist) {
915 if (var->dims.size() !=1)
916 throw2("dimension scale dataset must be 1 dimension, this is not true for variable ",
917 var->name);
918
919 // The var name is the object name, however, we would like the dimension name to be full path.
920 // so that the dim name can be served as the key for future handling.
921 (var->dims)[0]->name = var->fullpath;
922 (var->dims)[0]->newname = var->fullpath;
923 pair<set<string>::iterator,bool> setret;
924 setret = dimnamelist.insert((var->dims)[0]->name);
925 if (true == setret.second)
926 Insert_One_NameSizeMap_Element((var->dims)[0]->name,(var->dims)[0]->size,(var->dims)[0]->unlimited_dim);
927 }
928
929 // No dimension, add fake dim names, this may never happen for MeaSure
930 // but just for coherence and completeness.
931 // For Fake dimesnion
932 else {
933
934 set<hsize_t> fakedimsize;
935 pair<set<hsize_t>::iterator,bool> setsizeret;
936 for (auto ird= var->dims.begin(); ird != var->dims.end(); ++ird) {
937 Add_One_FakeDim_Name(*ird);
938 setsizeret = fakedimsize.insert((*ird)->size);
939 if (false == setsizeret.second)
940 Adjust_Duplicate_FakeDim_Name(*ird);
941 }
942// Just for debugging
943#if 0
944 for (int i = 0; i < var->dims.size(); ++i) {
945 Add_One_FakeDim_Name((var->dims)[i]);
946 bool gotoMainLoop = false;
947 for (int j =i-1; j>=0 && !gotoMainLoop; --j) {
948 if (((var->dims)[i])->size == ((var->dims)[j])->size){
949 Adjust_Duplicate_FakeDim_Name((var->dims)[i]);
950 gotoMainLoop = true;
951 }
952 }
953 }
954#endif
955
956 }//end of else
957}
958
959// Helper function to support dimensions of MeaSUrES SeaWiFS and OZone products
960void GMFile::Add_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone(Var *var,const Attribute*dimlistattr)
961{
962
963 BESDEBUG("h5", "Coming to Add_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone()"<<endl);
964 ssize_t objnamelen = -1;
965 hobj_ref_t rbuf;
966
967 vector<hvl_t> vlbuf;
968
969 hid_t dset_id = -1;
970 hid_t attr_id = -1;
971 hid_t atype_id = -1;
972 hid_t amemtype_id = -1;
973 hid_t aspace_id = -1;
974 hid_t ref_dset = -1;
975
976
977 if(nullptr == dimlistattr)
978 throw2("Cannot obtain the dimension list attribute for variable ",var->name);
979
980 if (0==var->rank)
981 throw2("The number of dimension should NOT be 0 for the variable ",var->name);
982
983 try {
984
985 vlbuf.resize(var->rank);
986
987 dset_id = H5Dopen(this->fileid,(var->fullpath).c_str(),H5P_DEFAULT);
988 if (dset_id < 0)
989 throw2("Cannot open the dataset ",var->fullpath);
990
991 attr_id = H5Aopen(dset_id,(dimlistattr->name).c_str(),H5P_DEFAULT);
992 if (attr_id <0 )
993 throw4("Cannot open the attribute ",dimlistattr->name," of HDF5 dataset ",var->fullpath);
994
995 atype_id = H5Aget_type(attr_id);
996 if (atype_id <0)
997 throw4("Cannot obtain the datatype of the attribute ",dimlistattr->name," of HDF5 dataset ",var->fullpath);
998
999 amemtype_id = H5Tget_native_type(atype_id, H5T_DIR_ASCEND);
1000
1001 if (amemtype_id < 0)
1002 throw2("Cannot obtain the memory datatype for the attribute ",dimlistattr->name);
1003
1004
1005 if (H5Aread(attr_id,amemtype_id,vlbuf.data()) <0)
1006 throw2("Cannot obtain the referenced object for the variable ",var->name);
1007
1008
1009 vector<char> objname;
1010 int vlbuf_index = 0;
1011
1012 // The dimension names of variables will be the HDF5 dataset names dereferenced from the DIMENSION_LIST attribute.
1013 for (auto ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
1014
1015 if(vlbuf[vlbuf_index].p== nullptr)
1016 throw4("The dimension doesn't exist. Var name is ",var->name,"; the dimension index is ",vlbuf_index);
1017
1018 rbuf =((hobj_ref_t*)vlbuf[vlbuf_index].p)[0];
1019 if ((ref_dset = H5RDEREFERENCE(attr_id, H5R_OBJECT, &rbuf)) < 0)
1020 throw2("Cannot dereference from the DIMENSION_LIST attribute for the variable ",var->name);
1021 // The above code works with dereferencing references generated with new H5R(H5R_ref_t) APIs with 1.12 and 1.13.
1022 // However, in case this h5rdeference2 API stops working with new APIs, the following #if 0 #endif block is a
1023 // way to handle this issue.
1024#if 0
1025
1026 rbuf =((hobj_ref_t*)vl_ref[0].p)[0];
1027 H5E_BEGIN_TRY {
1028 dset1 = H5Rdereference2(attr_id,H5P_DEFAULT,H5R_OBJECT,&ds_ref_buf);
1029 } H5E_END_TRY;
1030
1031 H5R_ref_t new_rbuf =((H5R_ref_t*)vlbuf[vlbuf_index].p)[0];
1032 if ((ref_dset = H5Ropen_object((H5R_ref_t *)&new_rbuf, H5P_DEFAULT, H5P_DEFAULT))<0)
1033 throw2("Cannot dereference from the DIMENSION_LIST attribute for the variable ",var->name);
1034 H5Rdestroy(&new_rbuf);
1035
1036#endif
1037 if ((objnamelen= H5Iget_name(ref_dset,nullptr,0))<=0)
1038 throw2("Cannot obtain the dataset name dereferenced from the DIMENSION_LIST attribute for the variable ",var->name);
1039 objname.resize(objnamelen+1);
1040 if ((objnamelen= H5Iget_name(ref_dset,objname.data(),objnamelen+1))<=0)
1041 throw2("Cannot obtain the dataset name dereferenced from the DIMENSION_LIST attribute for the variable ",var->name);
1042
1043 auto objname_str = string(objname.begin(),objname.end());
1044 string trim_objname = objname_str.substr(0,objnamelen);
1045 (*ird)->name = string(trim_objname.begin(),trim_objname.end());
1046
1047 pair<set<string>::iterator,bool> setret;
1048 setret = dimnamelist.insert((*ird)->name);
1049 if (true == setret.second)
1050 Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size,(*ird)->unlimited_dim);
1051 (*ird)->newname = (*ird)->name;
1052 H5Dclose(ref_dset);
1053 objname.clear();
1054 vlbuf_index++;
1055 }// end of for
1056
1057 if(vlbuf.size()!= 0) {
1058
1059 if ((aspace_id = H5Aget_space(attr_id)) < 0)
1060 throw2("Cannot get hdf5 dataspace id for the attribute ",dimlistattr->name);
1061
1062 if (H5Dvlen_reclaim(amemtype_id,aspace_id,H5P_DEFAULT,(void*)vlbuf.data())<0)
1063 throw2("Cannot successfully clean up the variable length memory for the variable ",var->name);
1064
1065 H5Sclose(aspace_id);
1066
1067 }
1068
1069 H5Tclose(atype_id);
1070 H5Tclose(amemtype_id);
1071 H5Aclose(attr_id);
1072 H5Dclose(dset_id);
1073
1074 }
1075
1076 catch(...) {
1077
1078 if(atype_id != -1)
1079 H5Tclose(atype_id);
1080
1081 if(amemtype_id != -1)
1082 H5Tclose(amemtype_id);
1083
1084 if(aspace_id != -1)
1085 H5Sclose(aspace_id);
1086
1087 if(attr_id != -1)
1088 H5Aclose(attr_id);
1089
1090 if(dset_id != -1)
1091 H5Dclose(dset_id);
1092
1093 throw;
1094 }
1095
1096}
1097
1098// Add MeaSURES OZone level 3Z dimension names
1099void GMFile::Add_Dim_Name_Mea_Ozonel3z() {
1100
1101 BESDEBUG("h5", "Coming to Add_Dim_Name_Mea_Ozonel3z()"<<endl);
1102 iscoard = true;
1103 bool use_dimscale = false;
1104
1105 for (auto irg = this->groups.begin(); irg != this->groups.end(); ++ irg) {
1106 if ("/Dimensions" == (*irg)->path) {
1107 use_dimscale = true;
1108 break;
1109 }
1110 }
1111
1112 if (false == use_dimscale) {
1113
1114 bool has_dimlist = false;
1115 bool has_class = false;
1116 bool has_reflist = false;
1117
1118 for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
1119 for(auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();ira++) {
1120 if ("DIMENSION_LIST" == (*ira)->name)
1121 has_dimlist = true;
1122 }
1123 if (true == has_dimlist)
1124 break;
1125 }
1126
1127 if (true == has_dimlist) {
1128 for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
1129
1130 for(auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();ira++) {
1131 if ("CLASS" == (*ira)->name)
1132 has_class = true;
1133 if ("REFERENCE_LIST" == (*ira)->name)
1134 has_reflist = true;
1135 if (true == has_class && true == has_reflist)
1136 break;
1137 }
1138
1139 if (true == has_class &&
1140 true == has_reflist)
1141 break;
1142
1143 }
1144 if (true == has_class && true == has_reflist)
1145 use_dimscale = true;
1146 } // end of if (true == has_dimlist)
1147 } // end of if (false == use_dimscale)
1148
1149 if (true == use_dimscale) {
1150
1151 pair<set<string>::iterator,bool> setret;
1152 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1153 Handle_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone((*irv));
1154 for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
1155 setret = dimnamelist.insert((*ird)->name);
1156 if(true == setret.second)
1157 Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size,(*ird)->unlimited_dim);
1158 }
1159 }
1160
1161 if (true == dimnamelist.empty())
1162 throw1("This product should have the dimension names, but no dimension names are found");
1163 } // end of if (true == use_dimscale)
1164
1165 else {
1166
1167 // Since the dim. size of each dimension of 2D lat/lon may be the same, so use multimap.
1168 multimap<hsize_t,string> ozonedimsize_to_dimname;
1169 pair<multimap<hsize_t,string>::iterator,multimap<hsize_t,string>::iterator> mm_er_ret;
1170 multimap<hsize_t,string>::iterator irmm;
1171
1172 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1173 bool is_cv = check_cv((*irv)->name);
1174 if (true == is_cv) {
1175 if ((*irv)->dims.size() != 1)
1176 throw3("The coordinate variable", (*irv)->name," must be one dimension for the zonal average product");
1177 ozonedimsize_to_dimname.insert(pair<hsize_t,string>(((*irv)->dims)[0]->size,(*irv)->fullpath));
1178 }
1179 }// end of for
1180
1181 set<hsize_t> fakedimsize;
1182 pair<set<hsize_t>::iterator,bool> setsizeret;
1183 pair<set<string>::iterator,bool> setret;
1184 pair<set<string>::iterator,bool> tempsetret;
1185 set<string> tempdimnamelist;
1186 bool fakedimflag = false;
1187
1188 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1189
1190 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
1191
1192 fakedimflag = true;
1193 mm_er_ret = ozonedimsize_to_dimname.equal_range((*ird)->size);
1194 for (irmm = mm_er_ret.first; irmm!=mm_er_ret.second;irmm++) {
1195 setret = tempdimnamelist.insert(irmm->second);
1196 if (true == setret.second) {
1197 (*ird)->name = irmm->second;
1198 (*ird)->newname = (*ird)->name;
1199 setret = dimnamelist.insert((*ird)->name);
1200 if(setret.second) Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size,(*ird)->unlimited_dim);
1201 fakedimflag = false;
1202 break;
1203 }
1204 }
1205
1206 if (true == fakedimflag) {
1207 Add_One_FakeDim_Name(*ird);
1208 setsizeret = fakedimsize.insert((*ird)->size);
1209 if (false == setsizeret.second)
1210 Adjust_Duplicate_FakeDim_Name(*ird);
1211 }
1212
1213 } // end of for
1214 tempdimnamelist.clear();
1215 fakedimsize.clear();
1216 } // end of for
1217 } // end of else
1218}
1219
1220// This is a special helper function for MeaSURES ozone products
1221bool GMFile::check_cv(const string & varname) const {
1222
1223 BESDEBUG("h5", "Coming to check_cv()"<<endl);
1224 const string lat_name ="Latitude";
1225 const string time_name ="Time";
1226 const string ratio_pressure_name ="MixingRatioPressureLevels";
1227 const string profile_pressure_name ="ProfilePressureLevels";
1228 const string wave_length_name ="Wavelength";
1229
1230 if (lat_name == varname)
1231 return true;
1232 else if (time_name == varname)
1233 return true;
1234 else if (ratio_pressure_name == varname)
1235 return true;
1236 else if (profile_pressure_name == varname)
1237 return true;
1238 else if (wave_length_name == varname)
1239 return true;
1240 else
1241 return false;
1242}
1243
1244// Add Dimension names for GPM products
1245void GMFile::Add_Dim_Name_GPM()
1246{
1247
1248 BESDEBUG("h5", "Coming to Add_Dim_Name_GPM()"<<endl);
1249 // This is used to create a dimension name set.
1250 pair<set<string>::iterator,bool> setret;
1251
1252 // The commented code is for an old version of GPM products. May remove them later. KY 2015-06-16
1253 // We need to create a fakedim name to fill in. To make the dimension name unique, we use a counter.
1254#if 0
1255 // int dim_count = 0;
1256 // map<string,string> varname_to_fakedim;
1257 // map<int,string> gpm_dimsize_to_fakedimname;
1258#endif
1259
1260 // We find that GPM has an attribute DimensionNames(nlon,nlat) in this case.
1261 // We will use this attribute to specify the dimension names.
1262 for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
1263
1264 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1265
1266 if("DimensionNames" == (*ira)->name) {
1267
1268 Retrieve_H5_Attr_Value(*ira,(*irv)->fullpath);
1269 string dimname_value((*ira)->value.begin(),(*ira)->value.end());
1270
1271 vector<string> ind_elems;
1272 char sep=',';
1273 HDF5CFUtil::Split(dimname_value.data(),sep,ind_elems);
1274
1275 if(ind_elems.size() != (size_t)((*irv)->getRank())) {
1276 throw2("The number of dims obtained from the <DimensionNames> attribute is not equal to the rank ",
1277 (*irv)->name);
1278 }
1279
1280 for(unsigned int i = 0; i<ind_elems.size(); ++i) {
1281
1282 ((*irv)->dims)[i]->name = ind_elems[i];
1283
1284 // Generate a dimension name if the dimension name is missing.
1285 // The routine will ensure that the fakeDim name is unique.
1286 if(((*irv)->dims)[i]->name==""){
1287 Add_One_FakeDim_Name(((*irv)->dims)[i]);
1288// For debugging
1289#if 0
1290 string fakedim = "FakeDim";
1291 stringstream sdim_count;
1292 sdim_count << dim_count;
1293 fakedim = fakedim + sdim_count.str();
1294 dim_count++;
1295 ((*irv)->dims)[i]->name = fakedim;
1296 ((*irv)->dims)[i]->newname = fakedim;
1297 ind_elems[i] = fakedim;
1298#endif
1299 }
1300
1301 else {
1302 ((*irv)->dims)[i]->newname = ind_elems[i];
1303 setret = dimnamelist.insert(((*irv)->dims)[i]->name);
1304
1305 if (true == setret.second) {
1306 Insert_One_NameSizeMap_Element(((*irv)->dims)[i]->name,
1307 ((*irv)->dims)[i]->size,
1308 ((*irv)->dims)[i]->unlimited_dim);
1309 }
1310 else {
1311 if(dimname_to_dimsize[((*irv)->dims)[i]->name] !=((*irv)->dims)[i]->size)
1312 throw5("Dimension ",((*irv)->dims)[i]->name, "has two sizes",
1313 ((*irv)->dims)[i]->size,dimname_to_dimsize[((*irv)->dims)[i]->name]);
1314
1315 }
1316 }
1317
1318 }// end of for
1319 break;
1320
1321 } //end of if
1322 } //end of for
1323
1324#if 0
1325 if(false == has_dim_name_attr) {
1326
1327 throw4( "The variable ", (*irv)->name, " doesn't have the DimensionNames attribute.",
1328 "We currently don't support this case. Please report to the NASA data center.");
1329 }
1330
1331#endif
1332 } //end of for
1333
1334}
1335
1336// Add Dimension names for Aquarius level 3 products
1337void GMFile::Add_Dim_Name_Aqu_L3()
1338{
1339 BESDEBUG("h5", "Coming to Add_Dim_Name_Aqu_L3()"<<endl);
1340 for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
1341 if ("l3m_data" == (*irv)->name) {
1342 ((*irv)->dims)[0]->name = "lat";
1343 ((*irv)->dims)[0]->newname = "lat";
1344 ((*irv)->dims)[1]->name = "lon";
1345 ((*irv)->dims)[1]->newname = "lon";
1346 break;
1347 }
1348
1349// For the time being, don't assign dimension names to palette,
1350// we will see if tools can pick up l3m and then make decisions.
1351#if 0
1352 if ("palette" == (*irv)->name) {
1353//"h5","coming to palette" <<endl;
1354 ((*irv)->dims)[0]->name = "paldim0";
1355 ((*irv)->dims)[0]->newname = "paldim0";
1356 ((*irv)->dims)[1]->name = "paldim1";
1357 ((*irv)->dims)[1]->newname = "paldim1";
1358 }
1359#endif
1360
1361 }// end of for
1362}
1363
1364// Add dimension names for OSMAPL2S(note: the SMAP change their structures. The code doesn't not apply to them.)
1365void GMFile::Add_Dim_Name_OSMAPL2S(){
1366
1367 BESDEBUG("h5", "Coming to Add_Dim_Name_OSMAPL2S()"<<endl);
1368 string tempvarname ="";
1369 string key = "_lat";
1370 string osmapl2sdim0 ="YDim";
1371 string osmapl2sdim1 ="XDim";
1372
1373 // Since the dim. size of each dimension of 2D lat/lon may be the same, so use multimap.
1374 multimap<hsize_t,string> osmapl2sdimsize_to_dimname;
1375 pair<multimap<hsize_t,string>::iterator,multimap<hsize_t,string>::iterator> mm_er_ret;
1376 multimap<hsize_t,string>::iterator irmm;
1377
1378 // Generate dimension names based on the size of "???_lat"(one coordinate variable)
1379 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1380 tempvarname = (*irv)->name;
1381 if ((tempvarname.size() > key.size())&&
1382 (key == tempvarname.substr(tempvarname.size()-key.size(),key.size()))){
1383 if ((*irv)->dims.size() !=2)
1384 throw1("Currently only 2D lat/lon is supported for OSMAPL2S");
1385 osmapl2sdimsize_to_dimname.insert(pair<hsize_t,string>(((*irv)->dims)[0]->size,osmapl2sdim0));
1386 osmapl2sdimsize_to_dimname.insert(pair<hsize_t,string>(((*irv)->dims)[1]->size,osmapl2sdim1));
1387 break;
1388 }
1389 }
1390
1391 set<hsize_t> fakedimsize;
1392 pair<set<hsize_t>::iterator,bool> setsizeret;
1393 pair<set<string>::iterator,bool> setret;
1394 pair<set<string>::iterator,bool> tempsetret;
1395 set<string> tempdimnamelist;
1396 bool fakedimflag = false;
1397
1398
1399 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1400 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
1401
1402 fakedimflag = true;
1403 mm_er_ret = osmapl2sdimsize_to_dimname.equal_range((*ird)->size);
1404 for (irmm = mm_er_ret.first; irmm!=mm_er_ret.second;irmm++) {
1405 setret = tempdimnamelist.insert(irmm->second);
1406 if (setret.second) {
1407 (*ird)->name = irmm->second;
1408 (*ird)->newname = (*ird)->name;
1409 setret = dimnamelist.insert((*ird)->name);
1410 if(setret.second) Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size,(*ird)->unlimited_dim);
1411 fakedimflag = false;
1412 break;
1413 }
1414 }
1415
1416 if (true == fakedimflag) {
1417 Add_One_FakeDim_Name(*ird);
1418 setsizeret = fakedimsize.insert((*ird)->size);
1419 if (!setsizeret.second)
1420 Adjust_Duplicate_FakeDim_Name(*ird);
1421 }
1422 } // end of for ird
1423 tempdimnamelist.clear();
1424 fakedimsize.clear();
1425 } // end of for irv
1426}
1427
1428//Add dimension names for ACOS level2S or OCO2 level1B products
1429void GMFile::Add_Dim_Name_ACOS_L2S_OCO2_L1B(){
1430
1431 BESDEBUG("h5", "Coming to Add_Dim_Name_ACOS_L2S_OCO2_L1B()"<<endl);
1432 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1433 set<hsize_t> fakedimsize;
1434 pair<set<hsize_t>::iterator,bool> setsizeret;
1435 for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
1436 Add_One_FakeDim_Name(*ird);
1437 setsizeret = fakedimsize.insert((*ird)->size);
1438 if (false == setsizeret.second)
1439 Adjust_Duplicate_FakeDim_Name(*ird);
1440 }
1441 } // end of for irv
1442}
1443
1444// Add dimension names for general products. Read the descrption of Check_General_Product_Pattern() for different patterns we support.
1445void GMFile::Add_Dim_Name_General_Product(){
1446
1447 BESDEBUG("h5", "Coming to Add_Dim_Name_General_Product()"<<endl);
1448
1449 // This general product should follow the HDF5 dimension scale model.
1450 if (GENERAL_DIMSCALE == this->gproduct_pattern){
1451 Add_Dim_Name_Dimscale_General_Product();
1452}
1453 // This general product has 2-D latitude,longitude
1454 else if (GENERAL_LATLON2D == this->gproduct_pattern)
1455 Add_Dim_Name_LatLon2D_General_Product();
1456 // This general product has 1-D latitude,longitude
1457 else if (GENERAL_LATLON1D == this->gproduct_pattern || GENERAL_LATLON_COOR_ATTR == this->gproduct_pattern)
1458 Add_Dim_Name_LatLon1D_Or_CoordAttr_General_Product();
1459
1460
1461}
1462
1463// We check four patterns under the General_Product category
1464// 1. General products that uses HDF5 dimension scales following netCDF-4 data model
1465// 2. General products that have 2-D lat/lon variables(lat/lon variable names are used to identify the case) under the root group or
1466// a special geolocation group
1467// 3. General products that have 1-D lat/lon variables(lat/lon variable names are used to identify the case) under the root group or
1468// a special geolocation group
1469// 4. General products that have some variables containing CF "coordinates" attributes. We can support some products if the "coordinates"
1470// attribute contains CF lat/lon units and the variable ranks are 2 or 1.
1471void GMFile::Check_General_Product_Pattern() {
1472
1473 BESDEBUG("h5", "Coming to Check_General_Product_Pattern()"<<endl);
1474 if (false == Check_Dimscale_General_Product_Pattern()) {
1475 //HERE add a check for the GPM. (choose 5 variables equally distance for the attribute)
1476 if (false == Check_And_Update_New_GPM_L3())
1477 if (false == Check_LatLon2D_General_Product_Pattern())
1478 if (false == Check_LatLon1D_General_Product_Pattern())
1479 Check_LatLon_With_Coordinate_Attr_General_Product_Pattern();
1480 }
1481
1482}
1483
1484// Check if this general product is netCDF4-like HDF5 file.
1485// We only need to check "DIMENSION_LIST","CLASS" and CLASS values.
1486bool GMFile::Check_Dimscale_General_Product_Pattern() {
1487
1488 BESDEBUG("h5", "Coming to Check_Dimscale_General_Product_Pattern()"<<endl);
1489 bool ret_value = false;
1490 bool has_dimlist = false;
1491 bool has_dimscalelist = false;
1492
1493 // Check if containing the "DIMENSION_LIST" attribute;
1494 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1495 for(auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();ira++) {
1496 if ("DIMENSION_LIST" == (*ira)->name) {
1497 has_dimlist = true;
1498 break;
1499 }
1500 }
1501 if (true == has_dimlist)
1502 break;
1503 }
1504
1505 // Check if containing both the attribute "CLASS" and the attribute "REFERENCE_LIST" for the same variable.
1506 // This is the dimension scale.
1507 // Actually "REFERENCE_LIST" is not necessary for a dimension scale dataset. If a dimension scale doesn't
1508 // have a "REFERENCE_LIST", it is still valid. But no other variables use this dimension scale. We found
1509 // such a case in a matched_airs_aqua product. KY 2012-12-03
1510 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1511 for(auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();ira++) {
1512 if ("CLASS" == (*ira)->name) {
1513
1514 Retrieve_H5_Attr_Value(*ira,(*irv)->fullpath);
1515 string class_value;
1516 class_value.resize((*ira)->value.size());
1517 copy((*ira)->value.begin(),(*ira)->value.end(),class_value.begin());
1518
1519 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
1520 // "DIMENSION_SCALE", which is 15.
1521 if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
1522 has_dimscalelist = true;
1523 break;
1524 }
1525 }
1526 }
1527 if (true == has_dimscalelist)
1528 break;
1529 }
1530
1531 if (true == has_dimscalelist) {
1532 if (true == has_dimlist ) {
1533 this->gproduct_pattern = GENERAL_DIMSCALE;
1534 ret_value = true;
1535 }
1536 else {
1537 //May fall into the single dimension scale case.
1538 //This is really, really rare,but we do have to check.
1539 // Check if NAME and _Netcdf4Dimid exists for this variable.
1540
1541 bool is_general_dimscale = false;
1542
1543 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1544
1545 bool has_class_dscale = false;
1546 bool has_name = false;
1547 bool has_netcdf4_id = false;
1548
1549 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();ira++) {
1550
1551 if ("CLASS" == (*ira)->name) {
1552 Retrieve_H5_Attr_Value(*ira,(*irv)->fullpath);
1553 string class_value;
1554 class_value.resize((*ira)->value.size());
1555 copy((*ira)->value.begin(),(*ira)->value.end(),class_value.begin());
1556
1557 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
1558 // "DIMENSION_SCALE", which is 15.
1559 if (0 == class_value.compare(0,15,"DIMENSION_SCALE"))
1560 has_class_dscale= true;
1561 }
1562 else if ("NAME" == (*ira)->name)
1563 has_name = true;
1564 else if ("_Netcdf4Dimid" == (*ira)->name)
1565 has_netcdf4_id = true;
1566 if(true == has_class_dscale && true == has_name && true == has_netcdf4_id)
1567 is_general_dimscale = true;
1568 }
1569
1570 if(true == is_general_dimscale)
1571 break;
1572 }
1573
1574 if (true == is_general_dimscale) {
1575 this->gproduct_pattern = GENERAL_DIMSCALE;
1576 ret_value = true;
1577 }
1578 }
1579 }
1580
1581 return ret_value;
1582}
1583
1584bool GMFile::Check_And_Update_New_GPM_L3() {
1585
1586 bool is_new_gpm_l3 = false;
1587 unsigned num_vars = this->vars.size();
1588 unsigned sel_steps = num_vars/5;
1589 string dim_name="DimensionNames";
1590 bool has_dim_name = false;
1591 if(sel_steps == 0)
1592 sel_steps = 1;
1593
1594 // Given DimensionNames exists in almost every variable in the new GPM product,
1595 // We will check the existence of this attribute for at most 5 variables.
1596
1597 vector<Var *>::iterator it_var_end;
1598
1599 if(sel_steps ==1)
1600 it_var_end = this->vars.end();
1601 else
1602 it_var_end = this->vars.begin()+5*sel_steps;
1603
1604 for (auto irv = this->vars.begin(); irv != it_var_end; irv+=sel_steps) {
1605 for(auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();ira++) {
1606
1607 if(H5FSTRING == (*ira)->getType()) {
1608 if((*ira)->name == dim_name){
1609 has_dim_name = true;
1610 break;
1611 }
1612 }
1613 }
1614 if(true == has_dim_name)
1615 break;
1616 }
1617
1618
1619
1620 // Files that can go to this step should be a small subset, now
1621 // we will check the "??GridHeader" for all the groups.
1622 if(true == has_dim_name) {
1623 string attr_name_subset = "GridHeader";
1624 BESDEBUG("h5", "GMFile::Check_And_Update_New_GPM_L3() has attribute <DimensionNames>. "<<endl);
1625 for (auto irg = this->groups.begin(); irg != this->groups.end(); ++ irg) {
1626 for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end();ira++) {
1627
1628 string attr_name = (*ira)->name;
1629
1630 // We identify this as a new GPM level 3 product.
1631 if (attr_name.find(attr_name_subset)!=string::npos) {
1632 this->product_type = GPM_L3_New;
1633 is_new_gpm_l3 = true;
1634 break;
1635 }
1636 }
1637 if (true == is_new_gpm_l3)
1638 break;
1639 }
1640 }
1641 return is_new_gpm_l3;
1642}
1643
1644// If having 2-D latitude/longitude,set the general product pattern.
1645// In this version, we only check if we have "latitude,longitude","Latitude,Longitude","lat,lon" and "cell_lat,cell_lon"names.
1646// The "cell_lat" and "cell_lon" come from SMAP. KY 2015-12-2
1647bool GMFile::Check_LatLon2D_General_Product_Pattern() {
1648
1649 BESDEBUG("h5", "Coming to Check_LatLon2D_General_Product_Pattern()"<<endl);
1650 bool ret_value = false;
1651
1652 ret_value = Check_LatLon2D_General_Product_Pattern_Name_Size("latitude","longitude");
1653 if(false == ret_value) {
1654 ret_value = Check_LatLon2D_General_Product_Pattern_Name_Size("Latitude","Longitude");
1655 if(false == ret_value) {
1656 ret_value = Check_LatLon2D_General_Product_Pattern_Name_Size("lat","lon");
1657 if(false == ret_value)
1658 ret_value = Check_LatLon2D_General_Product_Pattern_Name_Size("cell_lat","cell_lon");
1659 }
1660 }
1661
1662 // Make sure set the general product pattern flag for this case.
1663 if(true == ret_value)
1664 this->gproduct_pattern = GENERAL_LATLON2D;
1665 return ret_value;
1666
1667}
1668
1669// Helper function for Check_LatLon2D_General_Product_Pattern,we assume the lat and lon only present either under the root or
1670// a specific group Geolocation.
1671bool GMFile::Check_LatLon2D_General_Product_Pattern_Name_Size(const string & latname,const string & lonname) {
1672
1673 BESDEBUG("h5", "Coming to Check_LatLon2D_General_Product_Pattern_Name_Size()"<<endl);
1674 bool ret_value = false;
1675 bool ll_flag = false;
1676
1677 vector<size_t>lat_size(2,0);
1678 vector<size_t>lon_size(2,0);
1679
1680 const string designed_group1 = "/";
1681 const string designed_group2 = "/Geolocation/";
1682
1683 bool lat_flag_g1 = false;
1684 bool lon_flag_g1 = false;
1685 bool lat_flag_g2 = false;
1686 bool lon_flag_g2 = false;
1687
1688
1689 // This case allows to have both "lat and lon" under either group 1 or group 2 but on not both group 1 and 2.
1690 // This case doesn't allow "lat" and "lon" under separate groups.
1691 // Check if we have lat and lon at the only designated group,group 1 "/"
1692 lat_flag_g1 = is_var_under_group(latname,designed_group1,2,lat_size);
1693 lon_flag_g1 = is_var_under_group(lonname,designed_group1,2,lon_size);
1694 if(lat_flag_g1 == true && lon_flag_g1 == true) {
1695
1696 // Make sure the group 2 "/Geolocation" doesn't have the lat/lon
1697 lat_flag_g2 = is_var_under_group(latname,designed_group2,2,lat_size);
1698 if(lat_flag_g2 == false) {
1699 lon_flag_g2 = is_var_under_group(lonname,designed_group2,2,lon_size);
1700 if(lon_flag_g2 == false)
1701 ll_flag = true;
1702 }
1703 }// If the root doesn't have lat/lon, check the group 2 "/Geolocation".
1704 else if(lat_flag_g1 == false && lon_flag_g1 == false) {
1705 lat_flag_g2 = is_var_under_group(latname,designed_group2,2,lat_size);
1706 if(lat_flag_g2 == true) {
1707 lon_flag_g2 = is_var_under_group(lonname,designed_group2,2,lon_size);
1708 if(lon_flag_g2 == true)
1709 ll_flag = true;
1710 }
1711 }
1712
1713 // We are loose here since this is just to support some NASA products in a customized way.
1714 // If the first two cases don't exist, we allow to check another group"GeolocationData" and
1715 // see if Latitude and Longitude are present. (4 years ? from the first implementation, we got this case.)
1716 // KY 2020-02-27
1717 if(false == ll_flag) {
1718
1719 const string designed_group3 = "/GeolocationData/";
1720 if(is_var_under_group(latname,designed_group3,2,lat_size) &&
1721 is_var_under_group(lonname,designed_group3,2,lon_size))
1722 ll_flag = true;
1723 }
1724
1725#if 0
1726
1727 for (vector<Var *>::iterator irv = this->vars.begin();
1728 irv != this->vars.end(); ++irv) {
1729
1730 if((*irv)->rank == 2) {
1731 if((*irv)->name == latname) {
1732
1733 // Obtain the variable path
1734 string lat_path =HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
1735
1736 // Tackle only the root group or the name of the group as "/Geolocation"
1737 // By doing this, we assume that the file has lat/lon either under the root or under the "Geolocation
1738 // but not BOTH. The following code may generate wrong results if the file contains lat/lon under
1739 // both the root and /Geolocation. This is documented in https://jira.hdfgroup.org/browse/HFVHANDLER-175
1740 bool has_right_lat = false;
1741 if("/" == lat_path || "/Geolocation/" == lat_path)
1742 if("/" == lat_path || "/Geolocation/" == lat_path) {
1743 ll_flag++;
1744 lat_size[0] = (*irv)->getDimensions()[0]->size;
1745 lat_size[1] = (*irv)->getDimensions()[1]->size;
1746 }
1747
1748 }
1749 else if((*irv)->name == lonname) {
1750 string lon_path = HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
1751 if("/" == lon_path || "/Geolocation/" == lon_path) {
1752 ll_flag++;
1753 lon_size[0] = (*irv)->getDimensions()[0]->size;
1754 lon_size[1] = (*irv)->getDimensions()[1]->size;
1755 }
1756 }
1757 if(2 == ll_flag)
1758 break;
1759 } // if((*irv)->rank == 2)
1760 } // for (vector<Var *>::iterator irv = this->vars.begin();
1761
1762#endif
1763
1764 // Only when both lat/lon are found can we support this case.
1765 // Before that, we also need to check if the lat/lon shares the same dimensions.
1766 if(true == ll_flag) {
1767
1768 bool latlon_size_match = true;
1769 for (unsigned int size_index = 0; size_index <lat_size.size();size_index++) {
1770 if(lat_size[size_index] != lon_size[size_index]){
1771 latlon_size_match = false;
1772 break;
1773 }
1774 }
1775 if (true == latlon_size_match) {
1776 // If we do find the lat/lon pair, save them for later use.
1777 gp_latname = latname;
1778 gp_lonname = lonname;
1779 ret_value = true;
1780 }
1781
1782 }
1783
1784 return ret_value;
1785
1786}
1787
1788// If having 1-D latitude/longitude,set the general product pattern.
1789// In this version, we only check if we have "latitude,longitude","Latitude,Longitude","lat,lon" and "cell_lat,cell_lon"names.
1790// The "cell_lat" and "cell_lon" come from SMAP. KY 2015-12-2
1791bool GMFile::Check_LatLon1D_General_Product_Pattern() {
1792
1793 BESDEBUG("h5", "Coming to Check_LatLon1D_General_Product_Pattern()"<<endl);
1794 bool ret_value = false;
1795
1796 ret_value = Check_LatLon1D_General_Product_Pattern_Name_Size("latitude","longitude");
1797 if(false == ret_value) {
1798 ret_value = Check_LatLon1D_General_Product_Pattern_Name_Size("Latitude","Longitude");
1799 if(false == ret_value) {
1800 ret_value = Check_LatLon1D_General_Product_Pattern_Name_Size("lat","lon");
1801 if(false == ret_value)
1802 ret_value = Check_LatLon1D_General_Product_Pattern_Name_Size("cell_lat","cell_lon");
1803 }
1804 }
1805
1806 if(true == ret_value)
1807 this->gproduct_pattern = GENERAL_LATLON1D;
1808 return ret_value;
1809
1810}
1811
1812// Helper function for Check_LatLon1D_General_Product_Pattern.
1813// We only check if the lat/lon etc. pairs are under "/" or "/Geolocation". Other cases can be easily added.
1814bool GMFile::Check_LatLon1D_General_Product_Pattern_Name_Size(const string & latname,const string & lonname) {
1815
1816 BESDEBUG("h5", "Coming to Check_LatLon1D_General_Product_Pattern_Name_Size()"<<endl);
1817 bool ret_value = false;
1818 short ll_flag = 0;
1819 size_t lat_size = 0;
1820 size_t lon_size = 0;
1821
1822 for (vector<Var *>::iterator irv = this->vars.begin();
1823 irv != this->vars.end(); ++irv) {
1824
1825 if((*irv)->rank == 1) {
1826 if((*irv)->name == latname) {
1827
1828 string lat_path =HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
1829
1830 // Tackle only the root group or the name of the group as "/Geolocation"
1831 // May not generate the correct output. See https://jira.hdfgroup.org/browse/HFVHANDLER-175
1832 if("/" == lat_path || "/Geolocation/" == lat_path) {
1833 ll_flag++;
1834 lat_size = (*irv)->getDimensions()[0]->size;
1835 }
1836 }
1837 else if((*irv)->name == lonname) {
1838 string lon_path = HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
1839 if("/" == lon_path || "/Geolocation/" == lon_path) {
1840 ll_flag++;
1841 lon_size = (*irv)->getDimensions()[0]->size;
1842 }
1843 }
1844 if(2 == ll_flag)
1845 break;
1846 }
1847 }
1848
1849 if(2 == ll_flag) {
1850
1851 bool latlon_size_match_grid = true;
1852
1853 // When the size of latitude is equal to the size of longitude for a 1-D lat/lon, it is very possible
1854 // that this is not a regular grid but rather a profile with the lat,lon recorded as the function of time.
1855 // Adding the coordinate/dimension as the normal grid is wrong, so check out this case.
1856 // KY 2015-12-2
1857 if(lat_size == lon_size) {
1858
1859 // It is very unusual that lat_size = lon_size for a grid.
1860 latlon_size_match_grid = false;
1861
1862 // For a normal grid, a >2D variable should exist to have both lat and lon size,
1863 // if such a variable that has the same size exists, we will treat it as a normal grid.
1864 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1865 if((*irv)->rank >=2) {
1866 short ll_size_flag = 0;
1867 for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
1868 if(lat_size == (*ird)->size) {
1869 ll_size_flag++;
1870 if(2 == ll_size_flag){
1871 break;
1872 }
1873 }
1874 }
1875 if(2 == ll_size_flag) {
1876 latlon_size_match_grid = true;
1877 break;
1878 }
1879 }
1880 }
1881 }
1882
1883 // If the sizes of lat and lon match the grid, this is the lat/lon candidate.
1884 // Save the latitude and longitude names for later use.
1885 if (true == latlon_size_match_grid) {
1886 gp_latname = latname;
1887 gp_lonname = lonname;
1888 ret_value = true;
1889 }
1890 }
1891
1892 return ret_value;
1893}
1894
1895// This function checks if this general product contains "coordinates" attributes in some variables
1896// that can be used to handle CF friendly.
1897bool GMFile::Check_LatLon_With_Coordinate_Attr_General_Product_Pattern() {
1898
1899 BESDEBUG("h5", "Coming to Check_LatLon_With_Coordinate_Attr_General_Product_Pattern()"<<endl);
1900 bool ret_value = false;
1901 string co_attrname = "coordinates";
1902 string co_attrvalue="";
1903 string unit_attrname = "units";
1904 string lat_unit_attrvalue ="degrees_north";
1905 string lon_unit_attrvalue ="degrees_east";
1906
1907 bool coor_has_lat_flag = false;
1908 bool coor_has_lon_flag = false;
1909
1910 vector<Var*> tempvar_lat;
1911 vector<Var*> tempvar_lon;
1912
1913 // Check if having both lat, lon names stored in the coordinate attribute value by looping through rank >1 variables.
1914 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1915
1916 if((*irv)->rank >=2) {
1917 for (auto ira =(*irv)->attrs.begin(); ira !=(*irv)->attrs.end();++ira) {
1918
1919 // If having attribute "coordinates" for this variable, checking the values and
1920 // see if having lat/lon,latitude/longitude, Latitude/Longitude pairs.
1921 if((*ira)->name == co_attrname) {
1922 Retrieve_H5_Attr_Value((*ira),(*irv)->fullpath);
1923 string orig_attr_value((*ira)->value.begin(),(*ira)->value.end());
1924 vector<string> coord_values;
1925 char sep=' ';
1926 HDF5CFUtil::Split_helper(coord_values,orig_attr_value,sep);
1927
1928 for(auto irs=coord_values.begin();irs!=coord_values.end();++irs) {
1929 string coord_value_suffix1;
1930 string coord_value_suffix2;
1931 string coord_value_suffix3;
1932
1933 if((*irs).size() >=3) {
1934
1935 // both "lat" and "lon" have 3 characters.
1936 coord_value_suffix1 = (*irs).substr((*irs).size()-3,3);
1937
1938 // The word "latitude" has 8 characters and the word "longitude" has 9 characters.
1939 if((*irs).size() >=8){
1940 coord_value_suffix2 = (*irs).substr((*irs).size()-8,8);
1941 if((*irs).size() >=9)
1942 coord_value_suffix3 = (*irs).substr((*irs).size()-9,9);
1943 }
1944 }
1945
1946 // lat/longitude or latitude/lon pairs in theory are fine.
1947 if(coord_value_suffix1=="lat" || coord_value_suffix2 =="latitude" || coord_value_suffix2 == "Latitude")
1948 coor_has_lat_flag = true;
1949 else if(coord_value_suffix1=="lon" || coord_value_suffix3 =="longitude" || coord_value_suffix3 == "Longitude")
1950 coor_has_lon_flag = true;
1951 }
1952
1953 if(true == coor_has_lat_flag && true == coor_has_lon_flag)
1954 break;
1955 }// end of if((*ira)->name
1956 }// end of for (ira)
1957
1958 if(true == coor_has_lat_flag && true == coor_has_lon_flag)
1959 break;
1960 else {
1961 coor_has_lat_flag = false;
1962 coor_has_lon_flag = false;
1963 }
1964 } // end of if
1965 }// end of for
1966
1967 // Check the variable names that include latitude and longitude suffixes such as lat,latitude and Latitude.
1968 if(true == coor_has_lat_flag && true == coor_has_lon_flag) {
1969
1970 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1971
1972 bool var_is_lat = false;
1973 bool var_is_lon = false;
1974
1975 string varname = (*irv)->name;
1976 string ll_ssuffix;
1977 string ll_lsuffix1;
1978 string ll_lsuffix2;
1979 if(varname.size() >=3) {//lat/lon
1980 ll_ssuffix = varname.substr(varname.size()-3,3);
1981 if(varname.size() >=8) {//latitude/Latitude
1982 ll_lsuffix1 = varname.substr(varname.size()-8,8);
1983 if(varname.size() >=9)//Longitude/longitude
1984 ll_lsuffix2 = varname.substr(varname.size()-9,9);
1985 }
1986 }
1987 if(ll_ssuffix=="lat" || ll_lsuffix1 =="latitude" || ll_lsuffix1 == "Latitude")
1988 var_is_lat = true;
1989 else if(ll_ssuffix=="lon" || ll_lsuffix2 =="longitude" || ll_lsuffix2 == "Longitude")
1990 var_is_lon = true;
1991
1992 // Find the lat/lon candidate, save them to temporary vectors
1993 if(true == var_is_lat) {
1994 if((*irv)->rank > 0) {
1995 auto lat = new Var(*irv);
1996 tempvar_lat.push_back(lat);
1997 }
1998 }
1999 else if(true == var_is_lon) {
2000 if((*irv)->rank >0) {
2001 auto lon = new Var(*irv);
2002 tempvar_lon.push_back(lon);
2003 }
2004 }
2005 }// for of irv
2006
2007 // Build up latloncv_candidate_pairs, Name_Size_2Pairs struct,
2008 // 1) Compare the rank, dimension sizes and the dimension orders of tempvar_lon against tempvar_lat
2009 // rank >=2 the sizes,orders, should be consistent
2010 // rank =1, no check issue.
2011 // 2) If the conditions are fulfilled, save them to the Name_Size struct
2012 for (auto irlat = tempvar_lat.begin(); irlat!=tempvar_lat.end();++irlat) {
2013
2014 // Check the rank =1 case
2015 if((*irlat)->rank == 1)
2016 Build_lat1D_latlon_candidate(*irlat,tempvar_lon);
2017
2018 // Check the reank>=2 case
2019 else if((*irlat)->rank >1)
2020 Build_latg1D_latlon_candidate(*irlat,tempvar_lon);
2021 }
2022
2023#if 0
2024for(vector<struct Name_Size_2Pairs>::iterator ivs=latloncv_candidate_pairs.begin(); ivs!=latloncv_candidate_pairs.end();++ivs) {
2025cerr<<"struct lat lon names are " <<(*ivs).name1 <<" and " << (*ivs).name2 <<endl;
2026}
2027#endif
2028
2029 // Check if there is duplicate latitude variables for one longitude variable in the latloncv_candidate_pairs.
2030 // if yes, remove the ones that have duplicate latitude variables.
2031 // This will assure that the latloncv_candidate_pairs is one-to-one mapping between latitude and longitude.
2032 Build_unique_latlon_candidate();
2033
2034
2035 // Even if we find that there are qualified geo-location coordinate pairs, we still need to check
2036 // the geo-location variable rank.
2037 // If the rank of any one-pair is 2, this case is qualified for the category GENERAL_LATLON_COOR_ATTR.
2038 // If the rank of any one-pair is 1,
2039 // we will check if the sizes of the lat and the lon in a pair are the same.
2040 // If they are not the same, this case is qualified for the category GENERAL_LATLON_COOR_ATTR
2041 // else check if there is any variable that has the "coordinates" attribute and the "coordinates" attribute includes
2042 // the paths of this lat/lon pair. If the dimensions of such a variable have two sizes that are equal to the size of the lat,
2043 // this case is still qualfied for the category GENERAL_LATLON_COOR_ATTR.
2044 // NOTE: here we deliberately ignore the case when the rank of lat/lon is >2. In some recent developments, we find that
2045 // there are 3D lat/lon and some tools like Panoply can visualize those data. So maybe we need to accept some 3D lat/lon in the futurei(KY 2016-07-07).
2046 if(latloncv_candidate_pairs.size() >0) {
2047 int num_1d_rank = 0;
2048 int num_2d_rank = 0;
2049 int num_g2d_rank = 0;
2050 vector<struct Name_Size_2Pairs> temp_1d_latlon_pairs;
2051 for(auto ivs=latloncv_candidate_pairs.begin(); ivs!=latloncv_candidate_pairs.end();++ivs) {
2052 if(1 == (*ivs).rank) {
2053 num_1d_rank++;
2054 temp_1d_latlon_pairs.push_back(*ivs);
2055 }
2056 else if(2 == (*ivs).rank)
2057 num_2d_rank++;
2058 else if((*ivs).rank >2)
2059 num_g2d_rank++;
2060 }
2061
2062 // This is the GENERAL_LATLON_COOR_ATTR case.
2063 if (num_2d_rank !=0)
2064 ret_value = true;
2065 else if(num_1d_rank!=0) {
2066
2067 // Check if lat and lon share the same size and the dimension of a variable
2068 // that has the "coordinates" only holds one size.
2069 for(auto ivs=temp_1d_latlon_pairs.begin(); ivs!=temp_1d_latlon_pairs.end();++ivs) {
2070 if((*ivs).size1 != (*ivs).size2) {
2071 ret_value = true;
2072 break;
2073 }
2074 else {
2075
2076 // If 1-D lat and lon share the same size,we need to check if there is a variable
2077 // that has both lat and lon as the coordinates but only has one dimension that holds the size.
2078 // If this is true, this is not the GENERAL_LATLON_COOR_ATTR case(SMAP level 2 follows into the category).
2079
2080 ret_value = true;
2081 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2082
2083 if((*irv)->rank >=2) {
2084 for (auto ira =(*irv)->attrs.begin(); ira !=(*irv)->attrs.end();++ira) {
2085
2086 // Check if this variable has the "coordinates" attribute
2087 if((*ira)->name == co_attrname) {
2088 Retrieve_H5_Attr_Value((*ira),(*irv)->fullpath);
2089 string orig_attr_value((*ira)->value.begin(),(*ira)->value.end());
2090 vector<string> coord_values;
2091 char sep=' ';
2092 HDF5CFUtil::Split_helper(coord_values,orig_attr_value,sep);
2093 bool has_lat_flag = false;
2094 bool has_lon_flag = false;
2095 for (auto itcv=coord_values.begin();itcv!=coord_values.end();++itcv) {
2096 if((*ivs).name1 == (*itcv))
2097 has_lat_flag = true;
2098 else if((*ivs).name2 == (*itcv))
2099 has_lon_flag = true;
2100 }
2101 // Find both lat and lon, now check the dim. size
2102 if(true == has_lat_flag && true == has_lon_flag) {
2103 short has_same_ll_size = 0;
2104 for(auto ird = (*irv)->dims.begin();ird!=(*irv)->dims.end();++ird){
2105 if((*ird)->size == (*ivs).size1)
2106 has_same_ll_size++;
2107 }
2108 if(has_same_ll_size!=2){
2109 ret_value = false;
2110 break;
2111 }
2112 }
2113 }
2114 }// end of for ira
2115
2116 if(false == ret_value)
2117 break;
2118 }// end of if irv
2119 }// end of for irv
2120
2121 if(true == ret_value)
2122 break;
2123 }// else
2124 }// end of for ivs
2125 } // else of if num_1d_rank
2126 }// end of if of latloncv_candidate_pairs
2127
2128 release_standalone_var_vector(tempvar_lat);
2129 release_standalone_var_vector(tempvar_lon);
2130
2131 }
2132#if 0
2133if(true == ret_value)
2134cerr<<"This product is the coordinate type "<<endl;
2135#endif
2136 // Don't forget to set the flag for this general product pattern.
2137 if(true == ret_value)
2138 this->gproduct_pattern = GENERAL_LATLON_COOR_ATTR;
2139
2140 return ret_value;
2141}
2142
2143// Build 1-D latlon coordinate variables candidate for GENERAL_LATLON_COOR_ATTR.
2144void GMFile::Build_lat1D_latlon_candidate(const Var *lat,const vector<Var*> &lon_vec) {
2145
2146 BESDEBUG("h5", "Coming to Build_lat1D_latlon_candidate()"<<endl);
2147 set<string> lon_candidate_path;
2148 vector< pair<string,hsize_t> > lon_path_size_vec;
2149
2150 // Obtain the path and the size info. from all the potential qualified longitude candidate.
2151 for(auto irlon = lon_vec.begin(); irlon!=lon_vec.end();++irlon) {
2152
2153 if (lat->rank == (*irlon)->rank) {
2154 pair<string,hsize_t>lon_path_size;
2155 lon_path_size.first = (*irlon)->fullpath;
2156 lon_path_size.second = (*irlon)->getDimensions()[0]->size;
2157 lon_path_size_vec.push_back(lon_path_size);
2158 }
2159 }
2160
2161 // If there is only one potential qualified longitude for this latitude, just save this pair.
2162 if(lon_path_size_vec.size() == 1) {
2163
2164 Name_Size_2Pairs latlon_pair;
2165 latlon_pair.name1 = lat->fullpath;
2166 latlon_pair.name2 = lon_path_size_vec[0].first;
2167 latlon_pair.size1 = lat->getDimensions()[0]->size;
2168 latlon_pair.size2 = lon_path_size_vec[0].second;
2169 latlon_pair.rank = lat->rank;
2170 latloncv_candidate_pairs.push_back(latlon_pair);
2171
2172 }
2173 else if(lon_path_size_vec.size() >1) {
2174
2175 // For more than one potential qualified longitude, we can still find a qualified one
2176 // if we find there is only one longitude under the same group of this latitude.
2177 string lat_path = HDF5CFUtil::obtain_string_before_lastslash(lat->fullpath);
2178 pair<string,hsize_t> lon_final_path_size;
2179 short num_lon_match = 0;
2180 for (auto islon =lon_path_size_vec.begin();islon!=lon_path_size_vec.end();++islon) {
2181 // Search the longitude path and see if it matches with the latitude.
2182 if(HDF5CFUtil::obtain_string_before_lastslash((*islon).first)==lat_path) {
2183 num_lon_match++;
2184 if(1 == num_lon_match)
2185 lon_final_path_size = *islon;
2186 else if(num_lon_match > 1)
2187 break;
2188 }
2189 }
2190 if(num_lon_match ==1) {// insert this lat/lon pair to the struct
2191 Name_Size_2Pairs latlon_pair;
2192 latlon_pair.name1 = lat->fullpath;
2193 latlon_pair.name2 = lon_final_path_size.first;
2194 latlon_pair.size1 = lat->getDimensions()[0]->size;
2195 latlon_pair.size2 = lon_final_path_size.second;
2196 latlon_pair.rank = lat->rank;
2197 latloncv_candidate_pairs.push_back(latlon_pair);
2198 }
2199 }
2200
2201}
2202
2203// Build >1D latlon coordinate variables candidate for GENERAL_LATLON_COOR_ATTR.
2204void GMFile::Build_latg1D_latlon_candidate(Var *lat,const vector<Var*> & lon_vec) {
2205
2206 BESDEBUG("h5", "Coming to Build_latg1D_latlon_candidate()"<<endl);
2207 set<string> lon_candidate_path;
2208
2209 // We will check if the longitude shares the same dimensions of the latitude
2210 for(auto irlon = lon_vec.begin(); irlon!=lon_vec.end();++irlon) {
2211
2212 if (lat->rank == (*irlon)->rank) {
2213
2214 // Check the dim order and size.
2215 bool same_dim = true;
2216 for(int dim_index = 0; dim_index <lat->rank; dim_index++) {
2217 if(lat->getDimensions()[dim_index]->size !=
2218 (*irlon)->getDimensions()[dim_index]->size){
2219 same_dim = false;
2220 break;
2221 }
2222 }
2223 if(true == same_dim)
2224 lon_candidate_path.insert((*irlon)->fullpath);
2225 }
2226 }
2227
2228 // Check the size of the lon., if the size is not 1, see if we can find the pair under the same group.
2229 if(lon_candidate_path.size() > 1) {
2230
2231 string lat_path = HDF5CFUtil::obtain_string_before_lastslash(lat->fullpath);
2232 vector <string> lon_final_candidate_path_vec;
2233 for(auto islon_path =lon_candidate_path.begin();islon_path!=lon_candidate_path.end();++islon_path) {
2234
2235 // Search the path.
2236 if(HDF5CFUtil::obtain_string_before_lastslash(*islon_path)==lat_path)
2237 lon_final_candidate_path_vec.push_back(*islon_path);
2238 }
2239
2240 if(lon_final_candidate_path_vec.size() == 1) {// insert this lat/lon pair to the struct
2241
2242 Name_Size_2Pairs latlon_pair;
2243
2244 latlon_pair.name1 = lat->fullpath;
2245 latlon_pair.name2 = lon_final_candidate_path_vec[0];
2246 latlon_pair.size1 = lat->getDimensions()[0]->size;
2247 latlon_pair.size2 = lat->getDimensions()[1]->size;
2248 latlon_pair.rank = lat->rank;
2249 latloncv_candidate_pairs.push_back(latlon_pair);
2250 }
2251 else if(lon_final_candidate_path_vec.size() >1) {
2252
2253 // Under the same group, if we have two pairs lat/lon such as foo1_lat,foo1_lon, foo2_lat,foo2_lon, we will
2254 // treat {foo1_lat,foo1_lon} and {foo2_lat,foo2_lon} as two lat,lon coordinate candidates. This is essentially the SMAP L1B case.
2255 // We only compare three potential suffixes, lat/lon, latitude/longitude,Latitude/Longitude. We will treat the pair
2256 // latitude/Longitude and Latitude/longitude as a valid one.
2257
2258 string lat_name = HDF5CFUtil::obtain_string_after_lastslash(lat->fullpath);
2259 string lat_name_prefix1;
2260 string lat_name_prefix2;
2261
2262 // name prefix before the pair lat,note: no need to check if the last 3 characters are lat or lon. We've checked already.
2263 if(lat_name.size() >3) {
2264 lat_name_prefix1 = lat_name.substr(0,lat_name.size()-3);
2265 if(lat_name.size() >8)
2266 lat_name_prefix2 = lat_name.substr(0,lat_name.size()-8);
2267 }
2268 string lon_name_prefix1;
2269 string lon_name_prefix2;
2270
2271 for(auto ilon = lon_final_candidate_path_vec.begin(); ilon!=lon_final_candidate_path_vec.end();++ilon) {
2272 string lon_name = HDF5CFUtil::obtain_string_after_lastslash(*ilon);
2273 if(lon_name.size() >3) {
2274 lon_name_prefix1 = lon_name.substr(0,lon_name.size()-3);
2275 if(lon_name.size() >9)
2276 lon_name_prefix2 = lon_name.substr(0,lon_name.size()-9);
2277 }
2278 if((lat_name_prefix1 !="" && lat_name_prefix1 == lon_name_prefix1) ||
2279 (lat_name_prefix2 !="" && lat_name_prefix2 == lon_name_prefix2)) {// match lat,lon this one is the candidate
2280
2281 Name_Size_2Pairs latlon_pair;
2282 latlon_pair.name1 = lat->fullpath;
2283 latlon_pair.name2 = *ilon;
2284 latlon_pair.size1 = lat->getDimensions()[0]->size;
2285 latlon_pair.size2 = lat->getDimensions()[1]->size;
2286 latlon_pair.rank = lat->rank;
2287 latloncv_candidate_pairs.push_back(latlon_pair);
2288
2289 }
2290 }
2291 }// end else of if lon_final_candidate_path_vec
2292 }// end of if(lon_candidate_path.size() > 1)
2293
2294 else if(lon_candidate_path.size() == 1) {//insert this lat/lon pair to the struct
2295
2296 Name_Size_2Pairs latlon_pair;
2297
2298 latlon_pair.name1 = lat->fullpath;
2299 latlon_pair.name2 = *(lon_candidate_path.begin());
2300 latlon_pair.size1 = lat->getDimensions()[0]->size;
2301 latlon_pair.size2 = lat->getDimensions()[1]->size;
2302 latlon_pair.rank = lat->rank;
2303 latloncv_candidate_pairs.push_back(latlon_pair);
2304
2305 }
2306
2307}
2308
2309// We need to make sure that one lat maps to one lon in the lat/lon pairs.
2310// This routine removes the duplicate ones like (lat1,lon1) and (lat2,lon1).
2311void GMFile::Build_unique_latlon_candidate() {
2312
2313 BESDEBUG("h5", "Coming to Build_unique_latlon_candidate()"<<endl);
2314 set<int> duplicate_index;
2315 for(unsigned int i= 0; i<latloncv_candidate_pairs.size();i++) {
2316 for(unsigned int j=i+1;j<latloncv_candidate_pairs.size();j++) {
2317 if(latloncv_candidate_pairs[i].name2 == latloncv_candidate_pairs[j].name2) {
2318 duplicate_index.insert(i);
2319 duplicate_index.insert(j);
2320 }
2321 }
2322 }
2323
2324 // set is pre-sorted. we used a quick way to remove multiple elements.
2325 for(auto its= duplicate_index.rbegin();its!=duplicate_index.rend();++its) {
2326 latloncv_candidate_pairs[*its] = latloncv_candidate_pairs.back();
2327 latloncv_candidate_pairs.pop_back();
2328 }
2329}
2330// Leave the following code for the time being.
2331#if 0
2332// In this version, we only check if we have "latitude,longitude","Latitude,Longitude","lat,lon" names.
2333// This routine will check this case.
2334bool GMFile::Check_LatLonName_General_Product(int ll_rank) {
2335
2336 if(ll_rank <1 || ll_rank >2)
2337 throw2("Only support rank = 1 or 2 lat/lon case for the general product. The current rank is ",ll_rank);
2338 bool ret_value = false;
2339 size_t lat2D_dimsize0 = 0;
2340 size_t lat2D_dimsize1 = 0;
2341 size_t lon2D_dimsize0 = 0;
2342 size_t lon2D_dimsize1 = 0;
2343
2344 // The element order is latlon_flag,latilong_flag and LatLon_flag.
2345 vector<short>ll_flag(3,0);
2346
2347 vector<size_t>lat_size;
2348 vector<size_t>lon_size;
2349
2350 // We only need to check 2-D latlon
2351 if(2 == ll_rank) {
2352 //lat/lon is 2-D array, so the size is doubled.
2353 lat_size.assign(6,0);
2354 lon_size.assign(6,0);
2355 }
2356
2357 for (vector<Var *>::iterator irv = this->vars.begin();
2358 irv != this->vars.end(); ++irv) {
2359
2360 if((*irv)->rank == ll_rank) {
2361 if((*irv)->name == "lat") {
2362 ll_flag[0]++;
2363 if(ll_rank == 2) {
2364 lat_size[0] = (*irv)->getDimensions()[0]->size;
2365 lat_size[1] = (*irv)->getDimensions()[1]->size;
2366
2367 }
2368
2369 }
2370 else if((*irv)->name == "lon") {
2371 ll_flag[0]++;
2372 if(ll_rank == 2) {
2373 lon_size[0] = (*irv)->getDimensions()[0]->size;
2374 lon_size[1] = (*irv)->getDimensions()[1]->size;
2375
2376 }
2377
2378 }
2379 else if((*irv)->name == "latitude"){
2380 ll_flag[1]++;
2381 if(ll_rank == 2) {
2382 lat_size[2] = (*irv)->getDimensions()[0]->size;
2383 lat_size[3] = (*irv)->getDimensions()[1]->size;
2384
2385 }
2386 }
2387 else if((*irv)->name == "longitude"){
2388 ll_flag[1]++;
2389 if(ll_rank == 2) {
2390 lon_size[2] = (*irv)->getDimensions()[0]->size;
2391 lon_size[3] = (*irv)->getDimensions()[1]->size;
2392
2393 }
2394
2395 }
2396 else if((*irv)->name == "Latitude"){
2397 ll_flag[2]++;
2398 if(ll_rank == 2) {
2399 lat_size[4] = (*irv)->getDimensions()[0]->size;
2400 lat_size[5] = (*irv)->getDimensions()[1]->size;
2401
2402 }
2403
2404 }
2405 else if((*irv)->name == "Longitude"){
2406 ll_flag[2]++;
2407 if(ll_rank == 2) {
2408 lon_size[4] = (*irv)->getDimensions()[0]->size;
2409 lon_size[5] = (*irv)->getDimensions()[1]->size;
2410 }
2411 }
2412 }
2413 }
2414
2415 int total_llflag = 0;
2416 for (int i = 0; i < ll_flag.size();i++)
2417 if(2 == ll_flag[i])
2418 total_llflag ++;
2419
2420 // We only support 1 (L)lat(i)/(L)lon(g) pair.
2421 if(1 == total_llflag) {
2422 bool latlon_size_match = true;
2423 if(2 == ll_rank) {
2424 for (int size_index = 0; size_index <lat_size.size();size_index++) {
2425 if(lat_size[size_index] != lon_size[size_index]){
2426 latlon_size_match = false;
2427 break;
2428 }
2429 }
2430 }
2431
2432 if(true == latlon_size_match) {
2433 ret_value = true;
2434 if(2 == ll_flag[0]) {
2435 gp_latname = "lat";
2436 gp_lonname = "lon";
2437 }
2438 else if ( 2 == ll_flag[1]) {
2439 gp_latname = "latitude";
2440 gp_lonname = "longitude";
2441 }
2442
2443 else if (2 == ll_flag[2]){
2444 gp_latname = "Latitude";
2445 gp_lonname = "Longitude";
2446 }
2447 }
2448 }
2449
2450 return ret_value;
2451}
2452#endif
2453
2454// Add dimension names for the case that has 2-D lat/lon.
2455void GMFile::Add_Dim_Name_LatLon2D_General_Product() {
2456
2457 BESDEBUG("h5", "Coming to Add_Dim_Name_LatLon2D_General_Product()"<<endl);
2458 string latdimname0;
2459 string latdimname1;
2460 size_t latdimsize0 = 0;
2461 size_t latdimsize1 = 0;
2462
2463 // Need to generate fake dimensions.
2464 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2465
2466 set<hsize_t> fakedimsize;
2467 pair<set<hsize_t>::iterator,bool> setsizeret;
2468 int num_dup_dim_size = 0;
2469 for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
2470 Add_One_FakeDim_Name(*ird);
2471 setsizeret = fakedimsize.insert((*ird)->size);
2472
2473 // Avoid the same size dimension sharing the same dimension name.
2474 if (false == setsizeret.second){
2475 num_dup_dim_size++;
2476 Adjust_Duplicate_FakeDim_Name2(*ird,num_dup_dim_size);
2477 }
2478
2479#if 0
2480 if (false == setsizeret.second)
2481 Adjust_Duplicate_FakeDim_Name(*ird);
2482#endif
2483 }
2484
2485 // Find variable name that is latitude or lat or Latitude
2486 // Note that we don't need to check longitude since longitude dim. sizes should be the same as the latitude for this case.
2487 if((*irv)->name == gp_latname) {
2488 if((*irv)->rank != 2) {
2489 throw4("coordinate variables ",gp_latname,
2490 " must have rank 2 for the 2-D latlon case , the current rank is ",
2491 (*irv)->rank);
2492 }
2493 latdimname0 = (*irv)->getDimensions()[0]->name;
2494 latdimsize0 = (*irv)->getDimensions()[0]->size;
2495
2496 latdimname1 = (*irv)->getDimensions()[1]->name;
2497 latdimsize1 = (*irv)->getDimensions()[1]->size;
2498 }
2499 }
2500
2501
2502 // Now we need to change a dimension of a general variable that shares the same size of lat
2503 // to the dimension name of the lat.
2504 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2505 int lat_dim0_index = 0;
2506 int lat_dim1_index = 0;
2507 bool has_lat_dims_size = false;
2508
2509 for (unsigned int dim_index = 0; dim_index <(*irv)->dims.size(); dim_index++) {
2510
2511 // Find if having the first dimension size of lat
2512 if(((*irv)->dims[dim_index])->size == latdimsize0) {
2513
2514 // Find if having the second dimension size of lat
2515 lat_dim0_index = dim_index;
2516 for(unsigned int dim_index2 = dim_index+1;dim_index2 < (*irv)->dims.size();dim_index2++) {
2517 if(((*irv)->dims[dim_index2])->size == latdimsize1) {
2518 lat_dim1_index = dim_index2;
2519 has_lat_dims_size = true;
2520 break;
2521 }
2522 }
2523 }
2524 if(true == has_lat_dims_size)
2525 break;
2526 }
2527 // Find the lat's dimension sizes, change the (fake) dimension names.
2528 if(true == has_lat_dims_size) {
2529
2530 ((*irv)->dims[lat_dim0_index])->name = latdimname0;
2531#if 0
2532 //((*irv)->dims[lat_dim0_index])->newname = latdimname0;
2533#endif
2534 ((*irv)->dims[lat_dim1_index])->name = latdimname1;
2535#if 0
2536 //((*irv)->dims[lat_dim1_index])->newname = latdimname1;
2537#endif
2538
2539 }
2540 }
2541
2542 //When we generate Fake dimensions, we may encounter discontiguous Fake dimension names such
2543 // as FakeDim0, FakeDim9 etc. We would like to make Fake dimension names in contiguous order
2544 // FakeDim0,FakeDim1,etc.
2545
2546 // Obtain the tempdimnamelist set.
2547 set<string>tempdimnamelist;
2548
2549 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2550 for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird)
2551 tempdimnamelist.insert((*ird)->name);
2552 }
2553
2554 // Generate the final dimnamelist,it is a contiguous order: FakeDim0,FakeDim1 etc.
2555 set<string>finaldimnamelist;
2556 string finaldimname_base = "FakeDim";
2557
2558 for(unsigned int i = 0; i<tempdimnamelist.size();i++) {
2559 stringstream sfakedimindex;
2560 sfakedimindex << i;
2561 string finaldimname = finaldimname_base + sfakedimindex.str();
2562 finaldimnamelist.insert(finaldimname);
2563 }
2564
2565 // If the original tempdimnamelist is not the same as the finaldimnamelist,
2566 // we need to generate a map from original name to the final name.
2567 if(finaldimnamelist != tempdimnamelist) {
2568 map<string,string> tempdimname_to_finaldimname;
2569 auto tempit = tempdimnamelist.begin();
2570 auto finalit = finaldimnamelist.begin();
2571 while(tempit != tempdimnamelist.end()) {
2572 tempdimname_to_finaldimname[*tempit] = *finalit;
2573 tempit++;
2574 finalit++;
2575 }
2576
2577 // Change the dimension names of every variable to the final dimension name list.
2578 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2579 for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
2580 if(tempdimname_to_finaldimname.find((*ird)->name) !=tempdimname_to_finaldimname.end())
2581 (*ird)->name = tempdimname_to_finaldimname[(*ird)->name];
2582 else
2583 throw3("The dimension names ",(*ird)->name, "cannot be found in the dim. name list.");
2584 }
2585 }
2586 }
2587
2588
2589 dimnamelist.clear();
2590 dimnamelist = finaldimnamelist;
2591
2592 // We need to update dimname_to_dimsize map. This may be used in the future.
2593 dimname_to_dimsize.clear();
2594 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2595 for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
2596 if(finaldimnamelist.find((*ird)->name)!=finaldimnamelist.end()) {
2597 dimname_to_dimsize[(*ird)->name] = (*ird)->size;
2598 dimname_to_unlimited[(*ird)->name] = (*ird)->unlimited_dim;
2599 finaldimnamelist.erase((*ird)->name);
2600 }
2601
2602 }
2603 if(true == finaldimnamelist.empty())
2604 break;
2605 }
2606
2607 // Finally set dimension newname
2608 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2609 for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird)
2610 (*ird)->newname = (*ird)->name;
2611 }
2612
2613}
2614
2615// Add dimension names for the case that has 1-D lat/lon or CoordAttr..
2616//
2617void GMFile::Add_Dim_Name_LatLon1D_Or_CoordAttr_General_Product() {
2618
2619 BESDEBUG("h5", "Coming to Add_Dim_Name_LatLon1D_Or_CoordAttr_General_Product()"<<endl);
2620 // Only need to add the fake dimension names
2621 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2622 set<hsize_t> fakedimsize;
2623 pair<set<hsize_t>::iterator,bool> setsizeret;
2624 int num_dup_dim_size = 0;
2625 for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
2626 Add_One_FakeDim_Name(*ird);
2627 setsizeret = fakedimsize.insert((*ird)->size);
2628 // Avoid the same size dimension sharing the same dimension name.
2629 // HYRAX-629, we not only want to have the unique dimension name for
2630 // the same dimension size of different dimensions in a var, we also
2631 // want to reduce the number of dimension names.
2632 // So foo[100][100], foo2[100][100][100] should be something like
2633 // foo[FakeDim0][FakeDim1] foo2[FakeDim0][FakeDim1][FakeDim2]
2634 if (false == setsizeret.second){
2635 num_dup_dim_size++;
2636 Adjust_Duplicate_FakeDim_Name2(*ird,num_dup_dim_size);
2637 }
2638 // Comment out the original code for the time being.
2639#if 0
2640 if (false == setsizeret.second){
2641 num_dup_dim_size++;
2642 Adjust_Duplicate_FakeDim_Name(*ird,num_dup_dim_size);
2643 }
2644#endif
2645 }
2646 }
2647
2648// leave it here for debugging purpose
2649#if 0
2650for (vector<Var *>::iterator irv = this->vars.begin();
2651 irv != this->vars.end(); ++irv) {
2652cerr<<"Var name is "<<(*irv)->newname<<endl;
2653 for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
2654 ird != (*irv)->dims.end(); ++ird)
2655cerr<<"Dimension name is "<<(*ird)->newname <<endl;
2656}
2657#endif
2658
2659}
2660
2661// For netCDF-4-like HDF5 products, we need to add the dimension scales.
2662void GMFile::Add_Dim_Name_Dimscale_General_Product() {
2663
2664 BESDEBUG("h5", "Coming to Add_Dim_Name_Dimscale_General_Product()"<<endl);
2665 //cerr<<"coming to Add_Dim_Name_Dimscale_General_Product"<<endl;
2666 pair<set<string>::iterator,bool> setret;
2667 this->iscoard = true;
2668
2669 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2670
2671 // Obtain all the dimension names for this variable
2672 Handle_UseDimscale_Var_Dim_Names_General_Product((*irv));
2673
2674 // Need to update dimenamelist and dimname_to_dimsize and dimname_to_unlimited maps for future use.
2675 for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
2676 setret = dimnamelist.insert((*ird)->name);
2677 if (true == setret.second)
2678 Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size,(*ird)->unlimited_dim);
2679 }
2680 } // end of for
2681
2682 if (true == dimnamelist.empty())
2683 throw1("This product should have the dimension names, but no dimension names are found");
2684
2685}
2686
2687// Obtain dimension names for this variable when netCDF-4 model(using dimension scales) is followed.
2688void GMFile::Handle_UseDimscale_Var_Dim_Names_General_Product(Var *var) {
2689
2690 BESDEBUG("h5", "Coming to Handle_UseDimscale_Var_Dim_Names_General_Product()"<<endl);
2691 Attribute* dimlistattr = nullptr;
2692 bool has_dimlist = false;
2693 bool has_dimclass = false;
2694
2695 for(auto ira = var->attrs.begin(); ira != var->attrs.end();ira++) {
2696 if ("DIMENSION_LIST" == (*ira)->name) {
2697 dimlistattr = *ira;
2698 has_dimlist = true;
2699 }
2700 if ("CLASS" == (*ira)->name) {
2701
2702 Retrieve_H5_Attr_Value(*ira,var->fullpath);
2703 string class_value;
2704 class_value.resize((*ira)->value.size());
2705 copy((*ira)->value.begin(),(*ira)->value.end(),class_value.begin());
2706
2707 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2708 // "DIMENSION_SCALE", which is 15.
2709 if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
2710 has_dimclass = true;
2711 break;
2712 }
2713 }
2714
2715 } // end of for
2716
2717 // This is a general variable, we need to find the corresponding coordinate variables.
2718 if (true == has_dimlist)
2719 Add_UseDimscale_Var_Dim_Names_General_Product(var,dimlistattr);
2720
2721 // Dim name is the same as the variable name for dimscale variable
2722 else if(true == has_dimclass) {
2723 if (var->dims.size() !=1)
2724 throw2("Currently dimension scale dataset must be 1 dimension, this is not true for the dataset ",
2725 var->name);
2726
2727 // The var name is the object name, however, we would like the dimension name to be the full path.
2728 // so that the dim name can be served as the key for future handling.
2729 (var->dims)[0]->name = var->fullpath;
2730 (var->dims)[0]->newname = var->fullpath;
2731 pair<set<string>::iterator,bool> setret;
2732 setret = dimnamelist.insert((var->dims)[0]->name);
2733 if (true == setret.second)
2734 Insert_One_NameSizeMap_Element((var->dims)[0]->name,(var->dims)[0]->size,(var->dims)[0]->unlimited_dim);
2735 }
2736
2737 // No dimension, add fake dim names, this will rarely happen.
2738 else {
2739
2740 set<hsize_t> fakedimsize;
2741 pair<set<hsize_t>::iterator,bool> setsizeret;
2742 for (vector<Dimension *>::iterator ird= var->dims.begin();
2743 ird != var->dims.end(); ++ird) {
2744 Add_One_FakeDim_Name(*ird);
2745 setsizeret = fakedimsize.insert((*ird)->size);
2746 // Avoid the same size dimension sharing the same dimension name.
2747 if (false == setsizeret.second)
2748 Adjust_Duplicate_FakeDim_Name(*ird);
2749 }
2750 }
2751
2752}
2753
2754// Add dimension names for the case when HDF5 dimension scale is followed(netCDF4-like)
2755void GMFile::Add_UseDimscale_Var_Dim_Names_General_Product(Var *var,Attribute*dimlistattr)
2756{
2757
2758 BESDEBUG("h5", "Coming to Add_UseDimscale_Var_Dim_Names_General_Product()"<<endl);
2759 ssize_t objnamelen = -1;
2760 hobj_ref_t rbuf;
2761 //hvl_t *vlbuf = nullptr;
2762 vector<hvl_t> vlbuf;
2763
2764 hid_t dset_id = -1;
2765 hid_t attr_id = -1;
2766 hid_t atype_id = -1;
2767 hid_t amemtype_id = -1;
2768 hid_t aspace_id = -1;
2769 hid_t ref_dset = -1;
2770
2771 if(nullptr == dimlistattr)
2772 throw2("Cannot obtain the dimension list attribute for variable ",var->name);
2773
2774 else if (0==var->rank)
2775 throw2("The number of dimension should NOT be 0 for the variable ",var->name);
2776
2777 else {
2778 try {
2779
2780 vlbuf.resize(var->rank);
2781
2782 dset_id = H5Dopen(this->fileid,(var->fullpath).c_str(),H5P_DEFAULT);
2783 if (dset_id < 0)
2784 throw2("Cannot open the dataset ",var->fullpath);
2785
2786 attr_id = H5Aopen(dset_id,(dimlistattr->name).c_str(),H5P_DEFAULT);
2787 if (attr_id <0 )
2788 throw4("Cannot open the attribute ",dimlistattr->name," of HDF5 dataset ",var->fullpath);
2789
2790 atype_id = H5Aget_type(attr_id);
2791 if (atype_id <0)
2792 throw4("Cannot obtain the datatype of the attribute ",dimlistattr->name," of HDF5 dataset ",var->fullpath);
2793
2794 amemtype_id = H5Tget_native_type(atype_id, H5T_DIR_ASCEND);
2795
2796 if (amemtype_id < 0)
2797 throw2("Cannot obtain the memory datatype for the attribute ",dimlistattr->name);
2798
2799
2800 if (H5Aread(attr_id,amemtype_id,vlbuf.data()) <0)
2801 throw2("Cannot obtain the referenced object for the variable ",var->name);
2802
2803
2804 vector<char> objname;
2805 int vlbuf_index = 0;
2806
2807 // The dimension names of variables will be the HDF5 dataset names dereferenced from the DIMENSION_LIST attribute.
2808 for (auto ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
2809
2810 if(vlbuf[vlbuf_index].p== nullptr)
2811 throw4("The dimension doesn't exist. Var name is ",var->name,"; the dimension index is ",vlbuf_index);
2812 rbuf =((hobj_ref_t*)vlbuf[vlbuf_index].p)[0];
2813 if ((ref_dset = H5RDEREFERENCE(attr_id, H5R_OBJECT, &rbuf)) < 0)
2814 throw2("Cannot dereference from the DIMENSION_LIST attribute for the variable ",var->name);
2815
2816 if ((objnamelen= H5Iget_name(ref_dset,nullptr,0))<=0)
2817 throw2("Cannot obtain the dataset name dereferenced from the DIMENSION_LIST attribute for the variable ",var->name);
2818 objname.resize(objnamelen+1);
2819 if ((objnamelen= H5Iget_name(ref_dset,objname.data(),objnamelen+1))<=0)
2820 throw2("Cannot obtain the dataset name dereferenced from the DIMENSION_LIST attribute for the variable ",var->name);
2821
2822 auto objname_str = string(objname.begin(),objname.end());
2823
2824 // We need to remove the first character of the object name since the first character
2825 // of the object full path is always "/" and this will be changed to "_".
2826 // The convention of handling the dimension-scale general product is to remove the first "_".
2827 // Check the get_CF_string function of HDF5GMCF.cc.
2828 string trim_objname = objname_str.substr(0,objnamelen);
2829 (*ird)->name = string(trim_objname.begin(),trim_objname.end());
2830
2831 pair<set<string>::iterator,bool> setret;
2832 setret = dimnamelist.insert((*ird)->name);
2833 if (true == setret.second)
2834 Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size,(*ird)->unlimited_dim);
2835 (*ird)->newname = (*ird)->name;
2836 H5Dclose(ref_dset);
2837#if 0
2838 ref_dset = -1;
2839#endif
2840 objname.clear();
2841 vlbuf_index++;
2842 }// end of for ird
2843
2844 if(vlbuf.empty() == false) {
2845
2846 if ((aspace_id = H5Aget_space(attr_id)) < 0)
2847 throw2("Cannot get hdf5 dataspace id for the attribute ",dimlistattr->name);
2848
2849 if (H5Dvlen_reclaim(amemtype_id,aspace_id,H5P_DEFAULT,(void*)vlbuf.data())<0)
2850 throw2("Cannot successfully clean up the variable length memory for the variable ",var->name);
2851
2852 H5Sclose(aspace_id);
2853
2854 }
2855
2856 H5Tclose(atype_id);
2857 H5Tclose(amemtype_id);
2858 H5Aclose(attr_id);
2859 H5Dclose(dset_id);
2860 }
2861
2862 catch(...) {
2863
2864 if(atype_id != -1)
2865 H5Tclose(atype_id);
2866
2867 if(amemtype_id != -1)
2868 H5Tclose(amemtype_id);
2869
2870 if(aspace_id != -1)
2871 H5Sclose(aspace_id);
2872
2873 if(attr_id != -1)
2874 H5Aclose(attr_id);
2875
2876 if(dset_id != -1)
2877 H5Dclose(dset_id);
2878
2879 throw;
2880 }
2881 }
2882
2883}
2884
2885// Handle coordinate variables
2887
2888 BESDEBUG("h5", "GMFile:: Coming to Handle_CVar()"<<endl);
2889 // No coordinate variables are generated for ACOS_L2S or OCO2_L1B
2890 // Currently we support the three patterns for the general products:
2891 // 1) Dimensions follow HDF5 dimension scale specification
2892 // 2) Dimensions don't follow HDF5 dimension scale specification but have 1D lat/lon
2893 // 3) Dimensions don't follow HDF5 dimension scale specification bu have 2D lat/lon
2894 if (General_Product == this->product_type ||
2895 ACOS_L2S_OR_OCO2_L1B == this->product_type) {
2896 if (GENERAL_DIMSCALE == this->gproduct_pattern)
2897 Handle_CVar_Dimscale_General_Product();
2898 else if (GENERAL_LATLON1D == this->gproduct_pattern)
2899 Handle_CVar_LatLon1D_General_Product();
2900 else if (GENERAL_LATLON2D == this->gproduct_pattern)
2901 Handle_CVar_LatLon2D_General_Product();
2902 return;
2903 }
2904
2905 else if (Mea_SeaWiFS_L2 == this->product_type ||
2906 Mea_SeaWiFS_L3 == this->product_type)
2907 Handle_CVar_Mea_SeaWiFS();
2908 else if (Aqu_L3 == this->product_type)
2909 Handle_CVar_Aqu_L3();
2910 else if (OBPG_L3 == this->product_type)
2911 Handle_CVar_OBPG_L3();
2912 else if (OSMAPL2S == this->product_type)
2913 Handle_CVar_OSMAPL2S();
2914 else if (Mea_Ozone == this->product_type)
2915 Handle_CVar_Mea_Ozone();
2916 else if (GPMS_L3 == this->product_type || GPMM_L3 == this->product_type
2917 || GPM_L3_New == this->product_type )
2918 Handle_CVar_GPM_L3();
2919 else if (GPM_L1 == this->product_type)
2920 Handle_CVar_GPM_L1();
2921}
2922
2923// Handle GPM level 1 coordinate variables
2924void GMFile::Handle_CVar_GPM_L1() {
2925
2926 BESDEBUG("h5", "Coming to Handle_CVar_GPM_L1()"<<endl);
2927#if 0
2928 // Loop through the variable list to build the coordinates.
2929 for (vector<Var *>::iterator irv = this->vars.begin();
2930 irv != this->vars.end(); ++irv) {
2931 if((*irv)->name=="AlgorithmRuntimeInfo") {
2932 delete(*irv);
2933 this->vars.erase(irv);
2934 break;
2935 }
2936 }
2937#endif
2938
2939 // Loop through all variables to check 2-D "Latitude" and "Longitude".
2940 // Create coordinate variables based on 2-D "Latitude" and "Longitude".
2941 // Latitude[Xdim][YDim] Longitude[Xdim][YDim], Latitude <->Xdim, Longitude <->YDim.
2942 // Make sure to build cf dimension names cfdimname = latpath+ the lat dimension name.
2943 // We want to save dimension names of Latitude and Longitude since
2944 // the fake coordinate variables of these two dimensions should not be generated.
2945 // So we need to remember these dimension names.
2946 set<string> ll_dim_set;
2947 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
2948 if((*irv)->rank == 2 && (*irv)->name == "Latitude") {
2949 auto GMcvar = new GMCVar(*irv);
2950 size_t lat_pos = (*irv)->fullpath.rfind("Latitude");
2951 string lat_path = (*irv)->fullpath.substr(0,lat_pos);
2952 GMcvar->cfdimname = lat_path + ((*irv)->dims)[0]->name;
2953 ll_dim_set.insert(((*irv)->dims)[0]->name);
2954 GMcvar->cvartype = CV_EXIST;
2955 GMcvar->product_type = product_type;
2956 this->cvars.push_back(GMcvar);
2957 delete(*irv);
2958 irv = this->vars.erase(irv);
2959 }
2960
2961 if((*irv)->rank == 2 && (*irv)->name == "Longitude") {
2962 auto GMcvar = new GMCVar(*irv);
2963 size_t lon_pos = (*irv)->fullpath.rfind("Longitude");
2964 string lon_path = (*irv)->fullpath.substr(0,lon_pos);
2965 GMcvar->cfdimname = lon_path + ((*irv)->dims)[1]->name;
2966 ll_dim_set.insert(((*irv)->dims)[1]->name);
2967 GMcvar->cvartype = CV_EXIST;
2968 GMcvar->product_type = product_type;
2969 this->cvars.push_back(GMcvar);
2970 delete(*irv);
2971 irv = this->vars.erase(irv);
2972 }
2973 else {
2974 ++irv;
2975 }
2976 }// end of for irv
2977
2978#if 0
2979 // Loop through all variables and create a dim set.
2980 set<string> cvdimset;
2981 pair<set<string>::iterator,bool> setret;
2982 for (vector<Var *>::iterator irv = this->vars.begin();
2983 irv != this->vars.end(); ++irv) {
2984 for(vector<Dimension *>::iterator ird = (*irv)->dims.begin();
2985 ird != (*irv)->dims.end(); ++ird) {
2986 setret = cvdimset.insert((*ird)->name);
2987cerr<<"var name is "<<(*irv)->fullpath <<endl;
2988 if (true == setret.second) {
2989cerr<<"dim name is "<<(*ird)->name <<endl;
2990 Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size);
2991 }
2992 }
2993 }// for (vector<Var *>::iterator irv = this->vars.begin();...
2994#endif
2995
2996 // For each dimension, create a coordinate variable.
2997 // Here we just need to loop through the map dimname_to_dimsize,
2998 // use the name and the size to create coordinate variables.
2999 for (map<string,hsize_t>::const_iterator itd = dimname_to_dimsize.begin();
3000 itd!=dimname_to_dimsize.end();++itd) {
3001 // We will not create fake coordinate variables for the
3002 // dimensions of latitude and longitude.
3003 if((ll_dim_set.find(itd->first)) == ll_dim_set.end()) {
3004 auto GMcvar = new GMCVar();
3005 Create_Missing_CV(GMcvar,itd->first);
3006 this->cvars.push_back(GMcvar);
3007 }
3008 }//end of for (map<string,hsize_t>.
3009
3010}
3011
3012// Handle coordinate variables for GPM level 3
3013void GMFile::Handle_CVar_GPM_L3() {
3014
3015 BESDEBUG("h5", "Coming to Handle_CVar_GPM_L3()"<<endl);
3016 iscoard = true;
3017
3018 // Here we just need to loop through the map dimname_to_dimsize,
3019 // use the name and the size to create coordinate variables.
3020 for (map<string,hsize_t>::const_iterator itd = dimname_to_dimsize.begin();
3021 itd!=dimname_to_dimsize.end();++itd) {
3022
3023 auto GMcvar = new GMCVar();
3024 if("nlon" == itd->first || "nlat" == itd->first
3025 || "lnH" == itd->first || "ltH" == itd->first
3026 || "lnL" == itd->first || "ltL" == itd->first) {
3027 GMcvar->name = itd->first;
3028 GMcvar->newname = GMcvar->name;
3029 GMcvar->fullpath = GMcvar->name;
3030 GMcvar->rank = 1;
3031 GMcvar->dtype = H5FLOAT32;
3032 auto gmcvar_dim = new Dimension(itd->second);
3033 gmcvar_dim->name = GMcvar->name;
3034 gmcvar_dim->newname = gmcvar_dim->name;
3035 GMcvar->dims.push_back(gmcvar_dim);
3036 GMcvar->cfdimname = gmcvar_dim->name;
3037 if ("nlat" ==GMcvar->name || "ltH" == GMcvar->name
3038 || "ltL" == GMcvar->name)
3039 GMcvar->cvartype = CV_LAT_MISS;
3040 else if ("nlon" == GMcvar->name || "lnH" == GMcvar->name
3041 || "lnL" == GMcvar->name)
3042 GMcvar->cvartype = CV_LON_MISS;
3043 GMcvar->product_type = product_type;
3044 }
3045 else if (("nlayer" == itd->first && (28 == itd->second || 19 == itd->second)) ||
3046 ("hgt" == itd->first && 5 == itd->second) ||
3047 ("nalt" == itd->first && 5 == itd->second)){
3048 GMcvar->name = itd->first;
3049 GMcvar->newname = GMcvar->name;
3050 GMcvar->fullpath = GMcvar->name;
3051 GMcvar->rank = 1;
3052 GMcvar->dtype = H5FLOAT32;
3053 auto gmcvar_dim = new Dimension(itd->second);
3054 gmcvar_dim->name = GMcvar->name;
3055 gmcvar_dim->newname = gmcvar_dim->name;
3056 GMcvar->dims.push_back(gmcvar_dim);
3057 GMcvar->cfdimname = gmcvar_dim->name;
3058 GMcvar->cvartype = CV_SPECIAL;
3059 GMcvar->product_type = product_type;
3060 }
3061 else
3062 Create_Missing_CV(GMcvar,itd->first);
3063 this->cvars.push_back(GMcvar);
3064 }//end of for
3065}
3066
3067// Handle Coordinate variables for MeaSuRES SeaWiFS
3068void GMFile::Handle_CVar_Mea_SeaWiFS() {
3069
3070 BESDEBUG("h5", "Coming to Handle_CVar_Mea_SeaWiFS()"<<endl);
3071 pair<set<string>::iterator,bool> setret;
3072 set<string>tempdimnamelist = dimnamelist;
3073
3074 for (auto irs = dimnamelist.begin(); irs != dimnamelist.end();++irs) {
3075 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
3076 if ((*irs)== (*irv)->fullpath) {
3077
3078 if (!iscoard && (("/natrack" == (*irs))
3079 || "/nxtrack" == (*irs))) {
3080 ++irv;
3081 continue;
3082 }
3083
3084 if((*irv)->dims.size()!=1)
3085 throw3("Coard coordinate variable ",(*irv)->name, "is not 1D");
3086
3087 // Create Coordinate variables.
3088 tempdimnamelist.erase(*irs);
3089 auto GMcvar = new GMCVar(*irv);
3090 GMcvar->cfdimname = *irs;
3091 GMcvar->cvartype = CV_EXIST;
3092 GMcvar->product_type = product_type;
3093 this->cvars.push_back(GMcvar);
3094 delete(*irv);
3095 irv = this->vars.erase(irv);
3096 } // end of if irs
3097 else if(false == iscoard) {
3098 // 2-D lat/lon, natrack maps to lat, nxtrack maps to lon.
3099 if ((((*irs) =="/natrack") && ((*irv)->fullpath == "/latitude"))
3100 ||(((*irs) =="/nxtrack") && ((*irv)->fullpath == "/longitude"))) {
3101 tempdimnamelist.erase(*irs);
3102 auto GMcvar = new GMCVar(*irv);
3103 GMcvar->cfdimname = *irs;
3104 GMcvar->cvartype = CV_EXIST;
3105 GMcvar->product_type = product_type;
3106 this->cvars.push_back(GMcvar);
3107 delete(*irv);
3108 irv = this->vars.erase(irv);
3109 }
3110 else {
3111 ++irv;
3112 }
3113
3114 }// end of else if(false == iscoard)
3115 else {
3116 ++irv;
3117 }
3118 } // end of for irv ...
3119 } // end of for irs ...
3120
3121 // Creating the missing "third-dimension" according to the dimension names.
3122 // This may never happen for the current MeaSure SeaWiFS, but put it here for code coherence and completeness.
3123 // KY 12-30-2011
3124 for (set<string>::iterator irs = tempdimnamelist.begin();
3125 irs != tempdimnamelist.end();++irs) {
3126 auto GMcvar = new GMCVar();
3127 Create_Missing_CV(GMcvar,*irs);
3128 this->cvars.push_back(GMcvar);
3129 }
3130}
3131
3132// Handle Coordinate varibles for OSMAPL2S(Note: this function doesn't apply to SMAP)
3133void GMFile::Handle_CVar_OSMAPL2S() {
3134
3135 BESDEBUG("h5", "Coming to Handle_CVar_OSMAPL2S()"<<endl);
3136 pair<set<string>::iterator,bool> setret;
3137 set<string>tempdimnamelist = dimnamelist;
3138 string tempvarname;
3139 string key0 = "_lat";
3140 string key1 = "_lon";
3141 string osmapl2sdim0 ="YDim";
3142 string osmapl2sdim1 ="XDim";
3143
3144 bool foundkey0 = false;
3145 bool foundkey1 = false;
3146
3147 set<string> itset;
3148
3149 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
3150
3151 tempvarname = (*irv)->name;
3152
3153 if ((tempvarname.size() > key0.size())&&
3154 (key0 == tempvarname.substr(tempvarname.size()-key0.size(),key0.size()))){
3155
3156 foundkey0 = true;
3157
3158 if (dimnamelist.find(osmapl2sdim0)== dimnamelist.end())
3159 throw5("variable ",tempvarname," must have dimension ",osmapl2sdim0," , but not found ");
3160
3161 tempdimnamelist.erase(osmapl2sdim0);
3162 auto GMcvar = new GMCVar(*irv);
3163 GMcvar->newname = GMcvar->name; // Remove the path, just use the variable name
3164 GMcvar->cfdimname = osmapl2sdim0;
3165 GMcvar->cvartype = CV_EXIST;
3166 GMcvar->product_type = product_type;
3167 this->cvars.push_back(GMcvar);
3168 delete(*irv);
3169 irv = this->vars.erase(irv);
3170 }// end of if
3171
3172 else if ((tempvarname.size() > key1.size())&&
3173 (key1 == tempvarname.substr(tempvarname.size()-key1.size(),key1.size()))){
3174
3175 foundkey1 = true;
3176
3177 if (dimnamelist.find(osmapl2sdim1)== dimnamelist.end())
3178 throw5("variable ",tempvarname," must have dimension ",osmapl2sdim1," , but not found ");
3179
3180 tempdimnamelist.erase(osmapl2sdim1);
3181
3182 auto GMcvar = new GMCVar(*irv);
3183 GMcvar->newname = GMcvar->name;
3184 GMcvar->cfdimname = osmapl2sdim1;
3185 GMcvar->cvartype = CV_EXIST;
3186 GMcvar->product_type = product_type;
3187 this->cvars.push_back(GMcvar);
3188 delete(*irv);
3189 irv = this->vars.erase(irv);
3190 }// end of else if ((tempvarname.size() > key1.size())&& ...
3191 else {
3192 ++irv;
3193 }
3194 if (true == foundkey0 && true == foundkey1)
3195 break;
3196 } // end of for
3197
3198 for (auto irs = tempdimnamelist.begin(); irs != tempdimnamelist.end();++irs) {
3199
3200 auto GMcvar = new GMCVar();
3201 Create_Missing_CV(GMcvar,*irs);
3202 this->cvars.push_back(GMcvar);
3203 }
3204
3205}
3206
3207// Handle coordinate variables for Aquarius level 3 products
3208void GMFile::Handle_CVar_Aqu_L3() {
3209
3210 BESDEBUG("h5", "Coming to Handle_CVar_Aqu_L3()"<<endl);
3211 iscoard = true;
3212 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3213
3214 if ( "l3m_data" == (*irv)->name) {
3215 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
3216 auto GMcvar = new GMCVar();
3217 GMcvar->name = (*ird)->name;
3218 GMcvar->newname = GMcvar->name;
3219 GMcvar->fullpath = GMcvar->name;
3220 GMcvar->rank = 1;
3221 GMcvar->dtype = H5FLOAT32;
3222 auto gmcvar_dim = new Dimension((*ird)->size);
3223 gmcvar_dim->name = GMcvar->name;
3224 gmcvar_dim->newname = gmcvar_dim->name;
3225 GMcvar->dims.push_back(gmcvar_dim);
3226 GMcvar->cfdimname = gmcvar_dim->name;
3227 if ("lat" ==GMcvar->name ) GMcvar->cvartype = CV_LAT_MISS;
3228 if ("lon" == GMcvar->name ) GMcvar->cvartype = CV_LON_MISS;
3229 GMcvar->product_type = product_type;
3230 this->cvars.push_back(GMcvar);
3231 } // end of for
3232 } // end of if
3233 }//end of for
3234
3235}
3236
3237//Handle coordinate variables for MeaSuRES Ozone products
3238void GMFile::Handle_CVar_Mea_Ozone() {
3239
3240 BESDEBUG("h5", "Coming to Handle_CVar_Mea_Ozone()"<<endl);
3241 pair<set<string>::iterator,bool> setret;
3242 set<string>tempdimnamelist = dimnamelist;
3243
3244 if(false == iscoard)
3245 throw1("Measure Ozone level 3 zonal average product must follow COARDS conventions");
3246
3247 for (auto irs = dimnamelist.begin(); irs != dimnamelist.end();++irs) {
3248 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
3249 if ((*irs)== (*irv)->fullpath) {
3250
3251 if((*irv)->dims.size()!=1)
3252 throw3("Coard coordinate variable",(*irv)->name, "is not 1D");
3253
3254 // Create Coordinate variables.
3255 tempdimnamelist.erase(*irs);
3256 auto GMcvar = new GMCVar(*irv);
3257 GMcvar->cfdimname = *irs;
3258 GMcvar->cvartype = CV_EXIST;
3259 GMcvar->product_type = product_type;
3260 this->cvars.push_back(GMcvar);
3261 delete(*irv);
3262 irv = this->vars.erase(irv);
3263 } // end of if
3264 else {
3265 ++irv;
3266 }
3267 } // end of for irv
3268 } // end of for irs
3269
3270 for (auto irs = tempdimnamelist.begin(); irs != tempdimnamelist.end();irs++) {
3271
3272 auto GMcvar = new GMCVar();
3273 Create_Missing_CV(GMcvar,*irs);
3274 this->cvars.push_back(GMcvar);
3275 }
3276}
3277
3278// Handle coordinate variables for general products that use HDF5 dimension scales.
3279void GMFile::Handle_CVar_Dimscale_General_Product() {
3280
3281 BESDEBUG("h5", "Coming to Handle_CVar_Dimscale_General_Product"<<endl);
3282 pair<set<string>::iterator,bool> setret;
3283 set<string>tempdimnamelist = dimnamelist;
3284
3285 for (auto irs = dimnamelist.begin(); irs != dimnamelist.end();++irs) {
3286 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
3287
3288 // This is the dimension scale dataset; it should be changed to a coordinate variable.
3289 if ((*irs)== (*irv)->fullpath) {
3290 if((*irv)->dims.size()!=1)
3291 throw3("COARDS coordinate variable",(*irv)->name, "is not 1D");
3292
3293 // Create Coordinate variables.
3294 tempdimnamelist.erase(*irs);
3295 auto GMcvar = new GMCVar(*irv);
3296 GMcvar->cfdimname = *irs;
3297
3298 // Check if this is just a netCDF-4 dimension.
3299 bool is_netcdf_dimension = Is_netCDF_Dimension(*irv);
3300
3301 // If this is just the netcdf dimension, we
3302 // will fill in the index numbers.
3303 if (true == is_netcdf_dimension)
3304 GMcvar->cvartype = CV_FILLINDEX;
3305 else
3306 GMcvar->cvartype = CV_EXIST;
3307 GMcvar->product_type = product_type;
3308 this->cvars.push_back(GMcvar);
3309 delete(*irv);
3310 irv = this->vars.erase(irv);
3311 } // end of if irs
3312 else {
3313 ++irv;
3314 }
3315 } // end of for irv
3316 } // end of for irs
3317
3318 // Check if we have 2-D lat/lon CVs, and if yes, add those to the CV list.
3319 Update_M2DLatLon_Dimscale_CVs();
3320
3321 // Add other missing coordinate variables.
3322 for (auto irs = tempdimnamelist.begin(); irs != tempdimnamelist.end();irs++) {
3323 auto GMcvar = new GMCVar();
3324 Create_Missing_CV(GMcvar,*irs);
3325 this->cvars.push_back(GMcvar);
3326 }
3327
3328
3329//Debugging
3330#if 0
3331for (set<string>::iterator irs = dimnamelist.begin();
3332 irs != dimnamelist.end();irs++) {
3333cerr<<"dimension name is "<<(*irs)<<endl;
3334}
3335#endif
3336
3337}
3338
3339
3340// Check if we have 2-D lat/lon CVs in a netCDF-4-like file, and if yes, add those to the CV list.
3341// This routine is a really complicate one. There are 9 steps to generate right 2-D lat/lon CVs.
3342void GMFile::Update_M2DLatLon_Dimscale_CVs() {
3343
3344 BESDEBUG("h5", "Coming to Update_M2DLatLon_Dimscale_CVs()"<<endl);
3345 // If this is not a file that only includes 1-D lat/lon CVs
3346 if(false == Check_1DGeolocation_Dimscale()) {
3347
3348 // Define temporary vectors to store 1-D lat/lon CVs
3349 vector<GMCVar*> tempcvar_1dlat;
3350 vector<GMCVar*> tempcvar_1dlon;
3351
3352 // 1. Obtain 1-D lat/lon CVs(only search the CF units and the reserved lat/lon names)
3353 Obtain_1DLatLon_CVs(tempcvar_1dlat,tempcvar_1dlon);
3354
3355 // Define temporary vectors to store 2-D lat/lon Vars
3356 vector<Var*> tempcvar_2dlat;
3357 vector<Var*> tempcvar_2dlon;
3358
3359 // This map remembers the positions of the latlon vars in the vector var.
3360 // Remembering the positions avoids the searching of these lat and lon again when
3361 // deleting them for the var vector and adding them(only the CVs) to the CV vector.
3362 // KY 2015-12-23
3363 map<string,int> latlon2d_path_to_index;
3364
3365 // 2. Obtain 2-D lat/lon variables(only search the CF units and the reserved names)
3366 Obtain_2DLatLon_Vars(tempcvar_2dlat,tempcvar_2dlon,latlon2d_path_to_index);
3367
3368#if 0
3369for(vector<GMCVar *>::iterator irv = tempcvar_1dlat.begin();irv != tempcvar_1dlat.end();++irv)
3370cerr<<"1-D lat variable full path is "<<(*irv)->fullpath <<endl;
3371for(vector<GMCVar *>::iterator irv = tempcvar_1dlon.begin();irv != tempcvar_1dlon.end();++irv)
3372cerr<<"1-D lon variable full path is "<<(*irv)->fullpath <<endl;
3373
3374for(vector<Var *>::iterator irv = tempcvar_2dlat.begin();irv != tempcvar_2dlat.end();++irv)
3375cerr<<"2-D lat variable full path is "<<(*irv)->fullpath <<endl;
3376for(vector<Var *>::iterator irv = tempcvar_2dlon.begin();irv != tempcvar_2dlon.end();++irv)
3377cerr<<"2-D lon variable full path is "<<(*irv)->fullpath <<endl;
3378#endif
3379
3380 // 3. Sequeeze the 2-D lat/lon vectors by removing the ones that share the same dims with 1-D lat/lon CVs.
3381 Obtain_2DLLVars_With_Dims_not_1DLLCVars(tempcvar_2dlat,tempcvar_2dlon,tempcvar_1dlat,tempcvar_1dlon,latlon2d_path_to_index);
3382
3383#if 0
3384for(vector<Var *>::iterator irv = tempcvar_2dlat.begin();irv != tempcvar_2dlat.end();++irv)
3385cerr<<"2-D Left lat variable full path is "<<(*irv)->fullpath <<endl;
3386for(vector<Var *>::iterator irv = tempcvar_2dlon.begin();irv != tempcvar_2dlon.end();++irv)
3387cerr<<"2-D Left lon variable full path is "<<(*irv)->fullpath <<endl;
3388#endif
3389
3390 // 4. Assemble the final 2-D lat/lon CV candidate vectors by checking if the corresponding 2-D lon of a 2-D lat shares
3391 // the same dimension and under the same group and if there is another pair of 2-D lat/lon under the same group.
3392 Obtain_2DLLCVar_Candidate(tempcvar_2dlat,tempcvar_2dlon,latlon2d_path_to_index);
3393
3394#if 0
3395for(vector<Var *>::iterator irv = tempcvar_2dlat.begin();irv != tempcvar_2dlat.end();++irv)
3396cerr<<"Final candidate 2-D Left lat variable full path is "<<(*irv)->fullpath <<endl;
3397for(vector<Var *>::iterator irv = tempcvar_2dlon.begin();irv != tempcvar_2dlon.end();++irv)
3398cerr<<"Final candidate 2-D Left lon variable full path is "<<(*irv)->fullpath <<endl;
3399#endif
3400
3401 // 5. Remove the 2-D lat/lon variables that are to be used as CVs from the vector that stores general variables
3402 // var2d_index, remembers the index of the 2-D lat/lon CVs in the original vector of vars.
3403 vector<int> var2d_index;
3404 for (map<string,int>::const_iterator it= latlon2d_path_to_index.begin();it!=latlon2d_path_to_index.end();++it)
3405 var2d_index.push_back(it->second);
3406
3407 Remove_2DLLCVar_Final_Candidate_from_Vars(var2d_index);
3408
3409 // 6. If we have 2-D CVs, COARDS should be turned off.
3410 if(tempcvar_2dlat.size()>0)
3411 iscoard = false;
3412
3413 // 7. Add the CVs based on the final 2-D lat/lon CV candidates.
3414 // We need to remember the dim names that the 2-D lat/lon CVs are associated with.
3415 set<string>dim_names_2d_cvs;
3416
3417 for(auto irv = tempcvar_2dlat.begin();irv != tempcvar_2dlat.end();++irv){
3418 auto lat = new GMCVar(*irv);
3419 // Latitude is always corresponding to the first dimension.
3420 lat->cfdimname = (*irv)->getDimensions()[0]->name;
3421 dim_names_2d_cvs.insert(lat->cfdimname);
3422 lat->cvartype = CV_EXIST;
3423 lat->product_type = product_type;
3424 this->cvars.push_back(lat);
3425 }
3426 for(auto irv = tempcvar_2dlon.begin();irv != tempcvar_2dlon.end();++irv){
3427 auto lon = new GMCVar(*irv);
3428 // Longitude is always corresponding to the second dimension.
3429 lon->cfdimname = (*irv)->getDimensions()[1]->name;
3430 dim_names_2d_cvs.insert(lon->cfdimname);
3431 lon->cvartype = CV_EXIST;
3432 lon->product_type = product_type;
3433 this->cvars.push_back(lon);
3434 }
3435
3436 // 8. Move the originally assigned 1-D CVs that are replaced by 2-D CVs back to the general variable list.
3437 // Also remove the CV created by the pure dimensions.
3438 // Dimension names are used to identify those 1-D CVs.
3439 for(auto ircv= this->cvars.begin();ircv !=this->cvars.end();) {
3440 if(1 == (*ircv)->rank) {
3441 if(dim_names_2d_cvs.find((*ircv)->cfdimname)!=dim_names_2d_cvs.end()) {
3442 if(CV_FILLINDEX == (*ircv)->cvartype) {// This is pure dimension
3443 delete(*ircv);
3444 ircv = this->cvars.erase(ircv);
3445 }
3446 else if(CV_EXIST == (*ircv)->cvartype) {// This var exists already
3447
3448 // Add this var. to the var list.
3449 Var *var = new Var(*ircv);
3450 this->vars.push_back(var);
3451
3452 // Remove this var. from the GMCVar list.
3453 delete(*ircv);
3454 ircv = this->cvars.erase(ircv);
3455
3456 }
3457 else {// the removed 1-D coordinate variable should be either the CV_FILLINDEX or CV_EXIST.
3458 if(CV_LAT_MISS == (*ircv)->cvartype)
3459 throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_LAT_MISS");
3460 else if(CV_LON_MISS == (*ircv)->cvartype)
3461 throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_LON_MISS");
3462 else if(CV_NONLATLON_MISS == (*ircv)->cvartype)
3463 throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_NONLATLON_MISS");
3464 else if(CV_MODIFY == (*ircv)->cvartype)
3465 throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_MODIFY");
3466 else if(CV_SPECIAL == (*ircv)->cvartype)
3467 throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_SPECIAL");
3468 else
3469 throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_UNSUPPORTED");
3470 }
3471
3472 }
3473 else
3474 ++ircv;
3475
3476 }
3477 else
3478 ++ircv;
3479
3480 }
3481
3482
3483#if 0
3484//if(iscoard == true)
3485//cerr<<"COARD is true"<<endl;
3486for(set<string>::iterator irs = grp_cv_paths.begin();irs != grp_cv_paths.end();++irs) {
3487cerr<<"group path is "<< (*irs)<<endl;
3488
3489}
3490#endif
3491
3492#if 0
3493//Print CVs
3494cerr<<"File name is "<< this->path <<endl;
3495cerr<<"CV names are the following "<<endl;
3496for (vector<GMCVar *>:: iterator i= this->cvars.begin(); i!=this->cvars.end(); ++i)
3497cerr<<(*i)->fullpath <<endl;
3498#endif
3499
3500
3501 // 9. release the resources allocated by the temporary vectors.
3502 release_standalone_GMCVar_vector(tempcvar_1dlat);
3503 release_standalone_GMCVar_vector(tempcvar_1dlon);
3504 release_standalone_var_vector(tempcvar_2dlat);
3505 release_standalone_var_vector(tempcvar_2dlon);
3506 }// if(false == Check_1DGeolocation_Dimscale())
3507#if 0
3508for (vector<GMCVar *>:: iterator i= this->cvars.begin(); i!=this->cvars.end(); ++i)
3509cerr<<(*i)->fullpath <<endl;
3510#endif
3511
3512}
3513
3514// If Check_1DGeolocation_Dimscale() is true, no need to build 2-D lat/lon coordinate variables.
3515// This function is introduced to avoid the performance penalty caused by handling the general 2-D lat/lon case.
3516bool GMFile::Check_1DGeolocation_Dimscale() {
3517
3518 BESDEBUG("h5", "Coming to Check_1DGeolocation_Dimscale()"<<endl);
3519 bool has_only_1d_geolocation_cv = false;
3520 bool has_1d_lat_cv_flag = false;
3521 bool has_1d_lon_cv_flag = false;
3522
3523 string lat_dimname;
3524 hsize_t lat_size = 0;
3525
3526 string lon_dimname;
3527 hsize_t lon_size = 0;
3528
3529 // We need to consider both 1-D lat/lon and the 1-D zonal average case(1-D lat only).
3530 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
3531 if((*ircv)->cvartype == CV_EXIST) {
3532 string attr_name ="units";
3533 string lat_unit_value = "degrees_north";
3534 string lon_unit_value = "degrees_east";
3535
3536 for(auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end();ira++) {
3537
3538 if(true == Is_Str_Attr(*ira,(*ircv)->fullpath,attr_name,lat_unit_value)) {
3539 lat_size = (*ircv)->getDimensions()[0]->size;
3540 lat_dimname = (*ircv)->getDimensions()[0]->name;
3541 has_1d_lat_cv_flag = true;
3542 break;
3543 }
3544 else if(true == Is_Str_Attr(*ira,(*ircv)->fullpath,attr_name,lon_unit_value)){
3545 lon_size = (*ircv)->getDimensions()[0]->size;
3546 lon_dimname = (*ircv)->getDimensions()[0]->name;
3547 has_1d_lon_cv_flag = true;
3548 break;
3549 }
3550 }
3551 }
3552 }
3553
3554 // If having 1-D lat/lon CVs, this is a good sign for only 1-D lat/lon CVs ,
3555 // just need to have a couple of checks.
3556 if(true == has_1d_lat_cv_flag ) {
3557
3558 if(true == has_1d_lon_cv_flag) {
3559
3560 // Come to the possible classic netCDF-4 case,
3561 if(0 == this->groups.size()) {
3562
3563 // Rarely happens when lat_size is the same as the lon_size.
3564 // However, still want to make sure there is a 2-D variable that uses both lat and lon dims.
3565 if(lat_size == lon_size) {
3566 bool var_has_latdim = false;
3567 bool var_has_londim = false;
3568 for (auto irv = this->vars.begin();
3569 irv != this->vars.end(); ++irv) {
3570 if((*irv)->rank >= 2) {
3571 for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
3572 if((*ird)->name == lat_dimname)
3573 var_has_latdim = true;
3574 else if((*ird)->name == lon_dimname)
3575 var_has_londim = true;
3576 }
3577 if(true == var_has_latdim && true == var_has_londim) {
3578 has_only_1d_geolocation_cv = true;
3579 break;
3580 }
3581 else {
3582 var_has_latdim = false;
3583 var_has_londim = false;
3584 }
3585 }
3586 }
3587 }
3588 else
3589 has_only_1d_geolocation_cv = true;
3590 }// if(0 == this->groups.size())
3591 else {
3592 // Multiple groups, need to check if having 2-D lat/lon pairs
3593 bool has_2d_latname_flag = false;
3594 bool has_2d_lonname_flag = false;
3595 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3596 if((*irv)->rank == 2) {
3597
3598 //Note: When the 2nd parameter is true in the function Is_geolatlon, it checks the lat/latitude/Latitude
3599 if(true == Is_geolatlon((*irv)->name,true))
3600 has_2d_latname_flag = true;
3601
3602 //Note: When the 2nd parameter is false in the function Is_geolatlon, it checks the lon/longitude/Longitude
3603 else if(true == Is_geolatlon((*irv)->name,false))
3604 has_2d_lonname_flag = true;
3605
3606 if((true == has_2d_latname_flag) && (true == has_2d_lonname_flag))
3607 break;
3608 }
3609 }
3610
3611 if(has_2d_latname_flag != true || has_2d_lonname_flag != true) {
3612
3613 //Check if having the 2-D lat/lon by checking if having lat/lon CF units(lon's units: degrees_east lat's units: degrees_north)
3614 has_2d_latname_flag = false;
3615 has_2d_lonname_flag = false;
3616
3617 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3618 if((*irv)->rank == 2) {
3619 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
3620
3621 if (false == has_2d_latname_flag) {
3622
3623 // When the third parameter of the function has_latlon_cf_units is set to true, it checks latitude
3624 has_2d_latname_flag = has_latlon_cf_units((*ira),(*irv)->fullpath,true);
3625 if(true == has_2d_latname_flag)
3626 break;
3627 else if(false == has_2d_lonname_flag) {
3628
3629 // When the third parameter of the function has_latlon_cf_units is set to false, it checks longitude
3630 has_2d_lonname_flag = has_latlon_cf_units((*ira),(*irv)->fullpath,false);
3631 if(true == has_2d_lonname_flag)
3632 break;
3633 }
3634 }
3635 else if(false == has_2d_lonname_flag) {
3636
3637 // Now has_2d_latname_flag is true, just need to check the has_2d_lonname_flag
3638 // When the third parameter of has_latlon_cf_units is set to false, it checks longitude
3639 has_2d_lonname_flag = has_latlon_cf_units((*ira),(*irv)->fullpath,false);
3640 if(true == has_2d_lonname_flag)
3641 break;
3642 }
3643 }
3644 if(true == has_2d_latname_flag && true == has_2d_lonname_flag)
3645 break;
3646 }
3647 }
3648 }// if(has_2d_latname_flag != true || has_2d_lonname_flag != true)
3649
3650 // If we cannot find either of 2-D any lat/lon variables, this file is treated as having only 1-D lat/lon.
3651 if(has_2d_latname_flag != true || has_2d_lonname_flag != true)
3652 has_only_1d_geolocation_cv = true;
3653 }
3654
3655 }//
3656 else {//Zonal average case, we do not need to find 2-D lat/lon CVs.
3657 has_only_1d_geolocation_cv = true;
3658 }
3659
3660 }
3661
3662#if 0
3663if(has_only_1d_geolocation_cv == true)
3664cerr <<"has only 1D lat/lon CVs. "<<endl;
3665else
3666cerr<<"Possibly has 2D lat/lon CVs. "<<endl;
3667#endif
3668
3669 return has_only_1d_geolocation_cv;
3670
3671}
3672
3673// Obtain the originally assigned 1-D lat/lon coordinate variables.
3674// This function should be used before generating any 2-D lat/lon CVs.
3675void GMFile::Obtain_1DLatLon_CVs(vector<GMCVar*> &cvar_1dlat,vector<GMCVar*> &cvar_1dlon) {
3676
3677 BESDEBUG("h5", "Coming to Obtain_1DLatLon_CVs()"<<endl);
3678 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
3679
3680 if((*ircv)->cvartype == CV_EXIST) {
3681
3682 string attr_name ="units";
3683 string lat_unit_value = "degrees_north";
3684 string lon_unit_value = "degrees_east";
3685
3686 for(auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end();ira++) {
3687
3688 // 1-D latitude
3689 if(true == Is_Str_Attr(*ira,(*ircv)->fullpath,attr_name,lat_unit_value)) {
3690 auto lat = new GMCVar(*ircv);
3691 lat->cfdimname = (*ircv)->getDimensions()[0]->name;
3692 lat->cvartype = (*ircv)->cvartype;
3693 lat->product_type = (*ircv)->product_type;
3694 cvar_1dlat.push_back(lat);
3695 }
3696 // 1-D longitude
3697 else if(true == Is_Str_Attr(*ira,(*ircv)->fullpath,attr_name,lon_unit_value)){
3698 auto lon = new GMCVar(*ircv);
3699 lon->cfdimname = (*ircv)->getDimensions()[0]->name;
3700 lon->cvartype = (*ircv)->cvartype;
3701 lon->product_type = (*ircv)->product_type;
3702 cvar_1dlon.push_back(lon);
3703 }
3704 }
3705 }// end of if((*ircv)->cvartype == CV_EXIST)
3706 }// end of for (ircv = this->cvars.begin();
3707
3708}
3709
3710// Obtain all 2-D lat/lon variables. We first check the lat/latitude/Latitude names, if not found, we check if CF lat/lon units are present.
3711// Latitude variables are saved in the vector var_2dlat. Longitude variables are saved in the vector var_2dlon.
3712// We also remember the index of these lat/lon in the original var vector.
3713void GMFile::Obtain_2DLatLon_Vars(vector<Var*> &var_2dlat,vector<Var*> &var_2dlon,map<string,int> & latlon2d_path_to_index) {
3714
3715 BESDEBUG("h5", "Coming to Obtain_2DLatLon_Vars()"<<endl);
3716 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3717 if((*irv)->rank == 2) {
3718
3719 //Note: When the 2nd parameter is true in the function Is_geolatlon, it checks the lat/latitude/Latitude
3720 if(true == Is_geolatlon((*irv)->name,true)) {
3721 Var *lat = new Var(*irv);
3722 var_2dlat.push_back(lat);
3723 latlon2d_path_to_index[(*irv)->fullpath]= distance(this->vars.begin(),irv);
3724 continue;
3725 }
3726 else {
3727
3728 bool has_2dlat = false;
3729 for (auto ira = (*irv)->attrs.begin();
3730 ira != (*irv)->attrs.end(); ++ira) {
3731
3732 // When the third parameter of has_latlon_cf_units is set to true, it checks latitude
3733 if(true == has_latlon_cf_units((*ira),(*irv)->fullpath,true)) {
3734 Var *lat = new Var(*irv);
3735 var_2dlat.push_back(lat);
3736 latlon2d_path_to_index[(*irv)->fullpath] = distance(this->vars.begin(),irv);
3737 has_2dlat = true;
3738 break;
3739 }
3740 }
3741
3742 if(true == has_2dlat)
3743 continue;
3744 }
3745
3746 //Note: When the 2nd parameter is false in the function Is_geolatlon, it checks the lon/longitude/Longitude
3747 if(true == Is_geolatlon((*irv)->name,false)) {
3748 Var *lon = new Var(*irv);
3749 latlon2d_path_to_index[(*irv)->fullpath] = distance(this->vars.begin(),irv);
3750 var_2dlon.push_back(lon);
3751 }
3752 else {
3753 for (auto ira = (*irv)->attrs.begin();
3754 ira != (*irv)->attrs.end(); ++ira) {
3755
3756 // When the third parameter of has_latlon_cf_units is set to false, it checks longitude
3757 if(true == has_latlon_cf_units((*ira),(*irv)->fullpath,false)) {
3758 Var *lon = new Var(*irv);
3759 latlon2d_path_to_index[(*irv)->fullpath] = distance(this->vars.begin(),irv);
3760 var_2dlon.push_back(lon);
3761 break;
3762 }
3763 }
3764 }
3765 } // if((*irv)->rank == 2)
3766 } // for (auto irv
3767}
3768
3769// Sequeeze the 2-D lat/lon vectors by removing the ones that share the same dims with 1-D lat/lon CVs.
3770// The latlon2d_path_to_index map also needs to be updated.
3771void GMFile::Obtain_2DLLVars_With_Dims_not_1DLLCVars(vector<Var*> &var_2dlat,
3772 vector<Var*> &var_2dlon,
3773 vector<GMCVar*> &cvar_1dlat,
3774 vector<GMCVar*> &cvar_1dlon,
3775 map<string,int> &latlon2d_path_to_index) {
3776
3777 BESDEBUG("h5", "Coming to Obtain_2DLLVars_With_Dims_not_1DLLCVars()"<<endl);
3778 // First latitude at var_2dlat
3779 for(auto irv = var_2dlat.begin();irv != var_2dlat.end();) {
3780 bool remove_2dlat = false;
3781 for(auto ircv = cvar_1dlat.begin();ircv != cvar_1dlat.end();++ircv) {
3782 for (auto ird = (*irv)->dims.begin(); ird!=(*irv)->dims.end(); ++ird) {
3783 if((*ird)->name == (*ircv)->getDimensions()[0]->name &&
3784 (*ird)->size == (*ircv)->getDimensions()[0]->size) {
3785 latlon2d_path_to_index.erase((*irv)->fullpath);
3786 delete(*irv);
3787 irv = var_2dlat.erase(irv);
3788 remove_2dlat = true;
3789 break;
3790 }
3791 }
3792 if(true == remove_2dlat)
3793 break;
3794 }
3795
3796 if(false == remove_2dlat)
3797 ++irv;
3798 }// end of for(auto irv = var_2dlat.begin()
3799
3800 // Second longitude
3801 for(auto irv = var_2dlon.begin();irv != var_2dlon.end();) {
3802 bool remove_2dlon = false;
3803 for(auto ircv = cvar_1dlon.begin();ircv != cvar_1dlon.end();++ircv) {
3804 for (auto ird = (*irv)->dims.begin();
3805 ird!=(*irv)->dims.end(); ++ird) {
3806 if((*ird)->name == (*ircv)->getDimensions()[0]->name &&
3807 (*ird)->size == (*ircv)->getDimensions()[0]->size) {
3808 latlon2d_path_to_index.erase((*irv)->fullpath);
3809 delete(*irv);
3810 irv = var_2dlon.erase(irv);
3811 remove_2dlon = true;
3812 break;
3813 }
3814 }
3815 if(true == remove_2dlon)
3816 break;
3817 }
3818
3819 if(false == remove_2dlon)
3820 ++irv;
3821 } // end of for(auto irv = var_2dlon.begin()
3822
3823}
3824
3825//Out of the collected 2-D lat/lon variables, we will select the final qualified 2-D lat/lon as CVs.
3826void GMFile::Obtain_2DLLCVar_Candidate(vector<Var*> &var_2dlat,
3827 vector<Var*> &var_2dlon,
3828 map<string,int>& latlon2d_path_to_index) {
3829 BESDEBUG("h5", "Coming to Obtain_2DLLCVar_Candidate()"<<endl);
3830 // First check 2-D lat, see if we have the corresponding 2-D lon(same dims, under the same group).
3831 // If no, remove that lat from the vector.
3832 vector<string> lon2d_group_paths;
3833
3834 for(auto irv_2dlat = var_2dlat.begin();irv_2dlat !=var_2dlat.end();) {
3835 for(auto irv_2dlon = var_2dlon.begin();irv_2dlon != var_2dlon.end();++irv_2dlon) {
3836 if(((*irv_2dlat)->getDimensions()[0]->name == (*irv_2dlon)->getDimensions()[0]->name) &&
3837 ((*irv_2dlat)->getDimensions()[0]->size == (*irv_2dlon)->getDimensions()[0]->size) &&
3838 ((*irv_2dlat)->getDimensions()[1]->name == (*irv_2dlon)->getDimensions()[1]->name) &&
3839 ((*irv_2dlat)->getDimensions()[1]->size == (*irv_2dlon)->getDimensions()[1]->size))
3840 lon2d_group_paths.push_back(HDF5CFUtil::obtain_string_before_lastslash((*irv_2dlon)->fullpath));
3841 }
3842 // Doesn't find any lons that shares the same dims,remove this lat from the 2dlat vector,
3843 // also update the latlon2d_path_to_index map
3844 if(0 == lon2d_group_paths.size()) {
3845 latlon2d_path_to_index.erase((*irv_2dlat)->fullpath);
3846 delete(*irv_2dlat);
3847 irv_2dlat = var_2dlat.erase(irv_2dlat);
3848 }
3849 else {// Find lons,check if they are under the same group
3850#if 0
3851 //string lat2d_group_path = (*irv_2dlat)->fullpath.substr(0,(*irv_2dlat)->fullpath.find_last_of("/"));
3852#endif
3853 string lat2d_group_path = HDF5CFUtil::obtain_string_before_lastslash((*irv_2dlat)->fullpath);
3854
3855 // Check how many lon2d shares the same group with the lat2d
3856 short lon2d_has_lat2d_group_path_flag = 0;
3857 for(auto ivs = lon2d_group_paths.begin();ivs!=lon2d_group_paths.end();++ivs) {
3858 if((*ivs)==lat2d_group_path)
3859 lon2d_has_lat2d_group_path_flag++;
3860 }
3861
3862 // No lon2d shares the same group with the lat2d, remove this lat2d
3863 if(0 == lon2d_has_lat2d_group_path_flag) {
3864 latlon2d_path_to_index.erase((*irv_2dlat)->fullpath);
3865 delete(*irv_2dlat);
3866 irv_2dlat = var_2dlat.erase(irv_2dlat);
3867 }
3868 // Only one lon2d, yes, keep it.
3869 else if (1== lon2d_has_lat2d_group_path_flag) {
3870 ++irv_2dlat;
3871 }
3872 // More than 1 lon2d, we will remove the lat2d, but save the group path so that we may
3873 // flatten the variable path stored in the coordinates attribute under this group.
3874 else {
3875 // Save the group path for the future use.
3876 grp_cv_paths.insert(lat2d_group_path);
3877 latlon2d_path_to_index.erase((*irv_2dlat)->fullpath);
3878 delete(*irv_2dlat);
3879 irv_2dlat = var_2dlat.erase(irv_2dlat);
3880 }
3881 }
3882
3883 //Clear the vector that stores the same dim. since it is only applied to this lat,
3884 lon2d_group_paths.clear();
3885 }
3886
3887#if 0
3888for(auto irv_2dlat = var_2dlat.begin();irv_2dlat !=var_2dlat.end();++irv_2dlat)
3889cerr<<"2 left 2-D lat variable full path is: "<<(*irv_2dlat)->fullpath <<endl;
3890#endif
3891
3892
3893 // Second check 2-D lon, see if we have the corresponding 2-D lat(same dims, under the same group).
3894 // If no, remove that lon from the vector.
3895 vector<string> lat2d_group_paths;
3896
3897 // Check the longitude
3898 for(auto irv_2dlon = var_2dlon.begin();irv_2dlon !=var_2dlon.end();) {
3899 for(auto irv_2dlat = var_2dlat.begin();irv_2dlat != var_2dlat.end();++irv_2dlat) {
3900 if(((*irv_2dlat)->getDimensions()[0]->name == (*irv_2dlon)->getDimensions()[0]->name) &&
3901 ((*irv_2dlat)->getDimensions()[0]->size == (*irv_2dlon)->getDimensions()[0]->size) &&
3902 ((*irv_2dlat)->getDimensions()[1]->name == (*irv_2dlon)->getDimensions()[1]->name) &&
3903 ((*irv_2dlat)->getDimensions()[1]->size == (*irv_2dlon)->getDimensions()[1]->size))
3904 lat2d_group_paths.push_back(HDF5CFUtil::obtain_string_before_lastslash((*irv_2dlat)->fullpath));
3905#if 0
3906 //lat2d_group_paths.push_back((*irv_2dlat)->fullpath.substr(0,(*irv_2dlat)->fullpath.find_last_of("/")));
3907#endif
3908 }
3909 // Doesn't find any lats that shares the same dims,remove this lon from this vector
3910 if(lat2d_group_paths.empty()) {
3911 latlon2d_path_to_index.erase((*irv_2dlon)->fullpath);
3912 delete(*irv_2dlon);
3913 irv_2dlon = var_2dlon.erase(irv_2dlon);
3914 }
3915 else {
3916 string lon2d_group_path = HDF5CFUtil::obtain_string_before_lastslash((*irv_2dlon)->fullpath);
3917
3918 // Check how many lat2d shares the same group with the lon2d
3919 short lat2d_has_lon2d_group_path_flag = 0;
3920 for(auto ivs = lat2d_group_paths.begin();ivs!=lat2d_group_paths.end();++ivs) {
3921 if((*ivs)==lon2d_group_path)
3922 lat2d_has_lon2d_group_path_flag++;
3923 }
3924
3925 // No lat2d shares the same group with the lon2d, remove this lon2d
3926 if(0 == lat2d_has_lon2d_group_path_flag) {
3927 latlon2d_path_to_index.erase((*irv_2dlon)->fullpath);
3928 delete(*irv_2dlon);
3929 irv_2dlon = var_2dlon.erase(irv_2dlon);
3930 }
3931 // Only one lat2d shares the same group with the lon2d, yes, keep it.
3932 else if (1== lat2d_has_lon2d_group_path_flag) {
3933 ++irv_2dlon;
3934 }
3935 // more than 1 lat2d, we will remove the lon2d, but save the group path so that we can
3936 // change the coordinates attribute for variables under this group later.
3937 else {
3938 // Save the group path for future "coordinates" modification.
3939 grp_cv_paths.insert(lon2d_group_path);
3940 latlon2d_path_to_index.erase((*irv_2dlon)->fullpath);
3941 delete(*irv_2dlon);
3942 irv_2dlon = var_2dlon.erase(irv_2dlon);
3943 }
3944 }
3945 //Clear the vector that stores the same dim. since it is only applied to this lon,
3946 lat2d_group_paths.clear();
3947 }
3948#if 0
3949for(vector<Var*>::iterator itv = var_2dlat.begin(); itv!= var_2dlat.end();++itv) {
3950cerr<<"Before unique, 2-D CV latitude name is "<<(*itv)->fullpath <<endl;
3951}
3952for(vector<Var*>::iterator itv = var_2dlon.begin(); itv!= var_2dlon.end();++itv) {
3953cerr<<"Before unique, 2-D CV longitude name is "<<(*itv)->fullpath <<endl;
3954}
3955#endif
3956
3957 // Final check var_2dlat and var_2dlon to remove non-qualified CVs.
3958 Obtain_unique_2dCV(var_2dlat,latlon2d_path_to_index);
3959 Obtain_unique_2dCV(var_2dlon,latlon2d_path_to_index);
3960#if 0
3961for(vector<Var*>::iterator itv = var_2dlat.begin(); itv!= var_2dlat.end();++itv) {
3962cerr<<"2-D CV latitude name is "<<(*itv)->fullpath <<endl;
3963}
3964for(vector<Var*>::iterator itv = var_2dlon.begin(); itv!= var_2dlon.end();++itv) {
3965cerr<<"2-D CV longitude name is "<<(*itv)->fullpath <<endl;
3966}
3967#endif
3968
3969 // This is to serve as a sanity check. This can help us find bugs in the first place.
3970 if(var_2dlat.size() != var_2dlon.size()) {
3971 throw1("Error in generating 2-D lat/lon CVs. The size of 2d-lat should be the same as that of 2d-lon.");
3972 }
3973}
3974
3975// If two vars in the 2-D lat or 2-D lon CV candidate vector share the same dim. , these two vars cannot be CVs.
3976// The group they belong to is the group candidate that the coordinates attribute of the variable under that group may be modified..
3977void GMFile::Obtain_unique_2dCV(vector<Var*> &var_ll,map<string,int>&latlon2d_path_to_index){
3978
3979 BESDEBUG("h5", "Coming to Obtain_unique_2dCV()"<<endl);
3980 vector<bool> var_share_dims(var_ll.size(),false);
3981
3982 for(unsigned int i = 0; i <var_ll.size();i++) {
3983
3984 // obtain the path of var_ll
3985 string var_ll_i_path = HDF5CFUtil::obtain_string_before_lastslash(var_ll[i]->fullpath);
3986
3987 // Check if two vars share the same dims.
3988 for(unsigned int j = i+1; j<var_ll.size();j++) {
3989 if((var_ll[i]->getDimensions()[0]->name == var_ll[j]->getDimensions()[0]->name)
3990 ||(var_ll[i]->getDimensions()[0]->name == var_ll[j]->getDimensions()[1]->name)
3991 ||(var_ll[i]->getDimensions()[1]->name == var_ll[j]->getDimensions()[0]->name)
3992 ||(var_ll[i]->getDimensions()[1]->name == var_ll[j]->getDimensions()[1]->name)){
3993 string var_ll_j_path = HDF5CFUtil::obtain_string_before_lastslash(var_ll[j]->fullpath);
3994
3995 // Compare var_ll_i_path and var_ll_j_path,only set the child group path be true and remember the path.
3996 // The variable at the parent group can be the coordinate variable.
3997 // Obtain the string size,
3998 // compare the string size, long.compare(0,shortlength,short)==0,
3999 // yes, save the long path(child group path), set the long path one true. Else save two paths, set both true
4000 if(var_ll_i_path.size() > var_ll_j_path.size()) {
4001
4002 // If var_ll_j_path is the parent group of var_ll_i_path,
4003 // set the shared dim. be true for the child group only,remember the path.
4004 if(var_ll_i_path.compare(0,var_ll_j_path.size(),var_ll_j_path)==0) {
4005 var_share_dims[i] = true;
4006 grp_cv_paths.insert(var_ll_i_path);
4007 }
4008 else {// Save both as shared, they cannot be CVs.
4009 var_share_dims[i] = true;
4010 var_share_dims[j] = true;
4011
4012 grp_cv_paths.insert(var_ll_i_path);
4013 grp_cv_paths.insert(var_ll_j_path);
4014 }
4015 }
4016 else if (var_ll_i_path.size() == var_ll_j_path.size()) {// Share the same group, remember both group paths.
4017 var_share_dims[i] = true;
4018 var_share_dims[j] = true;
4019 if(var_ll_i_path == var_ll_j_path)
4020 grp_cv_paths.insert(var_ll_i_path);
4021 else {
4022 grp_cv_paths.insert(var_ll_i_path);
4023 grp_cv_paths.insert(var_ll_j_path);
4024 }
4025 }
4026 else {
4027 // var_ll_i_path is the parent group of var_ll_j_path,
4028 // set the shared dim. be true for the child group,remember the path.
4029 if(var_ll_j_path.compare(0,var_ll_i_path.size(),var_ll_i_path)==0) {
4030 var_share_dims[j] = true;
4031 grp_cv_paths.insert(var_ll_j_path);
4032 }
4033 else {// Save both as shared, they cannot be CVs.
4034 var_share_dims[i] = true;
4035 var_share_dims[j] = true;
4036
4037 grp_cv_paths.insert(var_ll_i_path);
4038 grp_cv_paths.insert(var_ll_j_path);
4039
4040 }
4041 }
4042 }// end of if((var_ll[i]->getDimensions()[0]->name == var_ll[j]->getDimensions()[0]->name)
4043 }// end of for(int j = i+1; j<var_ll.size();j++)
4044 }// end of for( int i = 0; i <var_ll.size();i++)
4045
4046 // Remove the shared 2-D lat/lon CVs from the 2-D lat/lon CV candidates.
4047 int var_index = 0;
4048 for(auto itv = var_ll.begin(); itv!= var_ll.end();) {
4049 if(true == var_share_dims[var_index]) {
4050 latlon2d_path_to_index.erase((*itv)->fullpath);
4051 delete(*itv);
4052 itv = var_ll.erase(itv);
4053 }
4054 else {
4055 ++itv;
4056 }
4057 ++var_index;
4058 }
4059
4060}
4061
4062// When promoting a 2-D lat or lon to a coordinate variable, we need to remove them from the general variable vector.
4063void GMFile::Remove_2DLLCVar_Final_Candidate_from_Vars(vector<int> &var2d_index) {
4064
4065 BESDEBUG("h5", "Coming to Remove_2DLLCVar_Final_Candidate_from_Vars()"<<endl);
4066 //Sort the 2-D lat/lon var index according to the ascending order before removing the 2-D lat/lon vars
4067 sort(var2d_index.begin(),var2d_index.end());
4068 auto it = this->vars.begin();
4069
4070 // This is a performance optimiziation operation.
4071 // We find it is typical for swath files that have many many general variables but only have very few lat/lon CVs.
4072 // To reduce the looping through all variables and comparing the fullpath(string), we use index and remember
4073 // the position of 2-D CVs in the iterator. In this way, only a few operations are needed.
4074 for (unsigned int i = 0; i <var2d_index.size();i++) {
4075 if ( i == 0)
4076 advance(it,var2d_index[i]);
4077 else
4078 advance(it,var2d_index[i]-var2d_index[i-1]-1);
4079
4080 if(it == this->vars.end())
4081 throw1("Out of range to obtain 2D lat/lon variables");
4082 else {
4083 delete(*it);
4084 it = this->vars.erase(it);
4085 }
4086 }
4087}
4088
4089//This function is for generating the coordinates attribute for the 2-D lat/lon.
4090//It will check if this var can keep its "coordinates" attribute rather than rebuilding it.
4091//This function is used by Handle_Coor_Attr().
4092bool GMFile::Check_Var_2D_CVars(Var *var) {
4093
4094 BESDEBUG("h5", "Coming to Check_Var_2D_CVars()"<<endl);
4095 bool ret_value = true;
4096 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
4097 if((*ircv)->rank==2) {
4098 short first_dim_index = 0;
4099 short first_dim_times = 0;
4100 short second_dim_index = 0;
4101 short second_dim_times = 0;
4102 for (auto ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
4103 if((*ird)->name == ((*ircv)->getDimensions()[0])->name) {
4104 first_dim_index = distance(var->dims.begin(),ird);
4105 first_dim_times++;
4106 }
4107 else if((*ird)->name == ((*ircv)->getDimensions()[1])->name) {
4108 second_dim_index = distance(var->dims.begin(),ird);
4109 second_dim_times++;
4110 }
4111 }
4112 // The 2-D CV dimensions must only appear once as the dimension of the variable
4113 // It also must follow the dimension order of the 2-D lat/lon dimensions.
4114 if(first_dim_times == 1 && second_dim_times == 1) {
4115 if(first_dim_index < second_dim_index) {
4116 ret_value = false;
4117 break;
4118 }
4119 }
4120 }
4121 }
4122 return ret_value;
4123
4124}
4125
4126// This function flattens the variable path in the "coordinates" attribute.
4127// It is also used by Handle_Coor_Attr().
4128bool GMFile::Flatten_VarPath_In_Coordinates_Attr(Var *var) {
4129
4130 BESDEBUG("h5", "Coming to Flatten_VarPath_In_Coordinates_Attr()"<<endl);
4131 string co_attrname = "coordinates";
4132 bool need_flatten_coor_attr = false;
4133 string orig_coor_value;
4134 string flatten_coor_value;
4135 // Assume the separator is always a space.
4136 char sc = ' ';
4137 char backslash = '/';
4138
4139 for (auto ira =var->attrs.begin(); ira !=var->attrs.end();) {
4140
4141 // We only check the original attribute name
4142 // Remove the original "coordinates" attribute.
4143 // There is a case that the "coordinates" attribute doesn't need to be flattened.
4144 // We will skip this case. This is necessary since the "coordinates" may be revised
4145 // to add a path to follow the CF. Without skipping the case, the "coordinates" is
4146 // actually not handled but "makes the impression" it is handled. That makes the
4147 // this case ignored when the EnableCoorattrAddPath BES key is supposed to work on.
4148 // https://bugs.earthdata.nasa.gov/browse/HDFFEATURE-45
4149 if((*ira)->name == co_attrname) {
4150 Retrieve_H5_Attr_Value((*ira),var->fullpath);
4151 string orig_attr_value((*ira)->value.begin(),(*ira)->value.end());
4152 if(orig_attr_value.find_first_of(backslash)!=string::npos){
4153 orig_coor_value = orig_attr_value;
4154 need_flatten_coor_attr = true;
4155 delete(*ira);
4156 ira = var->attrs.erase(ira);
4157 }
4158 break;
4159 }
4160 else
4161 ++ira;
4162 }
4163
4164 if(true == need_flatten_coor_attr) {
4165
4166 // We need to loop through each element in the "coordinates".
4167 size_t ele_start_pos = 0;
4168 size_t cur_pos = orig_coor_value.find_first_of(sc);
4169 while(cur_pos !=string::npos) {
4170 string tempstr = orig_coor_value.substr(ele_start_pos,cur_pos-ele_start_pos);
4171 tempstr = get_CF_string(tempstr);
4172 flatten_coor_value += tempstr + sc;
4173 ele_start_pos = cur_pos+1;
4174 cur_pos = orig_coor_value.find_first_of(sc,cur_pos+1);
4175 }
4176 // Only one element
4177 if(ele_start_pos == 0)
4178 flatten_coor_value = get_CF_string(orig_coor_value);
4179 else // Add the last element
4180 flatten_coor_value += get_CF_string(orig_coor_value.substr(ele_start_pos));
4181
4182 // Generate the new "coordinates" attribute.
4183 auto attr = new Attribute();
4184 Add_Str_Attr(attr,co_attrname,flatten_coor_value);
4185 var->attrs.push_back(attr);
4186 var->coord_attr_add_path = false;
4187
4188 }
4189
4190 return true;
4191
4192}
4193// This function flattens the variable path in the "coordinates" attribute for hybrid EOS5.
4194// Will implement it later.
4195// It is also used by Handle_Coor_Attr().
4196#if 0
4197bool GMFile::Flatten_VarPath_In_Coordinates_Attr_EOS5(Var *var) {
4198
4199 BESDEBUG("h5", "Coming to Flatten_VarPath_In_Coordinates_Attr_EOS5()"<<endl);
4200 string co_attrname = "coordinates";
4201 bool has_coor_attr = false;
4202 string orig_coor_value;
4203 string flatten_coor_value;
4204 // Assume the separator is always a space.
4205 char sc = ' ';
4206
4207 for (auto ira =var->attrs.begin(); ira !=var->attrs.end();) {
4208
4209 // We only check the original attribute name
4210 // Remove the original "coordinates" attribute.
4211 if((*ira)->name == co_attrname) {
4212 Retrieve_H5_Attr_Value((*ira),var->fullpath);
4213
4214 string orig_attr_value((*ira)->value.begin(),(*ira)->value.end());
4215 orig_coor_value = orig_attr_value;
4216 has_coor_attr = true;
4217 delete(*ira);
4218 ira = var->attrs.erase(ira);
4219 break;
4220 }
4221 else
4222 ++ira;
4223 }
4224
4225 if(true == has_coor_attr) {
4226
4227 // We need to loop through each element in the "coordinates".
4228 // For EOS5: we need to find the swath name.
4229 size_t ele_start_pos = 0;
4230// cerr<<"orig_coor_value is "<<orig_coor_value <<endl;
4231 size_t cur_pos = orig_coor_value.find_first_of(sc);
4232 while(cur_pos !=string::npos) {
4233 string tempstr = orig_coor_value.substr(ele_start_pos,cur_pos-ele_start_pos);
4234 // Find the swath name
4235 // tempstr = "swath name" +"_"+tempstr;
4236 tempstr = get_CF_string(tempstr);
4237 flatten_coor_value += tempstr + sc;
4238 ele_start_pos = cur_pos+1;
4239 cur_pos = orig_coor_value.find_first_of(sc,cur_pos+1);
4240 }
4241 // Only one element
4242 if(ele_start_pos == 0) {
4243 // Find the swath name
4244 // tempstr = "swath name" +"_"+tempstr;
4245 flatten_coor_value = get_CF_string(tempstr);
4246 }
4247 else // Add the last element
4248 flatten_coor_value += get_CF_string(orig_coor_value.substr(ele_start_pos));
4249
4250 // Generate the new "coordinates" attribute.
4251 Attribute *attr = new Attribute();
4252 Add_Str_Attr(attr,co_attrname,flatten_coor_value);
4253 var->attrs.push_back(attr);
4254 }
4255
4256 return true;
4257
4258}
4259#endif
4260
4261
4262// The following two routines only handle one 2-D lat/lon CVs. It is replaced by the more general
4263// multi 2-D lat/lon CV routines. Leave it here just for references.
4264#if 0
4265bool GMFile::Check_2DLatLon_Dimscale(string & latname, string &lonname) {
4266
4267 // New code to support 2-D lat/lon, still in development.
4268 // Need to handle 2-D latitude and longitude cases.
4269 // 1. Searching only the coordinate variables and if getting either of the following, keep the current way,
4270 // (A) GMcvar no CV_FILLINDEX:(The 2-D latlon case should have fake CVs)
4271 // (B) CV_EXIST: Attributes contain units and units value is degrees_east or degrees_north(have lat/lon)
4272 // (B) CV_EXIST: variables have name pair{lat/latitude/Latitude,lon/longitude/Longitude}(have lat/lon)
4273 //
4274 // 2. if not 1), searching all the variables and see if finding variables {lat/latitude/Latitude,lon/longitude/Longitude};
4275 // If finding {lat/lon},{latitude,longitude},{latitude,Longitude} pair,
4276 // if the number of dimension of either variable is not 2, keep the current way.
4277 // else check if the dimension name of latitude and longitude are the same, not, keep the current way
4278 // check the units of this CV pair, if units of the latitude is not degrees_north,
4279 // change it to degrees_north.
4280 // if units of the longitude is not degrees_east, change it to degrees_east.
4281 // make iscoard false.
4282
4283 bool latlon_2d_cv_check1 = false;
4284
4285 // Some products(TOM MEaSURE) provide the true dimension scales for 2-D lat,lon. So relax this check.
4286 latlon_2d_cv_check1 = true;
4287#if 0
4288 // If having 2-D lat/lon, the corresponding dimension must be pure and the CV type must be FILLINDEX.
4289 for (auto ircv = this->cvars.begin();
4290 ircv != this->cvars.end(); ++ircv) {
4291 if((*ircv)->cvartype == CV_FILLINDEX){
4292 latlon_2d_cv_check1 = true;
4293 break;
4294 }
4295 }
4296#endif
4297
4298 bool latlon_2d_cv_check2 = true;
4299
4300 // There may still not be 2-D lat/lon. Check the units attributes and lat/lon pairs.
4301 if(true == latlon_2d_cv_check1) {
4302 BESDEBUG("h5","Coming to check if having 2d latlon coordinates for a netCDF-4 like product. "<<endl);
4303
4304 // check if units attribute values have CF lat/lon units "degrees_north" or "degrees_east".
4305 for (auto ircv = this->cvars.begin();
4306 ircv != this->cvars.end(); ++ircv) {
4307 if((*ircv)->cvartype == CV_EXIST) {
4308 for(auto ira = (*ircv)->attrs.begin();
4309 ira != (*ircv)->attrs.end();ira++) {
4310 string attr_name ="units";
4311 string lat_unit_value = "degrees_north";
4312 string lon_unit_value = "degrees_east";
4313
4314 // Considering the cross-section case, either is fine.
4315 if((true == Is_Str_Attr(*ira,(*ircv)->fullpath,attr_name,lat_unit_value)) ||
4316 (true == Is_Str_Attr(*ira,(*ircv)->fullpath,attr_name,lon_unit_value))) {
4317 latlon_2d_cv_check2= false;
4318 break;
4319 }
4320 }
4321 }
4322
4323 if(false == latlon_2d_cv_check2)
4324 break;
4325 }
4326 }
4327
4328 bool latlon_2d_cv_check3 = true;
4329
4330 // Even we cannot find the CF lat/lon attributes, we may still find lat/lon etc pairs.
4331 if(true == latlon_2d_cv_check1 && true == latlon_2d_cv_check2) {
4332
4333 short latlon_flag = 0;
4334 short LatLon_flag = 0;
4335 short latilong_flag = 0;
4336
4337 for (auto ircv = this->cvars.begin();
4338 ircv != this->cvars.end(); ++ircv) {
4339 if((*ircv)->cvartype == CV_EXIST) {
4340 if((*ircv)->name == "lat")
4341 latlon_flag++;
4342 else if((*ircv)->name == "lon")
4343 latlon_flag++;
4344 else if((*ircv)->name == "latitude")
4345 latilong_flag++;
4346 else if((*ircv)->name == "longitude")
4347 latilong_flag++;
4348 else if((*ircv)->name == "Latitude")
4349 LatLon_flag++;
4350 else if((*ircv)->name == "Longitude")
4351 LatLon_flag++;
4352 }
4353
4354 }
4355 if((2== latlon_flag) || (2 == latilong_flag) || (2 == LatLon_flag ))
4356 latlon_2d_cv_check3 = false;
4357 }
4358
4359 bool latlon_2d = false;
4360 short latlon_flag = 0;
4361 string latdim1,latdim2,londim1,londim2;
4362
4363 short LatLon_flag = 0;
4364 string Latdim1,Latdim2,Londim1,Londim2;
4365
4366 short latilong_flag = 0;
4367 string latidim1,latidim2,longdim1,longdim2;
4368
4369
4370 // Final check, we need to check if we have 2-D {lat/latitude/Latitude, lon/longitude/Longitude}
4371 // in the general variable list.
4372 // Here, depending on the future support, lat/lon pairs with other names(cell_lat,cell_lon etc) may be supported.
4373 // KY 2015-12-03
4374 if(true == latlon_2d_cv_check1 && true == latlon_2d_cv_check2 && true == latlon_2d_cv_check3) {
4375
4376 for (auto irv = this->vars.begin();
4377 irv != this->vars.end(); ++irv) {
4378
4379 //
4380 if((*irv)->rank == 2) {
4381 if((*irv)->name == "lat") {
4382 latlon_flag++;
4383 latdim1 = (*irv)->getDimensions()[0]->name;
4384 latdim2 = (*irv)->getDimensions()[1]->name;
4385
4386 }
4387 else if((*irv)->name == "lon") {
4388 latlon_flag++;
4389 londim1 = (*irv)->getDimensions()[0]->name;
4390 londim2 = (*irv)->getDimensions()[1]->name;
4391 }
4392 else if((*irv)->name == "latitude"){
4393 latilong_flag++;
4394 latidim1 = (*irv)->getDimensions()[0]->name;
4395 latidim2 = (*irv)->getDimensions()[1]->name;
4396 }
4397 else if((*irv)->name == "longitude"){
4398 latilong_flag++;
4399 longdim1 = (*irv)->getDimensions()[0]->name;
4400 longdim2 = (*irv)->getDimensions()[1]->name;
4401
4402 }
4403 else if((*irv)->name == "Latitude"){
4404 LatLon_flag++;
4405 Latdim1 = (*irv)->getDimensions()[0]->name;
4406 Latdim2 = (*irv)->getDimensions()[1]->name;
4407
4408 }
4409 else if((*irv)->name == "Longitude"){
4410 LatLon_flag++;
4411 Londim1 = (*irv)->getDimensions()[0]->name;
4412 Londim2 = (*irv)->getDimensions()[1]->name;
4413 }
4414
4415 }
4416 }
4417
4418 // Here we ensure that only one lat/lon(lati/long,Lati/Long) is in the file.
4419 // If we find >=2 pairs lat/lon and latitude/longitude, or Latitude/Longitude,
4420 // we will not treat this as a 2-D latlon Dimscale case. The data producer
4421 // should correct their mistakes.
4422 if(2 == latlon_flag) {
4423 if((2 == latilong_flag) || ( 2 == LatLon_flag))
4424 latlon_2d = false;
4425 else if((latdim1 == londim1) && (latdim2 == londim2)) {
4426 latname = "lat";
4427 lonname = "lon";
4428 latlon_2d = true;
4429 }
4430 }
4431 else if ( 2 == latilong_flag) {
4432 if( 2 == LatLon_flag)
4433 latlon_2d = false;
4434 else if ((latidim1 == longdim1) ||(latidim2 == longdim2)) {
4435 latname = "latitude";
4436 lonname = "longitude";
4437 latlon_2d = true;
4438 }
4439 }
4440 else if (2 == LatLon_flag){
4441 if ((Latdim1 == Londim1) ||(Latdim2 == Londim2)) {
4442 latname = "Latitude";
4443 lonname = "Longitude";
4444 latlon_2d = true;
4445 }
4446 }
4447 }
4448
4449 return latlon_2d;
4450}
4451
4452
4453// Update the coordinate variables for files that use HDF5 dimension scales and have 2-D lat/lon.
4454void GMFile::Update_2DLatLon_Dimscale_CV(const string &latname,const string &lonname) {
4455
4456 iscoard = false;
4457
4458 // Update latitude.
4459 for (auto irv = this->vars.begin();
4460 irv != this->vars.end(); ++irv) {
4461
4462 if((*irv)->rank == 2) {
4463
4464 // Find 2-D latitude
4465 if((*irv)->name == latname) {
4466
4467 // Obtain the first dimension of this variable
4468 string latdim0 = (*irv)->getDimensions()[0]->name;
4469//cerr<<"latdim0 is "<<latdim0 <<endl;
4470
4471 // Remove the CV corresponding to latdim0
4472 for (vector<GMCVar *>:: iterator i= this->cvars.begin(); i!=this->cvars.end(); ) {
4473 if((*i)->cfdimname == latdim0) {
4474 if(CV_FILLINDEX == (*i)->cvartype) {
4475 delete(*i);
4476 i = this->cvars.erase(i);
4477 }
4478 else if(CV_EXIST == (*i)->cvartype) {
4479 // Add this var. to the var list.
4480 Var *var = new Var(*i);
4481 this->vars.push_back(var);
4482 // Remove this var. from the GMCVar list.
4483 delete(*i);
4484 i = this->cvars.erase(i);
4485
4486 }
4487 else {// the latdimname should be either the CV_FILLINDEX or CV_EXIST.
4488 if(CV_LAT_MISS == (*i)->cvartype)
4489 throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_LAT_MISS");
4490 else if(CV_LON_MISS == (*i)->cvartype)
4491 throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_LON_MISS");
4492 else if(CV_NONLATLON_MISS == (*i)->cvartype)
4493 throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_NONLATLON_MISS");
4494 else if(CV_MODIFY == (*i)->cvartype)
4495 throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_MODIFY");
4496 else if(CV_SPECIAL == (*i)->cvartype)
4497 throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_SPECIAL");
4498 else
4499 throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_UNSUPPORTED");
4500
4501 }
4502 }
4503 else
4504 ++i;
4505 }
4506 // Add the 2-D latitude(latname) to the CV list.
4507 GMCVar* GMcvar = new GMCVar(*irv);
4508 GMcvar->cfdimname = latdim0;
4509 GMcvar->cvartype = CV_EXIST;
4510 GMcvar->product_type = product_type;
4511 this->cvars.push_back(GMcvar);
4512 delete(*irv);
4513 this->vars.erase(irv);
4514 break;
4515 }
4516 }
4517 }
4518
4519 // Update longitude.
4520 for (auto irv = this->vars.begin();
4521 irv != this->vars.end(); ++irv) {
4522
4523 if((*irv)->rank == 2) {
4524
4525 // Find 2-D longitude
4526 if((*irv)->name == lonname) {
4527
4528 // Obtain the second dimension of this variable
4529 string londim0 = (*irv)->getDimensions()[1]->name;
4530
4531 // Remove the CV corresponding to londim0
4532 for (vector<GMCVar *>:: iterator i= this->cvars.begin(); i!=this->cvars.end(); ) {
4533 // NEED more work!!! should also remove ntime from the GMCVar list but add it to the cvar list.Same for Lon.
4534 if((*i)->cfdimname == londim0) {
4535 if(CV_FILLINDEX == (*i)->cvartype) {
4536 delete(*i);
4537 i= this->cvars.erase(i);
4538 }
4539 else if(CV_EXIST == (*i)->cvartype) {
4540 // Add this var. to the var list.
4541 Var *var = new Var(*i);
4542 this->vars.push_back(var);
4543 // Remove this var. from the GMCVar list.
4544 delete(*i);
4545 i = this->cvars.erase(i);
4546 }
4547 else {// the latdimname should be either the CV_FILLINDEX or CV_EXIST.
4548 if(CV_LAT_MISS == (*i)->cvartype)
4549 throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_LAT_MISS");
4550 else if(CV_LON_MISS == (*i)->cvartype)
4551 throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_LON_MISS");
4552 else if(CV_NONLATLON_MISS == (*i)->cvartype)
4553 throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_NONLATLON_MISS");
4554 else if(CV_MODIFY == (*i)->cvartype)
4555 throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_MODIFY");
4556 else if(CV_SPECIAL == (*i)->cvartype)
4557 throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_SPECIAL");
4558 else
4559 throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_UNSUPPORTED");
4560 }
4561 }
4562 else
4563 ++i;
4564 }
4565
4566 // Add the 2-D longitude(lonname) to the CV list.
4567 GMCVar* GMcvar = new GMCVar(*irv);
4568 GMcvar->cfdimname = londim0;
4569 GMcvar->cvartype = CV_EXIST;
4570 GMcvar->product_type = product_type;
4571 this->cvars.push_back(GMcvar);
4572 delete(*irv);
4573 this->vars.erase(irv);
4574 break;
4575 }
4576 }
4577 }
4578}
4579#endif
4580
4581// Handle coordinate variables for general HDF5 products that have 1-D lat/lon
4582void GMFile::Handle_CVar_LatLon1D_General_Product() {
4583
4584 BESDEBUG("h5", "Coming to Handle_CVar_LatLon1D_General_Product()"<<endl);
4585 this->iscoard = true;
4586 Handle_CVar_LatLon_General_Product();
4587
4588}
4589
4590// Handle coordinate variables for general HDF5 products that have 2-D lat/lon
4591void GMFile::Handle_CVar_LatLon2D_General_Product() {
4592
4593 BESDEBUG("h5", "Coming to Handle_CVar_LatLon2D_General_Product()"<<endl);
4594 Handle_CVar_LatLon_General_Product();
4595
4596}
4597
4598// Routine to handle coordinate variables for general HDF5 product
4599// that have either 1-D or 2-D lat/lon
4600void GMFile::Handle_CVar_LatLon_General_Product() {
4601
4602 BESDEBUG("h5", "Coming to Handle_CVar_LatLon_General_Product()"<<endl);
4603 if((GENERAL_LATLON2D != this->gproduct_pattern)
4604 && GENERAL_LATLON1D != this->gproduct_pattern)
4605 throw1("This function only supports latlon 1D or latlon 2D general products");
4606
4607 pair<set<string>::iterator,bool> setret;
4608 set<string>tempdimnamelist = dimnamelist;
4609
4610 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4611
4612 // This is the dimension scale dataset; it should be changed to a coordinate variable.
4613 if (gp_latname== (*irv)->name) {
4614
4615 // For latitude, regardless 1D or 2D, the first dimension needs to be updated.
4616 // Create Coordinate variables.
4617 tempdimnamelist.erase(((*irv)->dims[0])->name);
4618 auto GMcvar = new GMCVar(*irv);
4619 GMcvar->cfdimname = ((*irv)->dims[0])->name;
4620 GMcvar->cvartype = CV_EXIST;
4621 GMcvar->product_type = product_type;
4622 this->cvars.push_back(GMcvar);
4623 delete(*irv);
4624 this->vars.erase(irv);
4625 break;
4626 } // end of if
4627 } // end of for
4628
4629 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4630
4631 // This is the dimension scale dataset; it should be changed to a coordinate variable.
4632 if (gp_lonname== (*irv)->name) {
4633
4634 // For 2-D lat/lon, the londimname should be the second dimension of the longitude
4635 // For 1-D lat/lon, the londimname should be the first dimension of the longitude
4636 // Create Coordinate variables.
4637 string londimname;
4638 if(GENERAL_LATLON2D == this->gproduct_pattern)
4639 londimname = ((*irv)->dims[1])->name;
4640 else
4641 londimname = ((*irv)->dims[0])->name;
4642
4643 tempdimnamelist.erase(londimname);
4644 auto GMcvar = new GMCVar(*irv);
4645 GMcvar->cfdimname = londimname;
4646 GMcvar->cvartype = CV_EXIST;
4647 GMcvar->product_type = product_type;
4648 this->cvars.push_back(GMcvar);
4649 delete(*irv);
4650 this->vars.erase(irv);
4651 break;
4652 } // end of if
4653 } // end of for
4654
4655 //
4656 // Add other missing coordinate variables.
4657 for (auto irs = tempdimnamelist.begin(); irs != tempdimnamelist.end();irs++) {
4658 auto GMcvar = new GMCVar();
4659 Create_Missing_CV(GMcvar,*irs);
4660 this->cvars.push_back(GMcvar);
4661 }
4662
4663}
4664
4665// Handle coordinate variables for OBPG level 3
4666void GMFile::Handle_CVar_OBPG_L3() {
4667
4668 BESDEBUG("h5", "Coming to Handle_CVar_OBPG_L3()"<<endl);
4669 if (GENERAL_DIMSCALE == this->gproduct_pattern)
4670 Handle_CVar_Dimscale_General_Product();
4671
4672 // Change the CV Type of the corresponding CVs of lat and lon from CV_FILLINDEX to CV_LATMISS or CV_LONMISS
4673 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4674
4675 // Here I try to avoid using the dimension name row and column to find the lat/lon dimension size.
4676 // So I am looking for a 2-D floating-point array or a 2-D array under the group geophsical_data.
4677 // This may be subject to change if OBPG level 3 change its arrangement of variables.
4678 // KY 2014-09-29
4679
4680 if((*irv)->rank == 2) {
4681
4682 if(((*irv)->fullpath.find("/geophsical_data") == 0) || ((*irv)->dtype == H5FLOAT32)) {
4683
4684 size_t lat_size = (*irv)->getDimensions()[0]->size;
4685 string lat_name = (*irv)->getDimensions()[0]->name;
4686 size_t lon_size = (*irv)->getDimensions()[1]->size;
4687 string lon_name = (*irv)->getDimensions()[1]->name;
4688 size_t temp_size = 0;
4689 string temp_name;
4690 H5DataType ll_dtype = (*irv)->dtype;
4691
4692 // We always assume that longitude size is greater than latitude size.
4693 if(lat_size >lon_size) {
4694 temp_size = lon_size;
4695 temp_name = lon_name;
4696 lon_size = lat_size;
4697 lon_name = lat_name;
4698 lat_size = temp_size;
4699 lat_name = temp_name;
4700 }
4701 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
4702 if((*ircv)->cvartype == CV_FILLINDEX) {
4703 if((*ircv)->getDimensions()[0]->size == lat_size &&
4704 (*ircv)->getDimensions()[0]->name == lat_name) {
4705 (*ircv)->cvartype = CV_LAT_MISS;
4706 (*ircv)->dtype = ll_dtype;
4707 for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ++ira) {
4708 if ((*ira)->name == "NAME") {
4709 delete (*ira);
4710 (*ircv)->attrs.erase(ira);
4711 break;
4712 }
4713 }
4714 }
4715 else if((*ircv)->getDimensions()[0]->size == lon_size &&
4716 (*ircv)->getDimensions()[0]->name == lon_name) {
4717 (*ircv)->cvartype = CV_LON_MISS;
4718 (*ircv)->dtype = ll_dtype;
4719 for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ++ira) {
4720 if ((*ira)->name == "NAME") {
4721 delete (*ira);
4722 (*ircv)->attrs.erase(ira);
4723 break;
4724 }
4725 }
4726 }
4727 }
4728 }
4729 break;
4730
4731 } // end of if(((*irv)->fullpath.find("/geophsical_data") == 0) || ((*irv)->dtype == H5FLOAT32))
4732 } // end of if((*irv)->rank == 2)
4733 } // end of for (auto irv = this->vars.begin();
4734
4735}
4736
4737// Handle some special variables. Currently only GPM and ACOS have these variables.
4739
4740 BESDEBUG("h5", "Coming to Handle_SpVar()"<<endl);
4741 if (ACOS_L2S_OR_OCO2_L1B == product_type)
4742 Handle_SpVar_ACOS_OCO2();
4743 else if(GPM_L1 == product_type) {
4744 // Loop through the variable list to build the coordinates.
4745 // These variables need to be removed.
4746 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4747 if((*irv)->name=="AlgorithmRuntimeInfo") {
4748 delete(*irv);
4749 this->vars.erase(irv);
4750 break;
4751 }
4752 }
4753 }
4754
4755 // GPM level-3 These variables need to be removed.
4756 else if(GPMM_L3 == product_type || GPMS_L3 == product_type || GPM_L3_New==product_type) {
4757
4758 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
4759 if((*irv)->name=="InputFileNames") {
4760 delete(*irv);
4761 irv = this->vars.erase(irv);
4762 }
4763 else if((*irv)->name=="InputAlgorithmVersions") {
4764 delete(*irv);
4765 irv = this->vars.erase(irv);
4766 }
4767 else if((*irv)->name=="InputGenerationDateTimes") {
4768 delete(*irv);
4769 irv = this->vars.erase(irv);
4770 }
4771 else {
4772 ++irv;
4773 }
4774 }
4775 }
4776
4777}
4778
4779// Handle special variables for ACOS.
4780void GMFile::Handle_SpVar_ACOS_OCO2() {
4781
4782 BESDEBUG("h5", "Coming to Handle_SpVar_ACOS_OCO2()"<<endl);
4783 //The ACOS or OCO2 have 64-bit variables. DAP2 doesn't support 64-bit variables.
4784 // So we will not handle attributes yet.
4785 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
4786
4787 if (H5INT64 == (*irv)->getType()) {
4788
4789 // First: Time Part of soundingid
4790 auto spvar = new GMSPVar(*irv);
4791 spvar->name = (*irv)->name +"_Time";
4792 spvar->newname = (*irv)->newname+"_Time";
4793 spvar->dtype = H5INT32;
4794 spvar->otype = (*irv)->getType();
4795 spvar->sdbit = 1;
4796
4797 // 2 digit hour, 2 digit min, 2 digit seconds
4798 spvar->numofdbits = 6;
4799 this->spvars.push_back(spvar);
4800
4801 // Second: Date Part of soundingid
4802 auto spvar2 = new GMSPVar(*irv);
4803 spvar2->name = (*irv)->name +"_Date";
4804 spvar2->newname = (*irv)->newname+"_Date";
4805 spvar2->dtype = H5INT32;
4806 spvar2->otype = (*irv)->getType();
4807 spvar2->sdbit = 7;
4808
4809 // 4 digit year, 2 digit month, 2 digit day
4810 spvar2->numofdbits = 8;
4811 this->spvars.push_back(spvar2);
4812
4813 delete(*irv);
4814 irv = this->vars.erase(irv);
4815 } // end of if (H5INT64 == (*irv)->getType())
4816 else {
4817 ++irv;
4818 }
4819 } // end of for (auto irv = this->vars.begin(); ...
4820}
4821
4822// Adjust Object names, For some products, NASA data centers don't need
4823// the fullpath of objects.
4825
4826 BESDEBUG("h5", "Coming to Adjust_Obj_Name()"<<endl);
4827 if(Mea_Ozone == product_type)
4828 Adjust_Mea_Ozone_Obj_Name();
4829
4830 if(GPMS_L3 == product_type || GPMM_L3 == product_type)
4831 Adjust_GPM_L3_Obj_Name();
4832
4833// Just for debugging
4834#if 0
4835for (auto irv2 = this->vars.begin();
4836 irv2 != this->vars.end(); irv2++) {
4837 for (auto ird = (*irv2)->dims.begin();
4838 ird !=(*irv2)->dims.end(); ird++) {
4839 cerr<<"Dimension name afet Adjust_Obj_Name "<<(*ird)->newname <<endl;
4840 }
4841}
4842#endif
4843
4844}
4845
4846// Adjust object names for GPM level 3 products
4847void GMFile:: Adjust_GPM_L3_Obj_Name() {
4848
4849 BESDEBUG("h5", "Coming to Adjust_GPM_L3_Obj_Name()"<<endl);
4850 string objnewname;
4851 // In this definition, root group is not considered as a group.
4852 if(this->groups.size() <= 1) {
4853 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4854 objnewname = HDF5CFUtil::obtain_string_after_lastslash((*irv)->newname);
4855 if (objnewname !="")
4856 (*irv)->newname = objnewname;
4857 }
4858 }
4859 else {
4860 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4861 size_t grid_group_path_pos = ((*irv)->newname.substr(1)).find_first_of("/");
4862 objnewname = ((*irv)->newname).substr(grid_group_path_pos+2);
4863 (*irv)->newname = objnewname;
4864 }
4865 }
4866}
4867
4868// Adjust object names for MeaSUREs OZone
4869void GMFile:: Adjust_Mea_Ozone_Obj_Name() {
4870
4871 BESDEBUG("h5", "Coming to Adjust_Mea_Ozone_Obj_Name()"<<endl);
4872 string objnewname;
4873 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4874 objnewname = HDF5CFUtil::obtain_string_after_lastslash((*irv)->newname);
4875 if (objnewname !="")
4876 (*irv)->newname = objnewname;
4877
4878#if 0
4879//Just for debugging
4880for (auto ird = (*irv)->dims.begin();
4881 ird !=(*irv)->dims.end();++ird) {
4882 cerr<<"Ozone dim. name "<<(*ird)->name <<endl;
4883 cerr<<"Ozone dim. new name "<<(*ird)->newname <<endl;
4884}
4885#endif
4886
4887 }
4888
4889 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
4890 objnewname = HDF5CFUtil::obtain_string_after_lastslash((*irv)->newname);
4891 if (objnewname !="")
4892 (*irv)->newname = objnewname;
4893#if 0
4894 //Just for debugging
4895for (auto ird = (*irv)->dims.begin();
4896 ird !=(*irv)->dims.end();++ird) {
4897 cerr<<"Ozone CV dim. name "<<(*ird)->name <<endl;
4898 cerr<<"Ozone CV dim. new name "<<(*ird)->newname <<endl;
4899}
4900#endif
4901 }
4902}
4903
4904// Flatten object names.
4905void GMFile::Flatten_Obj_Name(bool include_attr) {
4906
4907 BESDEBUG("h5", "GMFile::Coming to Flatten_Obj_Name()"<<endl);
4908 // General variables
4909 File::Flatten_Obj_Name(include_attr);
4910
4911 // Coordinate variables
4912 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
4913 (*irv)->newname = get_CF_string((*irv)->newname);
4914
4915 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird)
4916 (*ird)->newname = get_CF_string((*ird)->newname);
4917
4918 if (true == include_attr) {
4919 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira)
4920 (*ira)->newname = File::get_CF_string((*ira)->newname);
4921 }
4922
4923 }
4924
4925 // Special variables
4926 for (auto irv = this->spvars.begin(); irv != this->spvars.end(); ++irv) {
4927 (*irv)->newname = get_CF_string((*irv)->newname);
4928
4929 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird)
4930 (*ird)->newname = get_CF_string((*ird)->newname);
4931
4932 if (true == include_attr) {
4933 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira)
4934 (*ira)->newname = File::get_CF_string((*ira)->newname);
4935 }
4936 }
4937
4938// Just for debugging
4939#if 0
4940for (auto irv2 = this->vars.begin();
4941 irv2 != this->vars.end(); irv2++) {
4942 for (auto ird = (*irv2)->dims.begin();
4943 ird !=(*irv2)->dims.end(); ird++) {
4944 cerr<<"Dimension name afet Flatten_Obj_Name "<<(*ird)->newname <<endl;
4945 }
4946}
4947#endif
4948
4949
4950}
4951
4952// Rarely object name clashings may occur. This routine makes sure
4953// all object names are unique.
4954void GMFile::Handle_Obj_NameClashing(bool include_attr) {
4955
4956 BESDEBUG("h5", "GMFile::Coming to Handle_Obj_NameClashing()"<<endl);
4957 // objnameset will be filled with all object names that we are going to check the name clashing.
4958 // For example, we want to see if there are any name clashings for all variable names in this file.
4959 // objnameset will include all variable names. If a name clashing occurs, we can figure out from the set operation immediately.
4960 set<string>objnameset;
4961 Handle_GMCVar_NameClashing(objnameset);
4962 Handle_GMSPVar_NameClashing(objnameset);
4963 File::Handle_GeneralObj_NameClashing(include_attr,objnameset);
4964 if (true == include_attr) {
4965 Handle_GMCVar_AttrNameClashing();
4966 Handle_GMSPVar_AttrNameClashing();
4967 }
4968 // Moving to h5gmcfdap.cc, right after Adjust_Dim_Name
4969 //Handle_DimNameClashing();
4970}
4971
4972// Name clashings for coordinate variables
4973void GMFile::Handle_GMCVar_NameClashing(set<string> &objnameset ) {
4974
4975 GMHandle_General_NameClashing(objnameset,this->cvars);
4976}
4977
4978// Name clashings for special variables(like 64-bit integer variables)
4979void GMFile::Handle_GMSPVar_NameClashing(set<string> &objnameset ) {
4980
4981 GMHandle_General_NameClashing(objnameset,this->spvars);
4982}
4983
4984// This routine handles attribute name clashings for coordinate variables.
4985void GMFile::Handle_GMCVar_AttrNameClashing() {
4986
4987 BESDEBUG("h5", "Coming to Handle_GMCVar_AttrNameClashing()"<<endl);
4988 set<string> objnameset;
4989
4990 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
4991 Handle_General_NameClashing(objnameset,(*irv)->attrs);
4992 objnameset.clear();
4993 }
4994}
4995
4996// Attribute name clashings for special variables
4997void GMFile::Handle_GMSPVar_AttrNameClashing() {
4998
4999 BESDEBUG("h5", "Coming to Handle_GMSPVar_AttrNameClashing()"<<endl);
5000 set<string> objnameset;
5001
5002 for (auto irv = this->spvars.begin(); irv != this->spvars.end(); ++irv) {
5003 Handle_General_NameClashing(objnameset,(*irv)->attrs);
5004 objnameset.clear();
5005 }
5006}
5007
5008//class T must have member string newname
5009// The subroutine to handle name clashings,
5010// it builds up a map from original object names to clashing-free object names.
5011template<class T> void
5012GMFile::GMHandle_General_NameClashing(set <string>&objnameset, vector<T*>& objvec) {
5013
5014 BESDEBUG("h5", "Coming to GMHandle_General_NameClashing()"<<endl);
5015 pair<set<string>::iterator,bool> setret;
5016 set<string>::iterator iss;
5017
5018 vector<string> clashnamelist;
5019
5020 map<int,int> cl_to_ol;
5021 int ol_index = 0;
5022 int cl_index = 0;
5023
5024 typename vector<T*>::iterator irv;
5025
5026 for (irv = objvec.begin(); irv != objvec.end(); ++irv) {
5027
5028 setret = objnameset.insert((*irv)->newname);
5029 if (false == setret.second ) {
5030 clashnamelist.insert(clashnamelist.end(),(*irv)->newname);
5031 cl_to_ol[cl_index] = ol_index;
5032 cl_index++;
5033 }
5034 ol_index++;
5035 }
5036
5037
5038 // Now change the clashed elements to unique elements;
5039 // Generate the set which has the same size as the original vector.
5040 for (auto ivs=clashnamelist.begin(); ivs!=clashnamelist.end(); ++ivs) {
5041 int clash_index = 1;
5042 string temp_clashname = *ivs +'_';
5043 HDF5CFUtil::gen_unique_name(temp_clashname,objnameset,clash_index);
5044 *ivs = temp_clashname;
5045 }
5046
5047
5048 // Now go back to the original vector, make it unique.
5049 for (unsigned int i =0; i <clashnamelist.size(); i++)
5050 objvec[cl_to_ol[i]]->newname = clashnamelist[i];
5051
5052}
5053
5054// Handle dimension name clashings
5056
5057
5058 BESDEBUG("h5", "GMFile: Coming to Handle_DimNameClashing()"<<endl);
5059 // ACOS L2S or OCO2 L1B products doesn't need the dimension name clashing check based on our current understanding. KY 2012-5-16
5060 if (ACOS_L2S_OR_OCO2_L1B == product_type)
5061 return;
5062
5063 map<string,string>dimname_to_dimnewname;
5064 pair<map<string,string>::iterator,bool>mapret;
5065 set<string> dimnameset;
5066 vector<Dimension*>vdims;
5067 set<string> dimnewnameset;
5068 pair<set<string>::iterator,bool> setret;
5069
5070 // First: Generate the dimset/dimvar based on coordinate variables.
5071 for (auto irv = this->cvars.begin(); irv !=this->cvars.end(); ++irv) {
5072 for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
5073 setret = dimnameset.insert((*ird)->name);
5074 if (true == setret.second)
5075 vdims.push_back(*ird);
5076 }
5077 }
5078
5079 // For some cases, dimension names are provided but there are no corresponding coordinate
5080 // variables. For now, we will assume no such cases.
5081 // Actually, we find such a case in our fake testsuite. So we need to fix it.
5082 for(auto irv= this->vars.begin(); irv != this->vars.end();++irv) {
5083 for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
5084#if 0
5085 //setret = dimnameset.insert((*ird)->newname);
5086#endif
5087 setret = dimnameset.insert((*ird)->name);
5088 if (setret.second) vdims.push_back(*ird);
5089 }
5090 }
5091
5092 GMHandle_General_NameClashing(dimnewnameset,vdims);
5093
5094 // Third: Make dimname_to_dimnewname map
5095 for (auto ird = vdims.begin();ird!=vdims.end();++ird) {
5096 mapret = dimname_to_dimnewname.insert(pair<string,string>((*ird)->name,(*ird)->newname));
5097 if (false == mapret.second)
5098 throw4("The dimension name ",(*ird)->name," should map to ",
5099 (*ird)->newname);
5100 }
5101
5102 // Fourth: Change the original dimension new names to the unique dimension new names
5103 for (auto irv = this->cvars.begin(); irv !=this->cvars.end(); ++irv)
5104 for (auto ird = (*irv)->dims.begin(); ird!=(*irv)->dims.end();++ird)
5105 (*ird)->newname = dimname_to_dimnewname[(*ird)->name];
5106
5107 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv)
5108 for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird)
5109 (*ird)->newname = dimname_to_dimnewname[(*ird)->name];
5110
5111}
5112
5113// For COARDS, dim. names need to be the same as obj. names.
5115
5116 BESDEBUG("h5", "GMFile:Coming to Adjust_Dim_Name()"<<endl);
5117#if 0
5118 // Just for debugging
5119for (auto irv2 = this->vars.begin();
5120 irv2 != this->vars.end(); irv2++) {
5121 for (auto ird = (*irv2)->dims.begin();
5122 ird !=(*irv2)->dims.end(); ird++) {
5123 cerr<<"Dimension new name "<<(*ird)->newname <<endl;
5124 }
5125}
5126#endif
5127
5128 // Only need for COARD conventions.
5129 if( true == iscoard) {
5130 for (auto irv = this->cvars.begin(); irv !=this->cvars.end(); ++irv) {
5131#if 0
5132cerr<<"1D Cvariable name is "<<(*irv)->name <<endl;
5133cerr<<"1D Cvariable new name is "<<(*irv)->newname <<endl;
5134cerr<<"1D Cvariable dim name is "<<((*irv)->dims)[0]->name <<endl;
5135cerr<<"1D Cvariable dim new name is "<<((*irv)->dims)[0]->newname <<endl;
5136#endif
5137 if ((*irv)->dims.size()!=1)
5138 throw3("Coard coordinate variable ",(*irv)->name, "is not 1D");
5139 if ((*irv)->newname != (((*irv)->dims)[0]->newname)) {
5140 ((*irv)->dims)[0]->newname = (*irv)->newname;
5141
5142 // For all variables that have this dimension,the dimension newname should also change.
5143 for (auto irv2 = this->vars.begin(); irv2 != this->vars.end(); ++irv2) {
5144 for (auto ird = (*irv2)->dims.begin(); ird !=(*irv2)->dims.end(); ++ird) {
5145 // This is the key, the dimension name of this dimension
5146 // should be equal to the dimension name of the coordinate variable.
5147 // Then the dimension name matches and the dimension name should be changed to
5148 // the new dimension name.
5149 if ((*ird)->name == ((*irv)->dims)[0]->name)
5150 (*ird)->newname = ((*irv)->dims)[0]->newname;
5151 }
5152 }
5153 } // end of if ((*irv)->newname != (((*irv)->dims)[0]->newname))
5154 }// end of for (auto irv = this->cvars.begin(); ...
5155 } // end of if( true == iscoard)
5156
5157// Just for debugging
5158#if 0
5159for (auto irv2 = this->vars.begin();
5160 irv2 != this->vars.end(); irv2++) {
5161 for (auto ird = (*irv2)->dims.begin();
5162 ird !=(*irv2)->dims.end(); ird++) {
5163 cerr<<"Dimension name afet Adjust_Dim_Name "<<(*ird)->newname <<endl;
5164 }
5165}
5166#endif
5167
5168
5169}
5170
5171// Add supplemental CF attributes for some products.
5172void
5174
5175 BESDEBUG("h5", "GMFile::Coming to Add_Supplement_Attrs()"<<endl);
5176 if (General_Product == product_type || true == add_path) {
5177 File::Add_Supplement_Attrs(add_path);
5178
5179 if(true == add_path) {
5180 // Adding variable original name(origname) and full path(fullpath)
5181 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
5182 if (((*irv)->cvartype == CV_EXIST) || ((*irv)->cvartype == CV_MODIFY)) {
5183 auto attr = new Attribute();
5184 const string varname = (*irv)->name;
5185 const string attrname = "origname";
5186 Add_Str_Attr(attr,attrname,varname);
5187 (*irv)->attrs.push_back(attr);
5188 }
5189 }
5190
5191 for (auto irv = this->cvars.begin();
5192 irv != this->cvars.end(); ++irv) {
5193 // Turn off the fullnamepath attribute when zero_storage_size is 0.
5194 // Use the BES key since quite a few testing cases will be affected.
5195 // KY 2020-03-23
5196 if((*irv)->zero_storage_size == false
5197 || HDF5RequestHandler::get_no_zero_size_fullnameattr() == false) {
5198 if (((*irv)->cvartype == CV_EXIST) || ((*irv)->cvartype == CV_MODIFY)) {
5199 auto attr = new Attribute();
5200 const string varname = (*irv)->fullpath;
5201 const string attrname = "fullnamepath";
5202 Add_Str_Attr(attr,attrname,varname);
5203 (*irv)->attrs.push_back(attr);
5204 }
5205 }
5206 }
5207
5208 for (auto irv = this->spvars.begin(); irv != this->spvars.end(); ++irv) {
5209 auto attr = new Attribute();
5210 const string varname = (*irv)->name;
5211 const string attrname = "origname";
5212 Add_Str_Attr(attr,attrname,varname);
5213 (*irv)->attrs.push_back(attr);
5214 }
5215
5216 for (auto irv = this->spvars.begin(); irv != this->spvars.end(); ++irv) {
5217 // Turn off the fullnamepath attribute when zero_storage_size is 0.
5218 // Use the BES key since quite a few testing cases will be affected.
5219 // KY 2020-03-23
5220 if((*irv)->zero_storage_size == false
5221 || HDF5RequestHandler::get_no_zero_size_fullnameattr() == false) {
5222 auto attr = new Attribute();
5223 const string varname = (*irv)->fullpath;
5224 const string attrname = "fullnamepath";
5225 Add_Str_Attr(attr,attrname,varname);
5226 (*irv)->attrs.push_back(attr);
5227 }
5228 }
5229 }
5230 } // if (General_Product == product_type || true == add_path)
5231
5232 if(GPM_L1 == product_type || GPMS_L3 == product_type || GPMM_L3 == product_type)
5233 Add_GPM_Attrs();
5234 else if (Aqu_L3 == product_type)
5235 Add_Aqu_Attrs();
5236 else if (Mea_SeaWiFS_L2 == product_type || Mea_SeaWiFS_L3 == product_type)
5237 Add_SeaWiFS_Attrs();
5238
5239}
5240
5241// Add CF attributes for GPM products
5242void
5243GMFile:: Add_GPM_Attrs() {
5244
5245 BESDEBUG("h5", "Coming to Add_GPM_Attrs()"<<endl);
5246 vector<HDF5CF::Var *>::const_iterator it_v;
5247 vector<HDF5CF::Attribute *>::const_iterator ira;
5248 const string attr_name_be_replaced = "CodeMissingValue";
5249 const string attr_new_name = "_FillValue";
5250 const string attr2_name_be_replaced = "Units";
5251 const string attr2_new_name ="units";
5252
5253 // Need to convert String type CodeMissingValue to the corresponding _FilLValue
5254 // Create a function at HDF5CF.cc. use strtod,strtof,strtol etc. function to convert
5255 // string to the corresponding type.
5256 for (it_v = vars.begin(); it_v != vars.end(); ++it_v) {
5257 bool has_fvalue_attr = false;
5258 for(ira = (*it_v)->attrs.begin(); ira!= (*it_v)->attrs.end();ira++) {
5259 if(attr_new_name == (*ira)->name) {
5260 has_fvalue_attr = true;
5261 break;
5262 }
5263 }
5264
5265 if(false == has_fvalue_attr) {
5266 for(ira = (*it_v)->attrs.begin(); ira!= (*it_v)->attrs.end();ira++) {
5267 if(attr_name_be_replaced == (*ira)->name) {
5268 if((*ira)->dtype == H5FSTRING)
5269 Change_Attr_One_Str_to_Others((*ira),(*it_v));
5270 (*ira)->name = attr_new_name;
5271 (*ira)->newname = attr_new_name;
5272 }
5273 }
5274 }
5275
5276 }
5277
5278
5279 for (auto irv = this->cvars.begin();
5280 irv != this->cvars.end(); ++irv) {
5281 bool has_fvalue_attr = false;
5282
5283 for(ira = (*irv)->attrs.begin(); ira!= (*irv)->attrs.end();ira++) {
5284
5285 if(attr_new_name == (*ira)->name) {
5286 has_fvalue_attr = true;
5287 break;
5288 }
5289 }
5290 if(false == has_fvalue_attr) {
5291 for(ira = (*irv)->attrs.begin(); ira!= (*irv)->attrs.end();ira++) {
5292
5293 if(attr_name_be_replaced == (*ira)->name) {
5294 if((*ira)->dtype == H5FSTRING)
5295 Change_Attr_One_Str_to_Others((*ira),(*irv));
5296 (*ira)->name = attr_new_name;
5297 (*ira)->newname = attr_new_name;
5298 break;
5299 }
5300 }
5301 }
5302
5303
5304 if(product_type == GPM_L1) {
5305
5306 if ((*irv)->cvartype == CV_EXIST) {
5307 if((*irv)->name.find("Latitude") !=string::npos) {
5308 string unit_value = "degrees_north";
5309 Correct_GPM_L1_LatLon_units(*irv,unit_value);
5310
5311 }
5312 else if((*irv)->name.find("Longitude") !=string::npos) {
5313 string unit_value = "degrees_east";
5314 Correct_GPM_L1_LatLon_units(*irv,unit_value);
5315 }
5316 }
5317
5318
5319 else if ((*irv)->cvartype == CV_NONLATLON_MISS) {
5320
5321 string comment;
5322 const string attrname = "comment";
5323 auto attr = new Attribute();
5324
5325 {
5326 if((*irv)->name == "nchannel1")
5327 comment = "Number of Swath S1 channels (10V 10H 19V 19H 23V 37V 37H 89V 89H).";
5328 else if((*irv)->name == "nchannel2")
5329 comment = "Number of Swath S2 channels (166V 166H 183+/-3V 183+/-8V).";
5330 else if((*irv)->name == "nchan1")
5331 comment = "Number of channels in Swath 1.";
5332 else if((*irv)->name == "nchan2")
5333 comment = "Number of channels in Swath 2.";
5334 else if((*irv)->name == "VH")
5335 comment = "Number of polarizations.";
5336 else if((*irv)->name == "GMIxyz")
5337 comment = "x, y, z components in GMI instrument coordinate system.";
5338 else if((*irv)->name == "LNL")
5339 comment = "Linear and non-linear.";
5340 else if((*irv)->name == "nscan")
5341 comment = "Number of scans in the granule.";
5342 else if((*irv)->name == "nscan1")
5343 comment = "Typical number of Swath S1 scans in the granule.";
5344 else if((*irv)->name == "nscan2")
5345 comment = "Typical number of Swath S2 scans in the granule.";
5346 else if((*irv)->name == "npixelev")
5347 comment = "Number of earth view pixels in one scan.";
5348 else if((*irv)->name == "npixelht")
5349 comment = "Number of hot load pixels in one scan.";
5350 else if((*irv)->name == "npixelcs")
5351 comment = "Number of cold sky pixels in one scan.";
5352 else if((*irv)->name == "npixelfr")
5353 comment = "Number of full rotation earth view pixels in one scan.";
5354 else if((*irv)->name == "nfreq1")
5355 comment = "Number of frequencies in Swath 1.";
5356 else if((*irv)->name == "nfreq2")
5357 comment = "Number of frequencies in Swath 2.";
5358 else if((*irv)->name == "npix1")
5359 comment = "Number of pixels in Swath 1.";
5360 else if((*irv)->name == "npix2")
5361 comment = "Number of pixels in Swath 2.";
5362 else if((*irv)->name == "npix3")
5363 comment = "Number of pixels in Swath 3.";
5364 else if((*irv)->name == "npix4")
5365 comment = "Number of pixels in Swath 4.";
5366 else if((*irv)->name == "ncolds1")
5367 comment = "Maximum number of cold samples in Swath 1.";
5368 else if((*irv)->name == "ncolds2")
5369 comment = "Maximum number of cold samples in Swath 2.";
5370 else if((*irv)->name == "nhots1")
5371 comment = "Maximum number of hot samples in Swath 1.";
5372 else if((*irv)->name == "nhots2")
5373 comment = "Maximum number of hot samples in Swath 2.";
5374 else if((*irv)->name == "ntherm")
5375 comment = "Number of hot load thermisters.";
5376 else if((*irv)->name == "ntach")
5377 comment = "Number of tachometer readings.";
5378 else if((*irv)->name == "nsamt"){
5379 comment = "Number of sample types. ";
5380 comment = +"The types are: total science GSDR, earthview,hot load, cold sky.";
5381 }
5382 else if((*irv)->name == "nndiode")
5383 comment = "Number of noise diodes.";
5384 else if((*irv)->name == "n7")
5385 comment = "Number seven.";
5386 else if((*irv)->name == "nray")
5387 comment = "Number of angle bins in each NS scan.";
5388 else if((*irv)->name == "nrayMS")
5389 comment = "Number of angle bins in each MS scan.";
5390 else if((*irv)->name == "nrayHS")
5391 comment = "Number of angle bins in each HS scan.";
5392 else if((*irv)->name == "nbin")
5393 comment = "Number of range bins in each NS and MS ray. Bin interval is 125m.";
5394 else if((*irv)->name == "nbinHS")
5395 comment = "Number of range bins in each HS ray. Bin interval is 250m.";
5396 else if((*irv)->name == "nbinSZP")
5397 comment = "Number of range bins for sigmaZeroProfile.";
5398 else if((*irv)->name == "nbinSZPHS")
5399 comment = "Number of range bins for sigmaZeroProfile in each HS scan.";
5400 else if((*irv)->name == "nNP")
5401 comment = "Number of NP kinds.";
5402 else if((*irv)->name == "nearFar")
5403 comment = "Near reference, Far reference.";
5404 else if((*irv)->name == "foreBack")
5405 comment = "Forward, Backward.";
5406 else if((*irv)->name == "method")
5407 comment = "Number of SRT methods.";
5408 else if((*irv)->name == "nNode")
5409 comment = "Number of binNode.";
5410 else if((*irv)->name == "nDSD")
5411 comment = "Number of DSD parameters. Parameters are N0 and D0";
5412 else if((*irv)->name == "LS")
5413 comment = "Liquid, solid.";
5414 }
5415
5416 if(""==comment)
5417 delete attr;
5418 else {
5419 Add_Str_Attr(attr,attrname,comment);
5420 (*irv)->attrs.push_back(attr);
5421 }
5422
5423 }
5424 }
5425
5426 if(product_type == GPMS_L3 || product_type == GPMM_L3) {
5427 if ((*irv)->cvartype == CV_NONLATLON_MISS) {
5428
5429 string comment;
5430 const string attrname = "comment";
5431 auto attr = new Attribute();
5432
5433 {
5434 if((*irv)->name == "chn")
5435 comment = "Number of channels:Ku,Ka,KaHS,DPR.";
5436 else if((*irv)->name == "inst")
5437 comment = "Number of instruments:Ku,Ka,KaHS.";
5438 else if((*irv)->name == "tim")
5439 comment = "Number of hours(local time).";
5440 else if((*irv)->name == "ang"){
5441 comment = "Number of angles.The meaning of ang is different for each channel.";
5442 comment +=
5443 "For Ku channel all indices are used with the meaning 0,1,2,..6 =angle bins 24,";
5444 comment +=
5445 "(20,28),(16,32),(12,36),(8,40),(3,44),and (0,48).";
5446 comment +=
5447 "For Ka channel 4 indices are used with the meaning 0,1,2,3 = angle bins 12,(8,16),";
5448 comment +=
5449 "(4,20),and (0,24). For KaHS channel 4 indices are used with the meaning 0,1,2,3 =";
5450 comment += "angle bins(11,2),(7,16),(3,20),and (0.23).";
5451
5452 }
5453 else if((*irv)->name == "rt")
5454 comment = "Number of rain types: stratiform, convective,all.";
5455 else if((*irv)->name == "st")
5456 comment = "Number of surface types:ocean,land,all.";
5457 else if((*irv)->name == "bin"){
5458 comment = "Number of bins in histogram. The thresholds are different for different";
5459 comment +=" variables. see the file specification for this algorithm.";
5460 }
5461 else if((*irv)->name == "nvar") {
5462 comment = "Number of phase bins. Bins are counts of phase less than 100, ";
5463 comment +="counts of phase greater than or equal to 100 and less than 200, ";
5464 comment +="counts of phase greater than or equal to 200.";
5465 }
5466 else if((*irv)->name == "AD")
5467 comment = "Ascending or descending half of the orbit.";
5468 }
5469
5470 if(""==comment)
5471 delete attr;
5472 else {
5473 Add_Str_Attr(attr,attrname,comment);
5474 (*irv)->attrs.push_back(attr);
5475 }
5476
5477 }
5478 }
5479
5480
5481 if ((*irv)->cvartype == CV_SPECIAL) {
5482 if((*irv)->name == "nlayer" || (*irv)->name == "hgt"
5483 || (*irv)->name == "nalt") {
5484 auto attr = new Attribute();
5485 string unit_value = "km";
5486 Add_Str_Attr(attr,attr2_new_name,unit_value);
5487 (*irv)->attrs.push_back(attr);
5488
5489 auto attr1 = new Attribute();
5490 string attr1_axis="axis";
5491 string attr1_value = "Z";
5492 Add_Str_Attr(attr1,attr1_axis,attr1_value);
5493 (*irv)->attrs.push_back(attr1);
5494
5495 auto attr2 = new Attribute();
5496 string attr2_positive="positive";
5497 string attr2_value = "up";
5498 Add_Str_Attr(attr2,attr2_positive,attr2_value);
5499 (*irv)->attrs.push_back(attr2);
5500
5501 }
5502 if((*irv)->name == "hgt" || (*irv)->name == "nalt"){
5503 auto attr1 = new Attribute();
5504 string comment ="Number of heights above the earth ellipsoid";
5505 Add_Str_Attr(attr1,"comment",comment);
5506 (*irv)->attrs.push_back(attr1);
5507 }
5508
5509 }
5510
5511 }
5512
5513
5514// Old code, leave it for the time being
5515#if 0
5516 const string fill_value_attr_name = "_FillValue";
5517 vector<HDF5CF::Var *>::const_iterator it_v;
5518 vector<HDF5CF::Attribute *>::const_iterator ira;
5519
5520 for (it_v = vars.begin();
5521 it_v != vars.end(); ++it_v) {
5522
5523 bool has_fillvalue = false;
5524 for(ira = (*it_v)->attrs.begin(); ira!= (*it_v)->attrs.end();ira++) {
5525 if (fill_value_attr_name == (*ira)->name){
5526 has_fillvalue = true;
5527 break;
5528 }
5529
5530 }
5531
5532 // Add the fill value
5533 if (has_fillvalue != true ) {
5534
5535 if(H5FLOAT32 == (*it_v)->dtype) {
5536 Attribute* attr = new Attribute();
5537 float _FillValue = -9999.9;
5538 Add_One_Float_Attr(attr,fill_value_attr_name,_FillValue);
5539 (*it_v)->attrs.push_back(attr);
5540 }
5541 }
5542 }// for (it_v = vars.begin(); ...
5543#endif
5544
5545}
5546
5547// For GPM level 1 data, var must have names that contains either "Latitude" nor "Longitude".
5548void
5549GMFile:: Correct_GPM_L1_LatLon_units(Var *var, const string unit_value) {
5550
5551 BESDEBUG("h5", "Coming to Correct_GPM_L1_LatLon_units()"<<endl);
5552 const string Unit_name = "Units";
5553 const string unit_name = "units";
5554
5555 // Delete "units" and "Units"
5556 for(auto ira = var->attrs.begin(); ira!= var->attrs.end();) {
5557 if(unit_name == (*ira)->name) {
5558 delete(*ira);
5559 ira = var->attrs.erase(ira);
5560 }
5561 else if(Unit_name == (*ira)->name) {
5562 delete(*ira);
5563 ira = var->attrs.erase(ira);
5564 }
5565 else
5566 ++ira;
5567 }
5568 // Add the correct units for Latitude and Longitude
5569 // Note: the reason we do this way, for some versions of GPM, units is degrees,
5570 // rather than degrees_north.. So units also needs to be corrected to follow CF.
5571 auto attr = new Attribute();
5572 Add_Str_Attr(attr,unit_name,unit_value);
5573 var->attrs.push_back(attr);
5574}
5575
5576
5577
5578// Add attributes for Aquarius products
5579void
5580GMFile:: Add_Aqu_Attrs() {
5581
5582 BESDEBUG("h5", "Coming to Add_Aqu_Attrs()"<<endl);
5583 vector<HDF5CF::Var *>::const_iterator it_v;
5584 vector<HDF5CF::Attribute *>::const_iterator ira;
5585
5586 const string orig_longname_attr_name = "Parameter";
5587 const string longname_attr_name ="long_name";
5588 string longname_value;
5589
5590
5591 const string orig_units_attr_name = "Units";
5592 const string units_attr_name = "units";
5593 string units_value;
5594
5595 const string orig_valid_min_attr_name = "Data Minimum";
5596 const string valid_min_attr_name = "valid_min";
5597 float valid_min_value = 0;
5598
5599 const string orig_valid_max_attr_name = "Data Maximum";
5600 const string valid_max_attr_name = "valid_max";
5601 float valid_max_value = 0;
5602
5603 // The fill value is -32767.0. However, No _FillValue attribute is added.
5604 // So add it here. KY 2012-2-16
5605
5606 const string fill_value_attr_name = "_FillValue";
5607 float _FillValue = -32767.0;
5608
5609 for (ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
5610 if (orig_longname_attr_name == (*ira)->name) {
5611 Retrieve_H5_Attr_Value(*ira,"/");
5612 longname_value.resize((*ira)->value.size());
5613 copy((*ira)->value.begin(),(*ira)->value.end(),longname_value.begin());
5614
5615 }
5616 else if (orig_units_attr_name == (*ira)->name) {
5617 Retrieve_H5_Attr_Value(*ira,"/");
5618 units_value.resize((*ira)->value.size());
5619 copy((*ira)->value.begin(),(*ira)->value.end(),units_value.begin());
5620
5621 }
5622 else if (orig_valid_min_attr_name == (*ira)->name) {
5623 Retrieve_H5_Attr_Value(*ira,"/");
5624 memcpy(&valid_min_value,(void*)(&((*ira)->value[0])),(*ira)->value.size());
5625 }
5626
5627 else if (orig_valid_max_attr_name == (*ira)->name) {
5628 Retrieve_H5_Attr_Value(*ira,"/");
5629 memcpy(&valid_max_value,(void*)(&((*ira)->value[0])),(*ira)->value.size());
5630 }
5631
5632 }// end of for (ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira)
5633
5634 // New version Aqu(Q20112132011243.L3m_MO_SCI_V3.0_SSS_1deg.bz2) files seem to have CF attributes added.
5635 // In this case, we should not add extra CF attributes, or duplicate values may appear. KY 2015-06-20
5636 bool has_long_name = false;
5637 bool has_units = false;
5638 bool has_valid_min = false;
5639 bool has_valid_max = false;
5640 bool has_fillvalue = false;
5641
5642 for (it_v = vars.begin(); it_v != vars.end(); ++it_v) {
5643 if ("l3m_data" == (*it_v)->name) {
5644 for (ira = (*it_v)->attrs.begin(); ira != (*it_v)->attrs.end(); ++ira) {
5645 if (longname_attr_name == (*ira)->name)
5646 has_long_name = true;
5647 else if(units_attr_name == (*ira)->name)
5648 has_units = true;
5649 else if(valid_min_attr_name == (*ira)->name)
5650 has_valid_min = true;
5651 else if(valid_max_attr_name == (*ira)->name)
5652 has_valid_max = true;
5653 else if(fill_value_attr_name == (*ira)->name)
5654 has_fillvalue = true;
5655 }
5656 break;
5657 }
5658 } // end of for (it_v = vars.begin(); ...
5659
5660
5661 // Level 3 variable name is l3m_data
5662 for (it_v = vars.begin(); it_v != vars.end(); ++it_v) {
5663 if ("l3m_data" == (*it_v)->name) {
5664
5665 Attribute *attr = nullptr;
5666 // 1. Add the long_name attribute if no
5667 if(false == has_long_name) {
5668 attr = new Attribute();
5669 Add_Str_Attr(attr,longname_attr_name,longname_value);
5670 (*it_v)->attrs.push_back(attr);
5671 }
5672
5673 // 2. Add the units attribute
5674 if(false == has_units) {
5675 attr = new Attribute();
5676 Add_Str_Attr(attr,units_attr_name,units_value);
5677 (*it_v)->attrs.push_back(attr);
5678 }
5679
5680 // 3. Add the valid_min attribute
5681 if(false == has_valid_min) {
5682 attr = new Attribute();
5683 Add_One_Float_Attr(attr,valid_min_attr_name,valid_min_value);
5684 (*it_v)->attrs.push_back(attr);
5685 }
5686
5687 // 4. Add the valid_max attribute
5688 if(false == has_valid_max) {
5689 attr = new Attribute();
5690 Add_One_Float_Attr(attr,valid_max_attr_name,valid_max_value);
5691 (*it_v)->attrs.push_back(attr);
5692 }
5693
5694 // 5. Add the _FillValue attribute
5695 if(false == has_fillvalue) {
5696 attr = new Attribute();
5697 Add_One_Float_Attr(attr,fill_value_attr_name,_FillValue);
5698 (*it_v)->attrs.push_back(attr);
5699 }
5700
5701 break;
5702 }
5703 } // for (it_v = vars.begin(); ...
5704}
5705
5706// Add SeaWiFS attributes
5707void
5708GMFile:: Add_SeaWiFS_Attrs() {
5709
5710 BESDEBUG("h5", "Coming to Add_SeaWiFS_Attrs()"<<endl);
5711 // The fill value is -999.0. However, No _FillValue attribute is added.
5712 // So add it here. KY 2012-2-16
5713 const string fill_value_attr_name = "_FillValue";
5714 float _FillValue = -999.0;
5715 const string valid_range_attr_name = "valid_range";
5716 vector<HDF5CF::Var *>::const_iterator it_v;
5717 vector<HDF5CF::Attribute *>::const_iterator ira;
5718
5719
5720 for (it_v = vars.begin(); it_v != vars.end(); ++it_v) {
5721 if (H5FLOAT32 == (*it_v)->dtype) {
5722 bool has_fillvalue = false;
5723 bool has_validrange = false;
5724 for(ira = (*it_v)->attrs.begin(); ira!= (*it_v)->attrs.end();ira++) {
5725 if (fill_value_attr_name == (*ira)->name){
5726 has_fillvalue = true;
5727 break;
5728 }
5729
5730 else if(valid_range_attr_name == (*ira)->name) {
5731 has_validrange = true;
5732 break;
5733 }
5734
5735 }
5736 // Add the fill value
5737 if (has_fillvalue != true && has_validrange != true ) {
5738 auto attr = new Attribute();
5739 Add_One_Float_Attr(attr,fill_value_attr_name,_FillValue);
5740 (*it_v)->attrs.push_back(attr);
5741 }
5742 }
5743 }
5744}
5745
5746// Leave the following code for the time being
5747#if 0
5748// Handle the "coordinates" and "units" attributes of coordinate variables.
5750
5751 string co_attrname = "coordinates";
5752 string co_attrvalue="";
5753 string unit_attrname = "units";
5754 string nonll_unit_attrvalue ="level";
5755 string lat_unit_attrvalue ="degrees_north";
5756 string lon_unit_attrvalue ="degrees_east";
5757
5758 for (auto ircv = this->cvars.begin();
5759 ircv != this->cvars.end(); ++ircv) {
5760//cerr<<"CV name is "<<(*ircv)->name << " cv type is "<<(*ircv)->cvartype <<endl;
5761
5762 if ((*ircv)->cvartype == CV_NONLATLON_MISS) {
5763 Attribute * attr = new Attribute();
5764 Add_Str_Attr(attr,unit_attrname,nonll_unit_attrvalue);
5765 (*ircv)->attrs.push_back(attr);
5766 }
5767
5768 else if ((*ircv)->cvartype == CV_LAT_MISS) {
5769//cerr<<"Should add new attribute "<<endl;
5770 Attribute * attr = new Attribute();
5771// float temp = -999.9;
5772// Add_One_Float_Attr(attr,unit_attrname,temp);
5773 Add_Str_Attr(attr,unit_attrname,lat_unit_attrvalue);
5774 (*ircv)->attrs.push_back(attr);
5775//cerr<<"After adding new attribute "<<endl;
5776 }
5777
5778 else if ((*ircv)->cvartype == CV_LON_MISS) {
5779 Attribute * attr = new Attribute();
5780 Add_Str_Attr(attr,unit_attrname,lon_unit_attrvalue);
5781 (*ircv)->attrs.push_back(attr);
5782 }
5783 } // for (auto ircv = this->cvars.begin(); ...
5784
5785 // No need to handle MeaSUREs SeaWiFS level 2 products
5786 if(product_type == Mea_SeaWiFS_L2)
5787 return;
5788
5789 // GPM level 1 needs to be handled separately
5790 else if(product_type == GPM_L1) {
5791 Handle_GPM_l1_Coor_Attr();
5792 return;
5793 }
5794 // No need to handle products that follow COARDS.
5795 else if (true == iscoard) {
5796 // May need to check coordinates for 2-D lat/lon but cannot treat those lat/lon as CV case. KY 2015-12-10-TEMPPP
5797 return;
5798 }
5799
5800
5801 // Now handle the 2-D lat/lon case(note: this only applies to the one that dim. scale doesn't apply)
5802 for (auto ircv = this->cvars.begin();
5803 ircv != this->cvars.end(); ++ircv) {
5804 if((*ircv)->rank == 2) {
5805
5806 // The following code makes sure that the replacement only happens with the general 2-D lat/lon case.
5807 if(gp_latname == (*ircv)->name)
5808 Replace_Var_Str_Attr((*ircv),unit_attrname,lat_unit_attrvalue);
5809 else if(gp_lonname ==(*ircv)->name)
5810 Replace_Var_Str_Attr((*ircv),unit_attrname,lon_unit_attrvalue);
5811 }
5812 }
5813
5814 // Check the dimension names of 2-D lat/lon CVs
5815 string ll2d_dimname0,ll2d_dimname1;
5816 bool has_ll2d_coords = false;
5817 for (auto ircv = this->cvars.begin();
5818 ircv != this->cvars.end(); ++ircv) {
5819 if((*ircv)->rank == 2) {
5820 // Note: we should still use the original dim. name to match the general variables.
5821 ll2d_dimname0 = (*ircv)->getDimensions()[0]->name;
5822 ll2d_dimname1 = (*ircv)->getDimensions()[1]->name;
5823 if(ll2d_dimname0 !="" && ll2d_dimname1 !="")
5824 has_ll2d_coords = true;
5825 break;
5826 }
5827 }
5828
5829 if(true == has_ll2d_coords) {
5830
5831 for (auto irv = this->vars.begin();
5832 irv != this->vars.end(); ++irv) {
5833
5834 bool coor_attr_keep_exist = false;
5835
5836 // May need to delete only the "coordinates" with both 2-D lat/lon dim. KY 2015-12-07
5837 if(((*irv)->rank >=2)) {
5838
5839 short ll2dim_flag = 0;
5840 for (auto ird = (*irv)->dims.begin();
5841 ird != (*irv)->dims.end(); ++ ird) {
5842 if((*ird)->name == ll2d_dimname0)
5843 ll2dim_flag++;
5844 else if((*ird)->name == ll2d_dimname1)
5845 ll2dim_flag++;
5846 }
5847
5848 if(ll2dim_flag != 2)
5849 coor_attr_keep_exist = true;
5850
5851 // The following line doesn't apply to SMAP,it applies to Old SMAP Level 2 Simulation files.
5852 if(product_type == OSMAPL2S)
5853 coor_attr_keep_exist = true;
5854
5855 if (false == coor_attr_keep_exist) {
5856 for (auto ira =(*irv)->attrs.begin();
5857 ira !=(*irv)->attrs.end();) {
5858 if ((*ira)->newname == co_attrname) {
5859 delete (*ira);
5860 ira = (*irv)->attrs.erase(ira);
5861 }
5862 else {
5863 ++ira;
5864 }
5865 }// for (auto ira =(*irv)->attrs.begin(); ...
5866
5867 // Generate the "coordinates" attribute only for variables that have both 2-D lat/lon dim. names.
5868 for (auto ird = (*irv)->dims.begin();
5869 ird != (*irv)->dims.end(); ++ ird) {
5870 for (auto ircv = this->cvars.begin();
5871 ircv != this->cvars.end(); ++ircv) {
5872 if ((*ird)->name == (*ircv)->cfdimname)
5873 co_attrvalue = (co_attrvalue.empty())
5874 ?(*ircv)->newname:co_attrvalue + " "+(*ircv)->newname;
5875 }
5876 }
5877
5878 if (false == co_attrvalue.empty()) {
5879 Attribute * attr = new Attribute();
5880 Add_Str_Attr(attr,co_attrname,co_attrvalue);
5881 (*irv)->attrs.push_back(attr);
5882 }
5883
5884 co_attrvalue.clear();
5885 } // for (auto irv = this->vars.begin(); ...
5886 }
5887 }
5888 }
5889}
5890#endif
5891
5892
5893// Handle the "coordinates" and "units" attributes of coordinate variables.
5895
5896 BESDEBUG("h5", "GMFile::Coming to Handle_Coor_Attr()"<<endl);
5897 string co_attrname = "coordinates";
5898 string co_attrvalue="";
5899 string unit_attrname = "units";
5900 string nonll_unit_attrvalue ="level";
5901 string lat_unit_attrvalue ="degrees_north";
5902 string lon_unit_attrvalue ="degrees_east";
5903
5904 // Attribute units should be added for coordinate variables that
5905 // have the type CV_NONLATLON_MISS,CV_LAT_MISS and CV_LON_MISS.
5906 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
5907
5908 if ((*ircv)->cvartype == CV_NONLATLON_MISS) {
5909 auto attr = new Attribute();
5910 Add_Str_Attr(attr,unit_attrname,nonll_unit_attrvalue);
5911 (*ircv)->attrs.push_back(attr);
5912 }
5913 else if ((*ircv)->cvartype == CV_LAT_MISS) {
5914 auto attr = new Attribute();
5915 Add_Str_Attr(attr,unit_attrname,lat_unit_attrvalue);
5916 (*ircv)->attrs.push_back(attr);
5917 }
5918 else if ((*ircv)->cvartype == CV_LON_MISS) {
5919 auto attr = new Attribute();
5920 Add_Str_Attr(attr,unit_attrname,lon_unit_attrvalue);
5921 (*ircv)->attrs.push_back(attr);
5922 }
5923 } // end of for
5924
5925 // No need to handle MeaSUREs SeaWiFS level 2 products
5926 if(product_type == Mea_SeaWiFS_L2)
5927 return;
5928
5929 // GPM level 1 needs to be handled separately
5930 else if(product_type == GPM_L1) {
5931 Handle_GPM_l1_Coor_Attr();
5932 return;
5933 }
5934
5935 // Handle Lat/Lon with "coordinates" attribute.
5936 else if(product_type == General_Product && gproduct_pattern == GENERAL_LATLON_COOR_ATTR){
5937 Handle_LatLon_With_CoordinateAttr_Coor_Attr();
5938 return;
5939 }
5940 // No need to handle products that follow COARDS.
5941 else if (true == iscoard) {
5942
5943 // If we find that there are groups that should check the coordinates attribute of the variable.
5944 // We should flatten the path inside the coordinates.(this is the case mainly for netcdf-4 2D lat/lon case)
5945 if(grp_cv_paths.size() >0) {
5946 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
5947 if(grp_cv_paths.find(HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath)) != grp_cv_paths.end()){
5948
5949 // Check the "coordinates" attribute and flatten the values.
5950 Flatten_VarPath_In_Coordinates_Attr(*irv);
5951 }
5952 }
5953 }
5954 return;
5955 }
5956
5957 // Now handle the 2-D lat/lon case
5958 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
5959
5960 if((*ircv)->rank == 2 && (*ircv)->cvartype == CV_EXIST) {
5961
5962 //Note: When the 2nd parameter is true in the function Is_geolatlon, it checks the lat/latitude/Latitude
5963 // When the 2nd parameter is false in the function Is_geolatlon, it checks the lon/longitude/Longitude
5964 // The following code makes sure that the replacement only happens with the general 2-D lat/lon case.
5965 // The following code is commented out since we find an OMPS-NPP case that has the no-CF unit for
5966 // "Latitude". So just to check the latitude and longitude and if the units are not CF-compliant,
5967 // change them. KY 2020-02-27
5968#if 0
5969 if(gp_latname == (*ircv)->name) {
5970 // Only if gp_latname is not lat/latitude/Latitude, change the units
5971 if(false == Is_geolatlon(gp_latname,true))
5972 Replace_Var_Str_Attr((*ircv),unit_attrname,lat_unit_attrvalue);
5973 }
5974 else if(gp_lonname ==(*ircv)->name) {
5975 // Only if gp_lonname is not lon/longitude/Longitude, change the units
5976 if(false == Is_geolatlon(gp_lonname,false))
5977 Replace_Var_Str_Attr((*ircv),unit_attrname,lon_unit_attrvalue);
5978 }
5979#endif
5980
5981 // We meet several products that miss the 2-D latitude and longitude CF units although they
5982 // have the CV names like latitude/longitude, we should double check this case,
5983 // and add the correct CF units if possible. We will watch if this is the right way.
5984 //else if(true == Is_geolatlon((*ircv)->name,true))
5985 if(true == Is_geolatlon((*ircv)->name,true))
5986 Replace_Var_Str_Attr((*ircv),unit_attrname,lat_unit_attrvalue);
5987 else if(true == Is_geolatlon((*ircv)->name,false))
5988 Replace_Var_Str_Attr((*ircv),unit_attrname,lon_unit_attrvalue);
5989 }
5990 } // for (auto ircv = this->cvars.begin()
5991
5992 // If we find that there are groups that we should check the coordinates attribute of the variable under,
5993 // we should flatten the path inside the coordinates. Note this is for 2D-latlon CV netCDF-4-like case.
5994 if(grp_cv_paths.size() >0) {
5995 for (auto irv = this->vars.begin();
5996 irv != this->vars.end(); ++irv) {
5997 if(grp_cv_paths.find(HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath)) != grp_cv_paths.end()){
5998
5999 // Check the "coordinates" attribute and flatten the values.
6000 Flatten_VarPath_In_Coordinates_Attr(*irv);
6001 }
6002 }
6003 }
6004
6005 // Check if having 2-D lat/lon CVs
6006 bool has_ll2d_coords = false;
6007
6008 // Since iscoard is false up to this point, So the netCDF-4 like 2-D lat/lon case must fulfill if the program comes here.
6009 if(General_Product == this->product_type && GENERAL_DIMSCALE == this->gproduct_pattern)
6010 has_ll2d_coords = true;
6011 else {// For other cases. Need to see if there is a case. KY 2016-07-07
6012 string ll2d_dimname0;
6013 string ll2d_dimname1;
6014 for (auto ircv = this->cvars.begin();
6015 ircv != this->cvars.end(); ++ircv) {
6016 if((*ircv)->rank == 2) {
6017 // Note: we should still use the original dim. name to match the general variables.
6018 ll2d_dimname0 = (*ircv)->getDimensions()[0]->name;
6019 ll2d_dimname1 = (*ircv)->getDimensions()[1]->name;
6020 if(ll2d_dimname0 !="" && ll2d_dimname1 !="")
6021 has_ll2d_coords = true;
6022 break;
6023 }
6024 }
6025 }
6026
6027 // We now walk through all the >=2 vars and flatten the "coordinates"
6028 if(true == has_ll2d_coords) {
6029
6030 // For some netCDF-4-like 2-D lat/lon cases, we may need to forcely flatten the coordinates.
6031 // This case usually happens when the data producers follow the CF and the NASA DIWG guideline to
6032 // provide the absolute path of the coordinates as the value of the "coordinates" attribute.
6033 // The handler doesn't need to figure out the contents of the coordinates attribute but to
6034 // flatten the path inside.
6035 // However, the BES Key FORCENDCoorAttr must be set.
6036 bool force_flatten_coor_attr = HDF5RequestHandler::get_force_flatten_coor_attr();
6037
6038 // We also need to find if we have coordinates attribute for >=2D variables.
6039 // If not, the handler has to figure out the coordinates.
6040 bool has_coor_attr_ge_2d_vars = false;
6041 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
6042 if((*irv)->rank >=2){
6043 for (auto ira =(*irv)->attrs.begin(); ira !=(*irv)->attrs.end();++ira) {
6044 // We will check if we have the coordinate attribute
6045 if((*ira)->name == co_attrname) {
6046 has_coor_attr_ge_2d_vars = true;
6047 break;
6048 }
6049 }
6050 if(has_coor_attr_ge_2d_vars == true)
6051 break;
6052 }
6053 }
6054#if 0
6055 // Here we may need to consider the special case for HDF-EOS5. The "Data Fields" etc should not be
6056 // in the group path. May need to let DIWG provide a guideline for this issue.
6057 bool is_hybrid_eos5= false;
6058 if(force_flatten_coor_attr == true && has_coor_attr_ge_2d_vars == true)
6059 is_hybrid_eos5 = Is_Hybrid_EOS5();
6060#endif
6061 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
6062
6063 bool has_coor = false;
6064 for (auto ira =(*irv)->attrs.begin(); ira !=(*irv)->attrs.end();++ira) {
6065 // We will check if we have the coordinate attribute
6066 if((*ira)->name == co_attrname) {
6067 has_coor = true;
6068 break;
6069 }
6070 }
6071
6072 // The coordinates attribute is flattened by force.
6073 if(true == force_flatten_coor_attr && true == has_coor) {
6074#if 0
6075 if(is_hybrid_eos5 == true) {
6076 Flatten_VarPath_In_Coordinates_Attr_EOS5((*irv));
6077 }
6078 else
6079#endif
6080 Flatten_VarPath_In_Coordinates_Attr((*irv));
6081 }
6082
6083 else if(((*irv)->rank >=2) && (has_coor_attr_ge_2d_vars == false || false == force_flatten_coor_attr)) {
6084
6085 bool coor_attr_keep_exist = false;
6086
6087 // Check if this var is under group_cv_paths, no, then check if this var's dims are the same as the dims of 2-D CVars
6088 if(grp_cv_paths.find(HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath)) == grp_cv_paths.end())
6089
6090 // If finding this var is associated with 2-D lat/lon CVs, not keep the original "coordinates" attribute.
6091 coor_attr_keep_exist = Check_Var_2D_CVars(*irv);
6092 else {
6093 coor_attr_keep_exist = true;
6094 }
6095
6096 // The following two lines are just for old smap level 2 case.
6097 if(product_type == OSMAPL2S)
6098 coor_attr_keep_exist = true;
6099
6100 // Need to delete the original "coordinates" and rebuild the "coordinates" if this var is associated with the 2-D lat/lon CVs.
6101 if (false == coor_attr_keep_exist) {
6102 for (auto ira =(*irv)->attrs.begin(); ira !=(*irv)->attrs.end();) {
6103 if ((*ira)->newname == co_attrname) {
6104 delete (*ira);
6105 ira = (*irv)->attrs.erase(ira);
6106 }
6107 else {
6108 ++ira;
6109 }
6110 }// for (auto ira =(*irv)->attrs.begin(); ...
6111
6112 // Generate the new "coordinates" attribute.
6113 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ ird) {
6114 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
6115 if ((*ird)->name == (*ircv)->cfdimname)
6116 co_attrvalue = (co_attrvalue.empty())
6117 ?(*ircv)->newname:co_attrvalue + " "+(*ircv)->newname;
6118 }
6119 }
6120
6121 if (false == co_attrvalue.empty()) {
6122 auto attr = new Attribute();
6123 Add_Str_Attr(attr,co_attrname,co_attrvalue);
6124 (*irv)->attrs.push_back(attr);
6125 }
6126
6127 co_attrvalue.clear();
6128 (*irv)->coord_attr_add_path = false;
6129 } // for (auto irv = this->vars.begin(); ...
6130 }
6131 }
6132 }
6133}
6134
6135// Handle GPM level 1 coordiantes attributes.
6136void GMFile:: Handle_GPM_l1_Coor_Attr() {
6137
6138 BESDEBUG("h5", "Coming to Handle_GPM_l1_Coor_Attr()"<<endl);
6139 // Build a map from CFdimname to 2-D lat/lon variable name, should be something like: aa_list[cfdimname]=s1_latitude .
6140 // Loop all variables
6141 // Inner loop: for all dims of a var
6142 // if(dimname matches the dim(not cfdim) name of one of 2-D lat/lon,
6143 // check if the variable's full path contains the path of one of 2-D lat/lon,
6144 // yes, build its cfdimname = path+ dimname, check this cfdimname with the cfdimname of the corresponding 2-D lat/lon
6145 // If matched, save this latitude variable name as one of the coordinate variable.
6146 // else this is a 3rd-dimension cv, just use the dimension name(or the corresponding cv name maybe through a map).
6147
6148 // Prepare 1) 2-D CVar(lat,lon) corresponding dimension name set.
6149 // 2) cfdim name to cvar name map(don't need to use a map, just a holder. It should be fine.
6150
6151 // "coordinates" attribute name and value. We only need to provide this atttribute for variables that have 2-D lat/lon
6152 string co_attrname = "coordinates";
6153 string co_attrvalue="";
6154
6155 // 2-D cv dimname set.
6156 set<string> cvar_2d_dimset;
6157
6158 pair<map<string,string>::iterator,bool>mapret;
6159
6160 // Hold the mapping from cfdimname to 2-D cvar name. Something like nscan->lat, npixel->lon
6161 map<string,string>cfdimname_to_cvar2dname;
6162
6163 // Loop through cv variables to build 2-D cv dimname set and the mapping from cfdimname to 2-D cvar name.
6164 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
6165
6166 //This CVar must be 2-D array.
6167 if((*irv)->rank == 2) {
6168
6169#if 0
6170//cerr<<"2-D cv name is "<<(*irv)->name <<endl;
6171//cerr<<"2-D cv new name is "<<(*irv)->newname <<endl;
6172//cerr<<"(*irv)->cfdimname is "<<(*irv)->cfdimname <<endl;
6173#endif
6174
6175 for(auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird)
6176 cvar_2d_dimset.insert((*ird)->name);
6177
6178 // Generate cfdimname to cvar2d map
6179 mapret = cfdimname_to_cvar2dname.insert(pair<string,string>((*irv)->cfdimname,(*irv)->newname));
6180 if (false == mapret.second)
6181 throw4("The cf dimension name ",(*irv)->cfdimname," should map to 2-D coordinate variable",
6182 (*irv)->newname);
6183 }
6184 }
6185
6186 // Loop through the variable list to build the coordinates.
6187 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
6188
6189 // Only apply to >2D variables.
6190 if((*irv)->rank >=2) {
6191
6192 // The variable dimension names must be found in the 2D cvar dim. nameset.
6193 // The flag must be at least 2.
6194 short have_2d_dimnames_flag = 0;
6195 for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
6196 if (cvar_2d_dimset.find((*ird)->name)!=cvar_2d_dimset.end())
6197 have_2d_dimnames_flag++;
6198 }
6199
6200 // Final candidates to have 2-D CVar coordinates.
6201 if(have_2d_dimnames_flag >=2) {
6202
6203 // Obtain the variable path
6204 string var_path;
6205 if((*irv)->fullpath.size() > (*irv)->name.size())
6206 var_path=(*irv)->fullpath.substr(0,(*irv)->fullpath.size()-(*irv)->name.size());
6207 else
6208 throw4("The variable full path ",(*irv)->fullpath," doesn't contain the variable name ",
6209 (*irv)->name);
6210
6211 // A flag to identify if this variable really needs the 2-D coordinate variables.
6212 short cv_2d_flag = 0;
6213
6214 // 2-D coordinate variable names for the potential variable candidate
6215 vector<string> cv_2d_names;
6216
6217 // Dimension names of the 2-D coordinate variables.
6218 set<string> cv_2d_dimnames;
6219
6220 // Loop through the map from dim. name to coordinate name.
6221 for(map<string,string>::const_iterator itm = cfdimname_to_cvar2dname.begin();
6222 itm != cfdimname_to_cvar2dname.end();++itm) {
6223
6224 // Obtain the dimension name from the cfdimname.
6225 string reduced_dimname = HDF5CFUtil::obtain_string_after_lastslash(itm->first);
6226 string cfdim_path;
6227 if(itm->first.size() <= reduced_dimname.size())
6228 throw2("The cf dim. name of this dimension is not right.",itm->first);
6229 else
6230 cfdim_path= itm->first.substr(0,itm->first.size() - reduced_dimname.size());
6231 // cfdim_path will not be nullptr only when the cfdim name is for the 2-D cv var.
6232
6233 // Find the correct path,
6234 // Note:
6235 // var_path doesn't have to be the same as cfdim_path
6236 // consider the variable /a1/a2/foo and the latitude /a1/latitude(cfdimpath is /a1)
6237 // If there is no /a1/a2/latitude, the /a1/latitude can be used as the coordinate of /a1/a2/foo.
6238 // But we want to check if var_path is the same as cfdim_path first. So we check cfdimname_to_cvarname again.
6239 if(var_path == cfdim_path) {
6240 for (auto ird = (*irv)->dims.begin(); ird!=(*irv)->dims.end();++ird) {
6241 if(reduced_dimname == (*ird)->name) {
6242 cv_2d_flag++;
6243 cv_2d_names.push_back(itm->second);
6244 cv_2d_dimnames.insert((*ird)->name);
6245 }
6246 }
6247 }
6248 }
6249
6250 // Note:
6251 // var_path doesn't have to be the same as cfdim_path
6252 // consider the variable /a1/a2/foo and the latitude /a1/latitude(cfdimpath is /a1)
6253 // If there is no /a1/a2/latitude, the /a1/latitude can be used as the coordinate of /a1/a2/foo.
6254 // The variable has 2 coordinates(dimensions) if the flag is 2
6255 // If the flag is not 2, we will see if the above case stands.
6256 if(cv_2d_flag != 2) {
6257 cv_2d_flag = 0;
6258 // Loop through the map from dim. name to coordinate name.
6259 for(map<string,string>::const_iterator itm = cfdimname_to_cvar2dname.begin();
6260 itm != cfdimname_to_cvar2dname.end();++itm) {
6261 // Obtain the dimension name from the cfdimname.
6262 string reduced_dimname = HDF5CFUtil::obtain_string_after_lastslash(itm->first);
6263 string cfdim_path;
6264 if(itm->first.size() <= reduced_dimname.size())
6265 throw2("The cf dim. name of this dimension is not right.",itm->first);
6266 else
6267 cfdim_path= itm->first.substr(0,itm->first.size() - reduced_dimname.size());
6268 // cfdim_path will not be nullptr only when the cfdim name is for the 2-D cv var.
6269
6270 // Find the correct path,
6271 // Note:
6272 // var_path doesn't have to be the same as cfdim_path
6273 // consider the variable /a1/a2/foo and the latitude /a1/latitude(cfdimpath is /a1)
6274 // If there is no /a1/a2/latitude, the /a1/latitude can be used as the coordinate of /a1/a2/foo.
6275 //
6276 if(var_path.find(cfdim_path)!=string::npos) {
6277 for (auto ird = (*irv)->dims.begin(); ird!=(*irv)->dims.end();++ird) {
6278 if(reduced_dimname == (*ird)->name) {
6279 cv_2d_flag++;
6280 cv_2d_names.push_back(itm->second);
6281 cv_2d_dimnames.insert((*ird)->name);
6282 }
6283 }
6284 }
6285
6286 }
6287 }
6288
6289 // Now we got all cases.
6290 if(2 == cv_2d_flag) {
6291
6292 // Add latitude and longitude to the 'coordinates' attribute.
6293 co_attrvalue = cv_2d_names[0] + " " + cv_2d_names[1];
6294 if((*irv)->rank >2) {
6295 for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
6296
6297 // Add 3rd-dimension to the 'coordinates' attribute.
6298 if(cv_2d_dimnames.find((*ird)->name) == cv_2d_dimnames.end())
6299 co_attrvalue = co_attrvalue + " " +(*ird)->newname;
6300 }
6301 }
6302 auto attr = new Attribute();
6303 Add_Str_Attr(attr,co_attrname,co_attrvalue);
6304 (*irv)->attrs.push_back(attr);
6305 (*irv)->coord_attr_add_path = false;
6306 }
6307 }
6308 }
6309 }
6310}
6311
6312// This routine is for handling "coordinates" for the GENERAL_LATLON_COOR_ATTR pattern of General_Product.
6313void GMFile::Handle_LatLon_With_CoordinateAttr_Coor_Attr() {
6314
6315 BESDEBUG("h5", "Coming to Handle_LatLon_With_CoordinateAttr_Coor_Attr()"<<endl);
6316 string co_attrname = "coordinates";
6317
6318 // Loop through all rank >1 variables
6319 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
6320 if((*irv)->rank >= 2) {
6321 for (auto ira =(*irv)->attrs.begin(); ira !=(*irv)->attrs.end(); ++ira) {
6322 if((*ira)->name == co_attrname) {
6323 // If having the coordinates attribute, check if the "coordinates" variables match 2-D lat/lon CV condition,
6324 // if yes, flatten the coordinates attribute.
6325 string coor_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
6326 if(Coord_Match_LatLon_NameSize(coor_value) == true)
6327 Flatten_VarPath_In_Coordinates_Attr(*irv);
6328 // If the "coordinates" variables don't match the first condition, we can still check
6329 // if we can find the corresponding "coordinates" variables that match the names under the same group,
6330 // if yes, we add the path to the attribute "coordinates".
6331 else if(Coord_Match_LatLon_NameSize_Same_Group(coor_value,HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath)) == true)
6332 Add_VarPath_In_Coordinates_Attr(*irv,coor_value);
6333 // For other cases, we don't do anything with the "coordinates".
6334 break;
6335 }
6336 }
6337 }
6338 }
6339
6340}
6341
6342// We will check the "coordinates variables" stored in the coordinate attribute match the
6343// checked latlon_name_pairs for the GENERAL_LATLON_COOR_ATTR case.
6344bool GMFile::Coord_Match_LatLon_NameSize(const string & coord_values) {
6345
6346 BESDEBUG("h5", "Coming to Coord_Match_LatLon_NameSize()"<<endl);
6347 bool ret_value =false;
6348 vector<string> coord_values_vec;
6349 char sep=' ';
6350 int match_lat_name_pair_index = -1;
6351 int match_lon_name_pair_index = -2;
6352 int num_match_lat = 0;
6353 int num_match_lon = 0;
6354
6355
6356 // Decompose the coordinates attribute into a string vector.
6357 HDF5CFUtil::Split_helper(coord_values_vec,coord_values,sep);
6358
6359 // Some products ignore the first "/" of the coordinate path in the coordinate attribute, we will correct this
6360 if((coord_values_vec[0])[0] !='/') {
6361 for(auto irs=coord_values_vec.begin();irs!=coord_values_vec.end();++irs){
6362 if(((*irs).find_first_of('/'))!=string::npos) {
6363 *irs = '/' + (*irs);
6364 }
6365 }
6366 }
6367
6368 //Loop through all coordinate path stored in the coordinate patch vector,
6369 for(auto irs=coord_values_vec.begin();irs!=coord_values_vec.end();++irs){
6370
6371 // Loop through all the lat/lon pairs generated in the Check_LatLon_With_Coordinate_Attr routine
6372 // Remember the index and number appeared for both lat and lon.
6373 for(auto ivs=latloncv_candidate_pairs.begin(); ivs!=latloncv_candidate_pairs.end();++ivs) {
6374 if((*irs) == (*ivs).name1){
6375 match_lat_name_pair_index = distance(latloncv_candidate_pairs.begin(),ivs);
6376 num_match_lat++;
6377 }
6378 else if ((*irs) == (*ivs).name2) {
6379 match_lon_name_pair_index = distance(latloncv_candidate_pairs.begin(),ivs);
6380 num_match_lon++;
6381 }
6382 }
6383 }
6384 //Only when both index and the number of appearence match, we can set this be true.
6385 if((match_lat_name_pair_index == match_lon_name_pair_index) && (num_match_lat ==1) && (num_match_lon ==1))
6386 ret_value = true;
6387
6388 return ret_value;
6389
6390}
6391
6392//Some products only store the coordinate name(not full path) in the attribute coordinates, as
6393//long as it is valid, we should add the path to this coordinates.
6394bool GMFile::Coord_Match_LatLon_NameSize_Same_Group(const string &coord_values,const string &var_path) {
6395
6396 BESDEBUG("h5", "Coming to Coord_Match_LatLon_NameSize_Same_Group()"<<endl);
6397 bool ret_value =false;
6398 vector<string> coord_values_vec;
6399 char sep=' ';
6400 int match_lat_name_pair_index = -1;
6401 int match_lon_name_pair_index = -2;
6402 int num_match_lat = 0;
6403 int num_match_lon = 0;
6404
6405 HDF5CFUtil::Split_helper(coord_values_vec,coord_values,sep);
6406
6407 // Assume the 3rd-variable is also located under the same group if rank >=2
6408 for(auto irs=coord_values_vec.begin();irs!=coord_values_vec.end();++irs){
6409//cerr<<"coordinate values are "<<*irs <<endl;
6410 for(auto ivs=latloncv_candidate_pairs.begin(); ivs!=latloncv_candidate_pairs.end();++ivs) {
6411 string lat_name = HDF5CFUtil::obtain_string_after_lastslash((*ivs).name1);
6412 string lat_path = HDF5CFUtil::obtain_string_before_lastslash((*ivs).name1);
6413 string lon_name = HDF5CFUtil::obtain_string_after_lastslash((*ivs).name2);
6414 string lon_path = HDF5CFUtil::obtain_string_before_lastslash((*ivs).name2);
6415 if((*irs) == lat_name && lat_path == var_path){
6416 match_lat_name_pair_index = distance(latloncv_candidate_pairs.begin(),ivs);
6417 num_match_lat++;
6418 }
6419 else if ((*irs) == lon_name && lon_path == var_path) {
6420 match_lon_name_pair_index = distance(latloncv_candidate_pairs.begin(),ivs);
6421 num_match_lon++;
6422 }
6423 }
6424 }
6425
6426 if((match_lat_name_pair_index == match_lon_name_pair_index) && (num_match_lat ==1) && (num_match_lon ==1))
6427 ret_value = true;
6428
6429 return ret_value;
6430}
6431
6432// This is for the GENERAL_LATLON_COOR_ATTR pattern of General_Product.
6433void GMFile::Add_VarPath_In_Coordinates_Attr(Var *var, const string &coor_value) {
6434
6435 BESDEBUG("h5", "Coming to Add_VarPath_In_Coordinates_Attr()"<<endl);
6436 string new_coor_value;
6437 char sep =' ';
6438 string var_path = HDF5CFUtil::obtain_string_before_lastslash(var->fullpath) ;
6439 string var_flatten_path = get_CF_string(var_path);
6440
6441 // We need to loop through each element in the "coor_value".
6442 size_t ele_start_pos = 0;
6443 size_t cur_pos = coor_value.find_first_of(sep);
6444 while(cur_pos !=string::npos) {
6445 string tempstr = coor_value.substr(ele_start_pos,cur_pos-ele_start_pos);
6446 tempstr = var_flatten_path + tempstr;
6447 new_coor_value += tempstr + sep;
6448 ele_start_pos = cur_pos+1;
6449 cur_pos = coor_value.find_first_of(sep,cur_pos+1);
6450 }
6451
6452 if(ele_start_pos == 0)
6453 new_coor_value = var_flatten_path + coor_value;
6454 else
6455 new_coor_value += var_flatten_path + coor_value.substr(ele_start_pos);
6456
6457 string coor_attr_name = "coordinates";
6458 Replace_Var_Str_Attr(var,coor_attr_name,new_coor_value);
6459 var->coord_attr_add_path = false;
6460
6461}
6462
6463// Create Missing coordinate variables. Index numbers are used for these variables.
6464void GMFile:: Create_Missing_CV(GMCVar *GMcvar, const string& dimname) {
6465
6466 BESDEBUG("h5", "GMFile::Coming to Create_Missing_CV()"<<endl);
6467
6468 GMcvar->name = dimname;
6469 GMcvar->newname = GMcvar->name;
6470 GMcvar->fullpath = GMcvar->name;
6471 GMcvar->rank = 1;
6472 GMcvar->dtype = H5INT32;
6473 hsize_t gmcvar_dimsize = dimname_to_dimsize[dimname];
6474 bool unlimited_flag = dimname_to_unlimited[dimname];
6475 Dimension* gmcvar_dim = new Dimension(gmcvar_dimsize);
6476 gmcvar_dim->unlimited_dim = unlimited_flag;
6477 gmcvar_dim->name = dimname;
6478 gmcvar_dim->newname = dimname;
6479 GMcvar->dims.push_back(gmcvar_dim);
6480 GMcvar->cfdimname = dimname;
6481 GMcvar->cvartype = CV_NONLATLON_MISS;
6482 GMcvar->product_type = product_type;
6483}
6484
6485 // Check if this is just a netCDF-4 dimension. We need to check the dimension scale dataset attribute "NAME",
6486 // the value should start with "This is a netCDF dimension but not a netCDF variable".
6487bool GMFile::Is_netCDF_Dimension(Var *var) {
6488
6489 string netcdf_dim_mark = "This is a netCDF dimension but not a netCDF variable";
6490
6491 bool is_only_dimension = false;
6492
6493 for(auto ira = var->attrs.begin(); ira != var->attrs.end();ira++) {
6494
6495 if ("NAME" == (*ira)->name) {
6496
6497 Retrieve_H5_Attr_Value(*ira,var->fullpath);
6498 string name_value;
6499 name_value.resize((*ira)->value.size());
6500 copy((*ira)->value.begin(),(*ira)->value.end(),name_value.begin());
6501
6502 // Compare the attribute "NAME" value with the string netcdf_dim_mark. We only compare the string with the size of netcdf_dim_mark
6503 if (0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark))
6504 is_only_dimension = true;
6505
6506 break;
6507 }
6508 } // for(auto ira = var->attrs.begin(); ...
6509
6510 return is_only_dimension;
6511}
6512
6513// Handle attributes for special variables.
6514void
6516
6517}
6518
6519bool
6520GMFile::Is_Hybrid_EOS5() {
6521
6522 bool has_group_hdfeos = false;
6523 bool has_group_hdfeos_info = false;
6524
6525 // Too costly to check the dataset.
6526 // We will just check the attribute under /HDFEOS INFORMATION.
6527
6528 // First check if the HDFEOS groups are included
6529 for (auto irg = this->groups.begin(); irg != this->groups.end(); ++ irg) {
6530 if ("/HDFEOS" == (*irg)->path)
6531 has_group_hdfeos = true;
6532 else if("/HDFEOS INFORMATION" == (*irg)->path) {
6533 for(auto ira = (*irg)->attrs.begin();
6534 ira != (*irg)->attrs.end();ira++) {
6535 if("HDFEOSVersion" == (*ira)->name)
6536 has_group_hdfeos_info = true;
6537 }
6538 }
6539 if(true == has_group_hdfeos && true == has_group_hdfeos_info)
6540 break;
6541 }
6542
6543
6544 if(true == has_group_hdfeos && true == has_group_hdfeos_info)
6545 return true;
6546 else
6547 return false;
6548}
6549
6550void GMFile::Handle_Hybrid_EOS5() {
6551
6552 string eos_str="HDFEOS_";
6553 string eos_info_str="HDFEOS_INFORMATION_";
6554 string grid_str="GRIDS_";
6555 string swath_str="SWATHS_";
6556 string zas_str="ZAS_";
6557 string df_str="Data_Fields_";
6558 string gf_str="Geolocation_Fields_";
6559 for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
6560 string temp_var_name = (*irv)->newname;
6561
6562 bool remove_eos = Remove_EOS5_Strings(temp_var_name);
6563
6564 if(true == remove_eos)
6565 (*irv)->newname = get_CF_string(temp_var_name);
6566 else {//HDFEOS info and extra fields
6567 string::size_type eos_info_pos = temp_var_name.find(eos_info_str);
6568 if(eos_info_pos !=string::npos)
6569 (*irv)->newname = temp_var_name.erase(eos_info_pos,eos_info_str.size());
6570 else {// Check the extra fields
6571 if(Remove_EOS5_Strings_NonEOS_Fields(temp_var_name)==true)
6572 (*irv)->newname = get_CF_string(temp_var_name);
6573 }
6574 }
6575 }
6576
6577 // Now we need to handle the dimension names.
6578 for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
6579 for (auto ird = (*irv)->dims.begin(); ird!=(*irv)->dims.end(); ++ird) {
6580 string temp_dim_name = (*ird)->newname;
6581 bool remove_eos = Remove_EOS5_Strings(temp_dim_name);
6582
6583 if(true == remove_eos)
6584 (*ird)->newname = get_CF_string(temp_dim_name);
6585 else {//HDFEOS info and extra fields
6586 string::size_type eos_info_pos = temp_dim_name.find(eos_info_str);
6587 if(eos_info_pos !=string::npos)
6588 (*ird)->newname = temp_dim_name.erase(eos_info_pos,eos_info_str.size());
6589 else {// Check the extra fields
6590 if(Remove_EOS5_Strings_NonEOS_Fields(temp_dim_name)==true)
6591 (*ird)->newname = get_CF_string(temp_dim_name);
6592 }
6593 }
6594 }
6595 }
6596
6597 // We have to loop through all CVs.
6598 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
6599 string temp_var_name = (*irv)->newname;
6600
6601 bool remove_eos = Remove_EOS5_Strings(temp_var_name);
6602
6603 if(true == remove_eos)
6604 (*irv)->newname = get_CF_string(temp_var_name);
6605 else {//HDFEOS info and extra "fields"
6606 string::size_type eos_info_pos = temp_var_name.find(eos_info_str);
6607 if(eos_info_pos !=string::npos)
6608 (*irv)->newname = temp_var_name.erase(eos_info_pos,eos_info_str.size());
6609 else {// Check the extra fields
6610 if(Remove_EOS5_Strings_NonEOS_Fields(temp_var_name)==true)
6611 (*irv)->newname = get_CF_string(temp_var_name);
6612 }
6613 }
6614 }
6615 // Now we need to handle the dimension names.
6616 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); irv++) {
6617 for (auto ird = (*irv)->dims.begin(); ird!=(*irv)->dims.end(); ++ird) {
6618 string temp_dim_name = (*ird)->newname;
6619 bool remove_eos = Remove_EOS5_Strings(temp_dim_name);
6620
6621 if(true == remove_eos)
6622 (*ird)->newname = get_CF_string(temp_dim_name);
6623 else {// HDFEOS info and extra "fields"
6624 string::size_type eos_info_pos = temp_dim_name.find(eos_info_str);
6625 if(eos_info_pos !=string::npos)
6626 (*ird)->newname = temp_dim_name.erase(eos_info_pos,eos_info_str.size());
6627 else {// Check the extra "fields"
6628 if(Remove_EOS5_Strings_NonEOS_Fields(temp_dim_name)==true)
6629 (*ird)->newname = get_CF_string(temp_dim_name);
6630 }
6631 }
6632 }
6633 }
6634
6635 // Update the coordinate attribute values
6636 // We need to remove the HDFEOS special information from the coordinates attributes
6637 // since the variable names in the DAP output are already updated.
6638 for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
6639 for(auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();ira++) {
6640 // We cannot use Retrieve_Str_Attr_value for "coordinates" since "coordinates" may be added by the handler.
6641 // KY 2017-11-3
6642 if((*ira)->name == "coordinates") {
6643 string cor_values((*ira)->value.begin(),(*ira)->value.end()) ;
6644 bool change_cor_values = false;
6645 // Find the HDFEOS string
6646 if(cor_values.find(eos_str)==0) {
6647 if(cor_values.find(grid_str)!=string::npos) {// Grid
6648 cor_values = HDF5CFUtil::remove_substrings(cor_values,eos_str);
6649 cor_values = HDF5CFUtil::remove_substrings(cor_values,grid_str);
6650 string new_cor_values = HDF5CFUtil::remove_substrings(cor_values,df_str);
6651 if(new_cor_values.size() < cor_values.size()){//df_str is also removed.
6652 change_cor_values = true;
6653 cor_values = new_cor_values;
6654 }
6655 }
6656 else if(cor_values.find(zas_str)!=string::npos) {//ZA
6657 cor_values = HDF5CFUtil::remove_substrings(cor_values,eos_str);
6658 cor_values = HDF5CFUtil::remove_substrings(cor_values,zas_str);
6659 string new_cor_values = HDF5CFUtil::remove_substrings(cor_values,df_str);
6660 if(new_cor_values.size() < cor_values.size()){//df_str is also removed.
6661 change_cor_values = true;
6662 cor_values = new_cor_values;
6663 }
6664 }
6665 else if(cor_values.find(swath_str)!=string::npos) {//Swath
6666 cor_values = HDF5CFUtil::remove_substrings(cor_values,eos_str);
6667 cor_values = HDF5CFUtil::remove_substrings(cor_values,swath_str);
6668 string new_cor_values = HDF5CFUtil::remove_substrings(cor_values,df_str);
6669 if(new_cor_values.size() < cor_values.size()){//df_str is also removed.
6670 change_cor_values = true;
6671 cor_values = new_cor_values;
6672 }
6673 else {
6674 new_cor_values = HDF5CFUtil::remove_substrings(cor_values,gf_str);
6675 if(new_cor_values.size() < cor_values.size()){//gf_str is also removed.
6676 change_cor_values = true;
6677 cor_values = new_cor_values;
6678 }
6679 }
6680 }
6681 }
6682 if(true == change_cor_values) {//Update the coordinate values
6683 (*ira)->value.resize(cor_values.size());
6684 (*ira)->fstrsize=cor_values.size();
6685 (*ira)->strsize[0] = cor_values.size();
6686 copy(cor_values.begin(), cor_values.end(), (*ira)->value.begin());
6687 (*irv)->coord_attr_add_path = false;
6688 }
6689
6690 break;
6691 }
6692 }
6693
6694 }
6695}
6696
6697// This routine is for handling the hybrid-HDFEOS5 products that have to be treated as "general products"
6698bool GMFile:: Remove_EOS5_Strings(string &var_name) {
6699
6700 string eos_str="HDFEOS_";
6701 string grid_str="GRIDS_";
6702 string swath_str="SWATHS_";
6703 string zas_str="ZAS_";
6704 string df_str="Data_Fields_";
6705 string gf_str="Geolocation_Fields_";
6706 string temp_var_name = var_name;
6707
6708 bool remove_eos = false;
6709
6710 string::size_type eos_pos = temp_var_name.find(eos_str);
6711 if(eos_pos!=string::npos) {
6712 temp_var_name.erase(eos_pos,eos_str.size());
6713 // Check grid,swath and zonal
6714 string::size_type grid_pos=temp_var_name.find(grid_str);
6715 string::size_type grid_df_pos=string::npos;
6716 if(grid_pos!=string::npos)
6717 grid_df_pos = temp_var_name.find(df_str,grid_pos);
6718 string::size_type zas_pos = string::npos;
6719 string::size_type zas_df_pos=string::npos;
6720 if(grid_pos==string::npos || grid_df_pos ==string::npos)
6721 zas_pos=temp_var_name.find(zas_str);
6722 if(zas_pos!=string::npos)
6723 zas_df_pos=temp_var_name.find(df_str,zas_pos);
6724
6725 if(grid_pos !=string::npos && grid_df_pos!=string::npos) {
6726 temp_var_name.erase(grid_pos,grid_str.size());
6727 grid_df_pos = temp_var_name.find(df_str);
6728 temp_var_name.erase(grid_df_pos,df_str.size());
6729 remove_eos = true;
6730 }
6731 else if(zas_pos!=string::npos && zas_df_pos!=string::npos){
6732 temp_var_name.erase(zas_pos,zas_str.size());
6733 zas_df_pos = temp_var_name.find(df_str);
6734 temp_var_name.erase(zas_df_pos,df_str.size());
6735 remove_eos = true;
6736 }
6737 else {//Check both Geolocation and Data for Swath
6738
6739 string::size_type swath_pos=temp_var_name.find(swath_str);
6740 string::size_type swath_df_pos=string::npos;
6741 if(swath_pos!=string::npos)
6742 swath_df_pos=temp_var_name.find(df_str,swath_pos);
6743
6744 string::size_type swath_gf_pos=string::npos;
6745 if(swath_pos!=string::npos && swath_df_pos == string::npos)
6746 swath_gf_pos=temp_var_name.find(gf_str,swath_pos);
6747
6748 if(swath_pos !=string::npos) {
6749
6750 if(swath_df_pos!=string::npos) {
6751 temp_var_name.erase(swath_pos,swath_str.size());
6752 swath_df_pos = temp_var_name.find(df_str);
6753 temp_var_name.erase(swath_df_pos,df_str.size());
6754 remove_eos = true;
6755 }
6756 else if(swath_gf_pos!=string::npos) {
6757 temp_var_name.erase(swath_pos,swath_str.size());
6758 swath_gf_pos = temp_var_name.find(gf_str);
6759 temp_var_name.erase(swath_gf_pos,gf_str.size());
6760 remove_eos = true;
6761 }
6762 }
6763 }
6764 }
6765 if(true == remove_eos)
6766 var_name = temp_var_name;
6767
6768 return remove_eos;
6769}
6770
6771bool GMFile:: Remove_EOS5_Strings_NonEOS_Fields(string &var_name) {
6772
6773 string eos_str="HDFEOS_";
6774 string grid_str="GRIDS_";
6775 string swath_str="SWATHS_";
6776 string zas_str="ZAS_";
6777 string temp_var_name = var_name;
6778
6779 bool remove_eos = false;
6780
6781 string::size_type eos_pos = temp_var_name.find(eos_str);
6782 if(eos_pos!=string::npos) {
6783 temp_var_name.erase(eos_pos,eos_str.size());
6784 remove_eos = true;
6785
6786 // See if we need to further remove some fields
6787 if(temp_var_name.find(grid_str)==0)
6788 temp_var_name.erase(0,grid_str.size());
6789 else if(temp_var_name.find(swath_str)==0)
6790 temp_var_name.erase(0,swath_str.size());
6791 else if(temp_var_name.find(zas_str)==0)
6792 temp_var_name.erase(0,zas_str.size());
6793 }
6794 if(true == remove_eos)
6795 var_name = temp_var_name;
6796
6797
6798 return remove_eos;
6799}
6800
6801// We do have an AirMSPI HDF-EOS5 hybrid UTM product that has grid_mapping attribute.
6804}
6805
6808}
6809
6811
6812 // We need to remove the FakeDim added for the unsupported variables.
6813 // We found such a case in the AirMSPR product. A compound dataype array
6814 // is assigned to a FakeDim. We need to remove them.
6815 // KY 2017-11-2: no need to even check the unsupported_var_dspace now.
6816 if(this->unsupported_var_dtype == true) {
6817
6818 // Need to check if we have coordinate variables such as FakeDim?
6819 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end();) {
6820 if((*ircv)->newname.find("FakeDim")==0) {
6821 bool var_has_fakedim = false;
6822 for (auto irv2 = this->vars.begin(); irv2 != this->vars.end(); irv2++) {
6823 for (auto ird = (*irv2)->dims.begin(); ird !=(*irv2)->dims.end(); ird++) {
6824 if((*ird)->newname == (*ircv)->newname){
6825 var_has_fakedim = true;
6826 break;
6827 }
6828 }
6829 if(var_has_fakedim == true)
6830 break;
6831 }
6832 if(var_has_fakedim == false) {
6833 // Remove this cv, the variable is unsupported.
6834 delete(*ircv);
6835 ircv = this->cvars.erase(ircv);
6836 }
6837 else
6838 ++ircv;
6839 }
6840 else
6841 ++ircv;
6842 }
6843#if 0
6844 // We need to handle unlimited dimensions
6845 //if(removed_fakedim_vars.size()!=0) {
6846 //}
6847#endif
6848 }
6849
6850}
6851
6852//Rename NC4 NonCoordVars back to the original name. This is detected by CAR_ARCTAS files.
6853//By handling this way, the output will be the same as the netCDF handler output.
6854//Check HFVHANDLER-254 for more information.
6856
6857 if(true == this->have_nc4_non_coord) {
6858 string nc4_non_coord="_nc4_non_coord_";
6859 size_t nc4_non_coord_size= nc4_non_coord.size();
6860 for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
6861 if((*irv)->name.find(nc4_non_coord)==0)
6862 (*irv)->newname = (*irv)->newname.substr(nc4_non_coord_size,(*irv)->newname.size()-nc4_non_coord_size);
6863 }
6864
6865 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end();++ircv) {
6866 if((*ircv)->name.find(nc4_non_coord)==0)
6867 (*ircv)->newname = (*ircv)->newname.substr(nc4_non_coord_size,(*ircv)->newname.size()-nc4_non_coord_size);
6868 }
6869 }
6870
6871}
6872
6874
6875 BESDEBUG("h5", "GMFile::Coming to Add_Path_Coor_Attr()"<<endl);
6876 string co_attrname = "coordinates";
6877 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
6878 if((*irv)->coord_attr_add_path == true) {
6879 for (auto ira =(*irv)->attrs.begin(); ira !=(*irv)->attrs.end();++ira) {
6880 // We will check if we have the coordinate attribute
6881 if((*ira)->name == co_attrname) {
6882 string coor_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
6883 char sep=' ';
6884 vector<string>cvalue_vec;
6885 HDF5CFUtil::Split_helper(cvalue_vec,coor_value,sep);
6886 string new_coor_value;
6887 for (int i = 0; i<cvalue_vec.size();i++) {
6888 HDF5CFUtil::cha_co(cvalue_vec[i],(*irv)->fullpath);
6889 cvalue_vec[i] = get_CF_string(cvalue_vec[i]);
6890 if(i == 0)
6891 new_coor_value = cvalue_vec[i];
6892 else
6893 new_coor_value += sep+cvalue_vec[i];
6894
6895#if 0
6896//cout<<"co1["<<i<<"]= "<<cvalue_vec[i]<<endl;
6897#endif
6898 }
6899#if 0
6900//cout<<"new_coor_value is "<<new_coor_value<<endl;
6901#endif
6902 Replace_Var_Str_Attr((*irv),co_attrname,new_coor_value);
6903 break;
6904 }
6905 }
6906
6907 }
6908 }
6909}
6910
6911
6912
6913
6914// We will create some temporary coordinate variables. The resource allocoated
6915// for these variables need to be released.
6916void
6917GMFile::release_standalone_GMCVar_vector(vector<GMCVar*>&tempgc_vars){
6918
6919 for (auto i = tempgc_vars.begin(); i != tempgc_vars.end(); ) {
6920 delete(*i);
6921 i = tempgc_vars.erase(i);
6922 }
6923
6924}
6925
6926#if 0
6927void
6928GMFile::add_ignored_info_attrs(bool is_grp,bool is_first){
6929
6930}
6931void
6932GMFile::add_ignored_info_objs(bool is_dim_related, bool is_first) {
6933
6934}
6935#endif
6936
6937#if 0
6938bool
6939GMFile::ignored_dimscale_ref_list(Var *var) {
6940
6941 bool ignored_dimscale = true;
6942 if(General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern) {
6943
6944 bool has_dimscale = false;
6945 bool has_reference_list = false;
6946
6947 for(auto ira = var->attrs.begin(); ira != var->attrs.end();ira++) {
6948
6949 if((*ira)->name == "REFERENCE_LIST" &&
6950 false == HDF5CFUtil::cf_strict_support_type((*ira)->getType()))
6951 has_reference_list = true;
6952 if((*ira)->name == "CLASS") {
6953 Retrieve_H5_Attr_Value(*ira,var->fullpath);
6954 string class_value;
6955 class_value.resize((*ira)->value.size());
6956 copy((*ira)->value.begin(),(*ira)->value.end(),class_value.begin());
6957
6958 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
6959 // "DIMENSION_SCALE", which is 15.
6960 if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
6961 has_dimscale = true;
6962 }
6963 }
6964
6965 if(true == has_dimscale && true == has_reference_list) {
6966 ignored_dimscale= false;
6967 break;
6968 }
6969
6970 }
6971 }
6972 return ignored_dimscale;
6973}
6974
6975#endif
This class specifies the core engineering of mapping HDF5 to DAP by following CF.
include the entry functions to execute the handlers
This class represents one attribute.
Definition: HDF5CF.h:189
This class repersents one dimension of an HDF5 dataset(variable).
Definition: HDF5CF.h:145
This class retrieves all information from an HDF5 file.
Definition: HDF5CF.h:563
std::vector< Group * > groups
Non-root group vectors.
Definition: HDF5CF.h:795
virtual void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for datasets.
Definition: HDF5CF.cc:1273
virtual void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition: HDF5CF.cc:2208
virtual void Handle_Unsupported_Others(bool)
Handle other unmapped objects/attributes.
Definition: HDF5CF.cc:1320
virtual void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes.
Definition: HDF5CF.cc:727
std::vector< Var * > vars
Var vectors.
Definition: HDF5CF.h:789
virtual void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name.
Definition: HDF5CF.cc:2027
virtual void Retrieve_H5_Info(const char *path, hid_t file_id, bool)
Definition: HDF5CF.cc:168
std::vector< Attribute * > root_attrs
Root attribute vectors.
Definition: HDF5CF.h:792
virtual void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes.
Definition: HDF5CF.cc:916
virtual void Flatten_Obj_Name(bool)
Flatten the object name.
Definition: HDF5CF.cc:1370
virtual bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition: HDF5CF.cc:2188
This class is a derived class of CVar. It represents a coordinate variable for general HDF5 files.
Definition: HDF5CF.h:413
void Add_Supplement_Attrs(bool) override
Add supplemental attributes such as fullpath and original name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5173
void Add_Path_Coord_Attr()
Update the coordinate attribute to include path and also flatten.
Definition: HDF5GMCF.cc:6873
void Handle_Obj_NameClashing(bool)
Handle object name clashing for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4954
void Remove_Unused_FakeDimVars()
Unsupported datatype array may generate FakeDim. Remove them.
Definition: HDF5GMCF.cc:6810
void Update_Product_Type()
Update "product type" attributes for general HDF5 products.
Definition: HDF5GMCF.cc:239
bool Have_Grid_Mapping_Attrs() override
Check if having Grid Mapping Attrs.
Definition: HDF5GMCF.cc:6802
void Handle_SpVar_Attr() override
Handle special variable attributes for general NASA HDF5 products.
Definition: HDF5GMCF.cc:6515
void Handle_DimNameClashing() override
Definition: HDF5GMCF.cc:5055
void Handle_Grid_Mapping_Vars() override
Handle Grid Mapping Vars.
Definition: HDF5GMCF.cc:6806
void Adjust_H5_Attr_Value(Attribute *attr)
Adjust attribute values for general HDF5 products.
Definition: HDF5GMCF.cc:373
void Retrieve_H5_CVar_Supported_Attr_Values() override
Retrieve coordinate variable attributes.
Definition: HDF5GMCF.cc:329
void Retrieve_H5_Info(const char *path, hid_t file_id, bool include_attr) override
Retrieve DDS information from the HDF5 file; real implementation for general HDF5 products.
Definition: HDF5GMCF.cc:220
void Handle_SpVar() override
Handle special variables for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4738
void Remove_Unneeded_Objects()
Remove unneeded objects.
Definition: HDF5GMCF.cc:262
void Handle_CVar() override
Handle coordinate variables for general NASA HDF5 products.
Definition: HDF5GMCF.cc:2886
void Add_Dim_Name()
Add dimension name.
Definition: HDF5GMCF.cc:805
void Retrieve_H5_Supported_Attr_Values() override
Retrieve attribute values for the supported HDF5 datatypes for general HDF5 products.
Definition: HDF5GMCF.cc:343
void Handle_Coor_Attr() override
Handle "coordinates" attributes for general HDF5 products.
Definition: HDF5GMCF.cc:5894
void Rename_NC4_NonCoordVars()
Remove the _nc4_non_coord from the variable new names.
Definition: HDF5GMCF.cc:6855
void Handle_Unsupported_Dspace(bool) override
Handle unsupported HDF5 dataspaces for general HDF5 products.
Definition: HDF5GMCF.cc:578
void Handle_Unsupported_Others(bool) override
Handle other unmapped objects/attributes for general HDF5 products.
Definition: HDF5GMCF.cc:668
void Handle_Unsupported_Dtype(bool) override
Handle unsupported HDF5 datatypes for general HDF5 products.
Definition: HDF5GMCF.cc:391
void Flatten_Obj_Name(bool include_attr) override
Flatten the object name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4905
void Adjust_Dim_Name() override
Adjust dimension name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5114
void Adjust_Obj_Name() override
Adjust object names based on different general NASA HDF5 products.
Definition: HDF5GMCF.cc:4824
This class is a derived class of Var. It represents a special general HDF5 product(currently ACOS and...
Definition: HDF5CF.h:382
This class represents one HDF5 dataset(CF variable)
Definition: HDF5CF.h:256
const std::vector< Dimension * > & getDimensions() const
Get the list of the dimensions.
Definition: HDF5CF.h:315
static void Split(const char *s, int len, char sep, std::vector< std::string > &names)
Definition: HDF5CFUtil.cc:343