bes Updated for version 3.20.13
AggregationUtil.cc
1
2// This file is part of the "NcML Module" project, a BES module designed
3// to allow NcML files to be used to be used as a wrapper to add
4// AIS to existing datasets of any format.
5//
6// Copyright (c) 2009 OPeNDAP, Inc.
7// Author: Michael Johnson <m.johnson@opendap.org>
8//
9// For more information, please also see the main website: http://opendap.org/
10//
11// This library is free software; you can redistribute it and/or
12// modify it under the terms of the GNU Lesser General Public
13// License as published by the Free Software Foundation; either
14// version 2.1 of the License, or (at your option) any later version.
15//
16// This library is distributed in the hope that it will be useful,
17// but WITHOUT ANY WARRANTY; without even the implied warranty of
18// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19// Lesser General Public License for more details.
20//
21// You should have received a copy of the GNU Lesser General Public
22// License along with this library; if not, write to the Free Software
23// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24//
25// Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26//
27// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29#include "config.h"
30
31#include "AggregationUtil.h"
32
33// agg_util includes
34#include "AggMemberDataset.h"
35#include "AggregationException.h"
36#include "Dimension.h"
37
38// libdap includes
39#include <libdap/Array.h> // libdap
40#include <libdap/AttrTable.h>
41#include <libdap/BaseType.h>
42#include <libdap/DataDDS.h>
43#include <libdap/DDS.h>
44#include <libdap/Grid.h>
45#include "BESDebug.h"
46#include "BESStopWatch.h"
47
48// Outside includes (MINIMIZE THESE!)
49#include "NCMLDebug.h" // This the ONLY dependency on NCML Module I want in this class since the macros there are general it's ok...
50
51using libdap::Array;
52using libdap::AttrTable;
53using libdap::BaseType;
54using libdap::Constructor;
55using libdap::DataDDS;
56using libdap::DDS;
57using libdap::Grid;
58using libdap::Vector;
59using std::string;
60using std::vector;
61
62namespace agg_util {
63// Static class member used to track the position of the last CVs insertion
64// when building a JoinExisting aggregation.
65int AggregationUtil::d_last_added_cv_position = 0;
66
68// ArrayGetterInterface impls
69
70/* virtual */
71ArrayGetterInterface::~ArrayGetterInterface()
72{
73}
74
76// TopLevelArrayGetter impl
77
78TopLevelArrayGetter::TopLevelArrayGetter() :
79 ArrayGetterInterface()
80{
81}
82
83/* virtual */
84TopLevelArrayGetter::~TopLevelArrayGetter()
85{
86}
87
88/* virtual */
89TopLevelArrayGetter*
90TopLevelArrayGetter::clone() const
91{
92 return new TopLevelArrayGetter(*this);
93}
94
95/* virtual */
96libdap::Array*
97TopLevelArrayGetter::readAndGetArray(const std::string& name, const libdap::DDS& dds,
98 const libdap::Array* const pConstraintTemplate, const std::string& debugChannel) const
99{
100
101 BESStopWatch sw;
102 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("TopLevelArrayGetter::readAndGetArray", "");
103
104 // First, look up the BaseType
105 BaseType* pBT = AggregationUtil::getVariableNoRecurse(dds, name);
106
107 // Next, if it's not there, throw exception.
108 if (!pBT) {
109 throw AggregationException("TopLevelArrayGetter: "
110 "Did not find a variable named \"" + name + "\" at the top-level of the DDS!");
111 }
112
113 // Next, make sure it's an Array before we cast it
114 // Prefer using the enum type for speed rather than RTTI
115 if (pBT->type() != libdap::dods_array_c) {
116 throw AggregationException("TopLevelArrayGetter: "
117 "The top-level DDS variable named \"" + name + "\" was not of the expected type!"
118 " Expected:Array Found:" + pBT->type_name());
119 }
120
121 libdap::Array* pDatasetArray = static_cast<libdap::Array*>(pBT);
122
123 // If given, copy the constraints over to the found Array
124 if (pConstraintTemplate) {
125 agg_util::AggregationUtil::transferArrayConstraints(pDatasetArray, // into this dataset array to be read
126 *pConstraintTemplate, // from this template
127 false, // same rank Array's in template and loaded, don't skip first dim
128 false, // or the to dimension. copy whole thing.
129 !(debugChannel.empty()), // printDebug
130 debugChannel);
131 }
132
133 // Force a read() perhaps with constraints
134 pDatasetArray->set_send_p(true);
135 pDatasetArray->set_in_selection(true);
136 pDatasetArray->read();
137
138 return pDatasetArray;
139}
140
142// TopLevelGridDataArrayGetter impl
143
144TopLevelGridDataArrayGetter::TopLevelGridDataArrayGetter() :
146{
147}
148
149/* virtual */
150TopLevelGridDataArrayGetter::~TopLevelGridDataArrayGetter()
151{
152}
153
154/* virtual */
155TopLevelGridDataArrayGetter*
157{
158 return new TopLevelGridDataArrayGetter(*this);
159}
160
161/* virtual */
162libdap::Array*
163TopLevelGridDataArrayGetter::readAndGetArray(const std::string& name, const libdap::DDS& dds,
164 const libdap::Array* const pConstraintTemplate, const std::string& debugChannel) const
165{
166 BESStopWatch sw;
167 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("TopLevelGridDataArrayGetter::readAndGetArray", "");
168
169 // First, look up the BaseType
170 BaseType* pBT = AggregationUtil::getVariableNoRecurse(dds, name);
171
172 // Next, if it's not there, throw exception.
173 if (!pBT) {
174 throw AggregationException("TopLevelGridArrayGetter: "
175 "Did not find a variable named \"" + name + "\" at the top-level of the DDS!");
176 }
177
178 // Next, make sure it's a Grid before we cast it
179 // Prefer using the enum type for speed rather than RTTI
180 if (pBT->type() != libdap::dods_grid_c) {
181 throw AggregationException("TopLevelGridArrayGetter: "
182 "The top-level DDS variable named \"" + name + "\" was not of the expected type!"
183 " Expected:Grid Found:" + pBT->type_name());
184 }
185
186 // Grab the array and return it.
187 Grid* pDataGrid = static_cast<Grid*>(pBT);
188 Array* pDataArray = static_cast<Array*>(pDataGrid->array_var());
189 if (!pDataArray) {
190 throw AggregationException("TopLevelGridArrayGetter: "
191 "The data Array var for variable name=\"" + name + "\" was unexpectedly null!");
192 }
193
194 // If given, copy the constraints over to the found Array
195 if (pConstraintTemplate) {
196 agg_util::AggregationUtil::transferArrayConstraints(pDataArray, // into this data array to be read
197 *pConstraintTemplate, // from this template
198 false, // same rank Array's in template and loaded, don't skip first dim
199 false, // also don't skip in the to array
200 !(debugChannel.empty()), // printDebug
201 debugChannel);
202 }
203
204 // Force the read() on the Grid level since some handlers
205 // cannot handle a read on a subobject unless read() is called
206 // on the parent object. We have given the constraints to the
207 // data Array already.
208 // TODO make an option on whether to load the Grid's map
209 // vectors or not! I think for these cases we do not want them ever!
210 pDataGrid->set_send_p(true);
211 pDataGrid->set_in_selection(true);
212 pDataGrid->read();
213
214 // Also make sure the Array was read and if not call it as well.
215 if (!pDataArray->read_p()) {
216 pDataArray->set_send_p(true);
217 pDataArray->set_in_selection(true);
218 pDataArray->read();
219 }
220
221 return pDataArray;
222}
223
225// TopLevelGridMapArrayGetter impl
226
227TopLevelGridMapArrayGetter::TopLevelGridMapArrayGetter(const std::string& gridName) :
228 ArrayGetterInterface(), _gridName(gridName)
229{
230}
231
232/* virtual */
233TopLevelGridMapArrayGetter::~TopLevelGridMapArrayGetter()
234{
235}
236
237/* virtual */
238TopLevelGridMapArrayGetter*
240{
241 return new TopLevelGridMapArrayGetter(*this);
242}
243
244/* virtual */
245libdap::Array*
246TopLevelGridMapArrayGetter::readAndGetArray(const std::string& arrayName, const libdap::DDS& dds,
247 const libdap::Array* const pConstraintTemplate, const std::string& debugChannel) const
248{
249
250 BESStopWatch sw;
251 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("TopLevelGridMapArrayGetter::readAndGetArray", "");
252
253 // First, look up the Grid the map is in
254 BaseType* pBT = AggregationUtil::getVariableNoRecurse(dds, _gridName);
255
256 // Next, if it's not there, throw exception.
257 if (!pBT) {
258 throw AggregationException("Did not find a variable named \"" + _gridName + "\" at the top-level of the DDS!");
259 }
260
261 // Next, make sure it's a Grid before we cast it
262 // Prefer using the enum type for speed rather than RTTI
263 if (pBT->type() != libdap::dods_grid_c) {
265 "The top-level DDS variable named \"" + _gridName + "\" was not of the expected type!"
266 " Expected:Grid Found:" + pBT->type_name());
267 }
268
269 // Find the correct map
270 Grid* pDataGrid = static_cast<Grid*>(pBT);
271 Array* pMap = const_cast<Array*>(AggregationUtil::findMapByName(*pDataGrid, arrayName));
272 NCML_ASSERT_MSG(pMap,
273 "Expected to find the map with name " + arrayName + " within the Grid " + _gridName + " but failed to find it!");
274
275 // Prepare it to be read in so we can get the data
276 pMap->set_send_p(true);
277 pMap->set_in_selection(true);
278
279 // If given, copy the constraints over to the found Array
280 if (pConstraintTemplate) {
281 agg_util::AggregationUtil::transferArrayConstraints(pMap, // into this data array to be read
282 *pConstraintTemplate, // from this template
283 false, // same rank Array's in template and loaded, don't skip first dim
284 false, // also don't skip in the to array
285 !(debugChannel.empty()), // printDebug
286 debugChannel);
287 }
288
289 // Do the read
290 pMap->read();
291
292 return pMap;
293}
294
295/*********************************************************************************************************
296 * AggregationUtil Impl
297 */
298void AggregationUtil::performUnionAggregation(DDS* pOutputUnion, const ConstDDSList& datasetsInOrder)
299{
300 VALID_PTR(pOutputUnion);
301
302 // Union any non-aggregated variables from the template dataset into the aggregated dataset
303 // Because we want the joinExistingaggregation to build up the Coordinate Variables (CVs)
304 // in the order they are declared in the NCML file, we need to track the current position
305 // where the last one was inserted. We can do that with a field in the AggregationUtil
306 // class. Here we reset that field so that it starts at position 0. 12.13.11 jhrg
308
309 vector<const DDS*>::const_iterator endIt = datasetsInOrder.end();
310 vector<const DDS*>::const_iterator it;
311 for (it = datasetsInOrder.begin(); it != endIt; ++it) {
312 const DDS* pDDS = *it;
313 VALID_PTR(pDDS);
314
315 // Merge in the global attribute tables
316 unionAttrsInto(&(pOutputUnion->get_attr_table()),
317 // TODO there really should be const version of this in libdap::DDS
318 const_cast<DDS*>(pDDS)->get_attr_table());
319
320 // Merge in the variables, which carry their tables along with them since the AttrTable is
321 // within the variable's "namespace", or lexical scope.
322 unionAllVariablesInto(pOutputUnion, *pDDS);
323 }
324}
325
326void AggregationUtil::unionAttrsInto(AttrTable* pOut, const AttrTable& fromTableIn)
327{
328 // semantically const
329 AttrTable& fromTable = const_cast<AttrTable&>(fromTableIn);
330 AttrTable::Attr_iter endIt = fromTable.attr_end();
331 AttrTable::Attr_iter it;
332 for (it = fromTable.attr_begin(); it != endIt; ++it) {
333 const string& name = fromTable.get_name(it);
334 AttrTable::Attr_iter attrInOut;
335 bool foundIt = findAttribute(*pOut, name, attrInOut);
336 // If it's already in the output, then skip it
337 if (foundIt) {
338 BESDEBUG("ncml",
339 "Union of AttrTable: an attribute named " << name << " already exist in output, skipping it..." << endl);
340 continue;
341 }
342 else // put a copy of it into the output
343 {
344 // containers need deep copy
345 if (fromTable.is_container(it)) {
346 AttrTable* pOrigAttrContainer = fromTable.get_attr_table(it);
347 NCML_ASSERT_MSG(pOrigAttrContainer,
348 "AggregationUtil::mergeAttrTables(): expected non-null AttrTable for the attribute container: "
349 + name);
350 AttrTable* pClonedAttrContainer = new AttrTable(*pOrigAttrContainer);
351 VALID_PTR(pClonedAttrContainer);
352 pOut->append_container(pClonedAttrContainer, name);
353 BESDEBUG("ncml",
354 "Union of AttrTable: adding a deep copy of attribute=" << name << " to the merged output." << endl);
355 }
356 else // for a simple type
357 {
358 string type = fromTable.get_type(it);
359 vector<string>* pAttrTokens = fromTable.get_attr_vector(it);
360 VALID_PTR(pAttrTokens);
361 // append_attr makes a copy of the vector, so we don't have to do so here.
362 pOut->append_attr(name, type, pAttrTokens);
363 }
364 }
365 }
366}
367
368bool AggregationUtil::findAttribute(const AttrTable& inTable, const string& name, AttrTable::Attr_iter& attr)
369{
370 AttrTable& inTableSemanticConst = const_cast<AttrTable&>(inTable);
371 attr = inTableSemanticConst.simple_find(name);
372 return (attr != inTableSemanticConst.attr_end());
373}
374
375void AggregationUtil::unionAllVariablesInto(libdap::DDS* pOutputUnion, const ConstDDSList& datasetsInOrder)
376{
377 ConstDDSList::const_iterator endIt = datasetsInOrder.end();
378 ConstDDSList::const_iterator it;
379 for (it = datasetsInOrder.begin(); it != endIt; ++it) {
380 unionAllVariablesInto(pOutputUnion, *(*it));
381 }
382}
383
384void AggregationUtil::unionAllVariablesInto(libdap::DDS* pOutputUnion, const libdap::DDS& fromDDS, bool add_at_top)
385{
386 DDS& dds = const_cast<DDS&>(fromDDS); // semantically const
387 DDS::Vars_iter endIt = dds.var_end();
388 DDS::Vars_iter it;
389 for (it = dds.var_begin(); it != endIt; ++it) {
390 BaseType* var = *it;
391 if (var) {
392 bool addedVar = addCopyOfVariableIfNameIsAvailable(pOutputUnion, *var, add_at_top);
393 if (addedVar) {
394 BESDEBUG("ncml", "Variable name=" << var->name() << " wasn't in the union yet and was added." << endl);
395 }
396 else {
397 BESDEBUG("ncml",
398 "Variable name=" << var->name() << " was already in the union and was skipped." << endl);
399 }
400 }
401 }
402}
403
404// This method is used to 'initialize' a new JoinExisting aggregation so that
405// A set of Coordinate Variables (CVs) will be inserted _in the order they are
406// listed_ in the .ncml file.
408{
409 //cerr << "Called resetCVInsertionPosition" << endl;
410 d_last_added_cv_position = 0;
411}
412
413bool AggregationUtil::addCopyOfVariableIfNameIsAvailable(libdap::DDS* pOutDDS, const libdap::BaseType& varProto,
414 bool add_at_top)
415{
416 bool ret = false;
417 BaseType* existingVar = findVariableAtDDSTopLevel(*pOutDDS, varProto.name());
418 if (!existingVar) {
419 // Add the var. add_var does a clone, so we don't need to.
420 BESDEBUG("ncml2", "AggregationUtil::addCopyOfVariableIfNameIsAvailable: " << varProto.name() << endl);
421 if (add_at_top) {
422 // This provides a way to remember where the last CV was inserted and adds
423 // this one after it. That provides the behavior that all of the CVs are
424 // added at the beginning of the DDS but in the order they appear in the NCML.
425 // That will translate into a greater chance of success for users, I think ...
426 //
427 // See also similar code in AggregationElement::createAndAddCoordinateVariableForNewDimensio
428 // jhrg 10/17/11
429 //cerr << "d_last_added_cv_position: " << d_last_added_cv_position << endl;
430 DDS::Vars_iter pos = pOutDDS->var_begin() + d_last_added_cv_position;
431
432 pOutDDS->insert_var(pos, const_cast<BaseType*>(&varProto));
433
434 ++d_last_added_cv_position;
435 }
436 else {
437 pOutDDS->add_var(const_cast<BaseType*>(&varProto));
438 }
439
440 ret = true;
441 }
442 return ret;
443}
444
445void AggregationUtil::addOrReplaceVariableForName(libdap::DDS* pOutDDS, const libdap::BaseType& varProto)
446{
447 BaseType* existingVar = findVariableAtDDSTopLevel(*pOutDDS, varProto.name());
448
449 // If exists, nuke it.
450 if (existingVar) {
451 pOutDDS->del_var(varProto.name());
452 }
453
454 // Add the var. add_var does a clone, so we don't need to clone it here.
455 pOutDDS->add_var(const_cast<BaseType*>(&varProto));
456}
457
458libdap::BaseType*
459AggregationUtil::findVariableAtDDSTopLevel(const libdap::DDS& dds_const, const string& name)
460{
461 BaseType* ret = 0;
462 DDS& dds = const_cast<DDS&>(dds_const); // semantically const
463 DDS::Vars_iter endIt = dds.var_end();
464 DDS::Vars_iter it;
465 for (it = dds.var_begin(); it != endIt; ++it) {
466 BaseType* var = *it;
467 if (var && var->name() == name) {
468 ret = var;
469 break;
470 }
471 }
472 return ret;
473}
474
475template<class LibdapType>
476LibdapType*
477AggregationUtil::findTypedVariableAtDDSTopLevel(const libdap::DDS& dds, const string& name)
478{
479 BaseType* pBT = findVariableAtDDSTopLevel(dds, name);
480 if (pBT) {
481 return dynamic_cast<LibdapType*>(pBT);
482 }
483 else {
484 return 0;
485 }
486}
487
488#if 0
489void AggregationUtil::produceOuterDimensionJoinedArray(Array* pJoinedArray, const std::string& joinedArrayName,
490 const std::string& newOuterDimName, const std::vector<libdap::Array*>& fromVars, bool copyData)
491{
492 string funcName = "AggregationUtil::createOuterDimensionJoinedArray:";
493
494 NCML_ASSERT_MSG(fromVars.size() > 0, funcName + "Must be at least one Array in input!");
495
496 // uses the first one as template for type and shape
497 if (!validateArrayTypesAndShapesMatch(fromVars, true)) {
498 throw AggregationException(
499 funcName + " The input arrays must all have the same data type and dimensions but do not!");
500 }
501
502 // The first will be used to "set up" the pJoinedArray
503 Array* templateArray = fromVars[0];
504 VALID_PTR(templateArray);
505 BaseType* templateVar = templateArray->var();
506 NCML_ASSERT_MSG(templateVar, funcName + "Expected a non-NULL prototype BaseType in the first Array!");
507
508 // Set the template var for the type.
509 pJoinedArray->add_var(templateVar);
510 // and force the name to be the desired one, not the prototype's
511 pJoinedArray->set_name(joinedArrayName);
512
513 // Copy the attribute table from the template over... We're not merging or anything.
514 pJoinedArray->set_attr_table(templateArray->get_attr_table());
515
516 // Create a new outer dimension based on the number of inputs we have.
517 // These append_dim calls go left to right, so we need to add the new dim FIRST.
518 pJoinedArray->append_dim(fromVars.size(), newOuterDimName);
519
520 // Use the template to add inner dimensions to the new array
521 for (Array::Dim_iter it = templateArray->dim_begin(); it != templateArray->dim_end(); ++it) {
522 int dimSize = templateArray->dimension_size(it);
523 string dimName = templateArray->dimension_name(it);
524 pJoinedArray->append_dim(dimSize, dimName);
525 }
526
527 if (copyData) {
528 // Make sure we have capacity for the full length of the up-ranked shape.
529 pJoinedArray->reserve_value_capacity(pJoinedArray->length());
530 // Glom the data together in
531 joinArrayData(pJoinedArray, fromVars, false, // we already reserved the space
532 true); // but please clear the Vector buffers after you use each Array in fromVars to help on memory.
533 }
534}
535#endif
536
537bool AggregationUtil::validateArrayTypesAndShapesMatch(const std::vector<libdap::Array*>& arrays,
538 bool enforceMatchingDimNames)
539{
540 NCML_ASSERT(arrays.size() > 0);
541 bool valid = true;
542 Array* pTemplate = 0;
543 for (vector<Array*>::const_iterator it = arrays.begin(); it != arrays.end(); ++it) {
544 // Set the template from the first one.
545 if (!pTemplate) {
546 pTemplate = *it;
547 VALID_PTR(pTemplate);
548 continue;
549 }
550
551 valid = (valid && doTypesMatch(*pTemplate, **it) && doShapesMatch(*pTemplate, **it, enforceMatchingDimNames));
552 // No use continuing
553 if (!valid) {
554 break;
555 }
556 }
557 return valid;
558}
559
560bool AggregationUtil::doTypesMatch(const libdap::Array& lhsC, const libdap::Array& rhsC)
561{
562 // semantically const
563 Array& lhs = const_cast<Array&>(lhsC);
564 Array& rhs = const_cast<Array&>(rhsC);
565 return (lhs.var() && rhs.var() && lhs.var()->type() == rhs.var()->type());
566}
567
568bool AggregationUtil::doShapesMatch(const libdap::Array& lhsC, const libdap::Array& rhsC, bool checkDimNames)
569{
570 // semantically const
571 Array& lhs = const_cast<Array&>(lhsC);
572 Array& rhs = const_cast<Array&>(rhsC);
573
574 // Check the number of dims matches first.
575 bool valid = true;
576 if (lhs.dimensions() != rhs.dimensions()) {
577 valid = false;
578 }
579 else {
580 // Then iterate on both in sync and compare.
581 Array::Dim_iter rhsIt = rhs.dim_begin();
582 for (Array::Dim_iter lhsIt = lhs.dim_begin(); lhsIt != lhs.dim_end(); (++lhsIt, ++rhsIt)) {
583 valid = (valid && (lhs.dimension_size(lhsIt) == rhs.dimension_size(rhsIt)));
584
585 if (checkDimNames) {
586 valid = (valid && (lhs.dimension_name(lhsIt) == rhs.dimension_name(rhsIt)));
587 }
588 }
589 }
590 return valid;
591}
592
593unsigned int AggregationUtil::collectVariableArraysInOrder(std::vector<Array*>& varArrays,
594 const std::string& collectVarName, const ConstDDSList& datasetsInOrder)
595{
596 unsigned int count = 0;
597 ConstDDSList::const_iterator endIt = datasetsInOrder.end();
598 ConstDDSList::const_iterator it;
599 for (it = datasetsInOrder.begin(); it != endIt; ++it) {
600 DDS* pDDS = const_cast<DDS*>(*it);
601 VALID_PTR(pDDS);
602 Array* pVar = dynamic_cast<Array*>(findVariableAtDDSTopLevel(*pDDS, collectVarName));
603 if (pVar) {
604 varArrays.push_back(pVar);
605 count++;
606 }
607 }
608 return count;
609}
610
612{
613 Array* pArr = dynamic_cast<Array*>(pBT);
614 if (pArr && (pArr->dimensions() == 1)) {
615 // only one dimension, so grab the first and make sure we only got one.
616 Array::Dim_iter it = pArr->dim_begin();
617 bool matches = (pArr->dimension_name(it) == pArr->name());
618 NCML_ASSERT_MSG((++it == pArr->dim_end()),
619 "Logic error: NCMLUtil::isCoordinateVariable(): expected one dimension from Array, but got more!");
620 return matches;
621 }
622 else {
623 return false;
624 }
625}
626
627#if 0
628void AggregationUtil::joinArrayData(Array* pAggArray, const std::vector<Array*>& varArrays,
629 bool reserveStorage/*=true*/, bool clearDataAfterUse/*=false*/)
630{
631 // Make sure we get a pAggArray with a type var we can deal with.
632 VALID_PTR(pAggArray->var());
633 NCML_ASSERT_MSG(pAggArray->var()->is_simple_type(),
634 "AggregationUtil::joinArrayData: the output Array is not of a simple type! Can't aggregate!");
635
636 // If the caller wants us to do it, sum up length() and reserve that much.
637 if (reserveStorage) {
638 // Figure it how much we need...
639 unsigned int totalLength = 0;
640 {
641 vector<Array*>::const_iterator it;
642 vector<Array*>::const_iterator endIt = varArrays.end();
643 for (it = varArrays.begin(); it != endIt; ++it) {
644 Array* pArr = *it;
645 if (pArr) {
646 totalLength += pArr->length();
647 }
648 }
649 }
650 pAggArray->reserve_value_capacity(totalLength);
651 }
652
653 // For each Array, make sure it's read in and copy its data into the output.
654 unsigned int nextElt = 0; // keeps track of where we are to write next in the output
655 vector<Array*>::const_iterator it;
656 vector<Array*>::const_iterator endIt = varArrays.end();
657 for (it = varArrays.begin(); it != endIt; ++it) {
658 Array* pArr = *it;
659 VALID_PTR(pArr);
660 NCML_ASSERT_MSG(pArr->var() && (pArr->var()->type() == pAggArray->var()->type()),
661 "AggregationUtil::joinArrayData: one of the arrays to join has different type than output! Can't aggregate!");
662
663 // Make sure it was read in...
664 if (!pArr->read_p()) {
665 pArr->read();
666 }
667
668 // Copy it in with the Vector call and update our location
669 nextElt += pAggArray->set_value_slice_from_row_major_vector(*pArr, nextElt);
670
671 if (clearDataAfterUse) {
672 pArr->clear_local_data();
673 }
674 }
675
676 // That's all folks!
677}
678#endif
679
681// struct dimension
682// {
683// int size; ///< The unconstrained dimension size.
684// string name; ///< The name of this dimension.
685// int start; ///< The constraint start index
686// int stop; ///< The constraint end index
687// int stride; ///< The constraint stride
688// int c_size; ///< Size of dimension once constrained
689// };
690
692void AggregationUtil::printDimensions(std::ostream& os, const libdap::Array& fromArray)
693{
694 os << "Array dimensions: " << endl;
695 Array& theArray = const_cast<Array&>(fromArray);
696 Array::Dim_iter it;
697 Array::Dim_iter endIt = theArray.dim_end();
698 for (it = theArray.dim_begin(); it != endIt; ++it) {
699 Array::dimension d = *it;
700 os << "Dim = {" << endl;
701 os << "name=" << d.name << endl;
702 os << "size=" << d.size << endl;
703 os << " }" << endl;
704 }
705 os << "End Array dimensions." << endl;
706}
707
708void AggregationUtil::printConstraints(std::ostream& os, const Array& rcArray)
709{
710 os << "Array constraints: " << endl;
711 Array& theArray = const_cast<Array&>(rcArray);
712 Array::Dim_iter it;
713 Array::Dim_iter endIt = theArray.dim_end();
714 for (it = theArray.dim_begin(); it != endIt; ++it) {
715 Array::dimension d = *it;
716 os << "Dim = {" << endl;
717 os << "name=" << d.name << endl;
718 os << "start=" << d.start << endl;
719 os << "stride=" << d.stride << endl;
720 os << "stop=" << d.stop << endl;
721 os << " }" << endl;
722 }
723 os << "End Array constraints" << endl;
724}
725
726void AggregationUtil::printConstraintsToDebugChannel(const std::string& debugChannel, const libdap::Array& fromArray)
727{
728 ostringstream oss;
729 BESDEBUG(debugChannel, "Printing constraints for Array: " << fromArray.name() << ": " << oss.str() << endl);
730 AggregationUtil::printConstraints(oss, fromArray);
731 BESDEBUG(debugChannel, oss.str() << endl);
732}
733
734void AggregationUtil::transferArrayConstraints(Array* pToArray, const Array& fromArrayConst, bool skipFirstFromDim,
735 bool skipFirstToDim, bool printDebug /* = false */, const std::string& debugChannel /* = "agg_util" */)
736{
737 VALID_PTR(pToArray);
738 Array& fromArray = const_cast<Array&>(fromArrayConst);
739
740 // Make sure there's no constraints on output. Shouldn't be, but...
741 pToArray->reset_constraint();
742
743 // Ensure the dimensionalities will work out.
744 int skipDelta = ((skipFirstFromDim) ? (1) : (0));
745 // If skipping output as well, subtract out the delta.
746 // If we go negative, also an error.
747 if (skipFirstToDim) {
748 skipDelta -= 1;
749 }
750 if (skipDelta < 0 || (pToArray->dimensions() + skipDelta != const_cast<Array&>(fromArrayConst).dimensions())) {
751 throw AggregationException("AggregationUtil::transferArrayConstraints: "
752 "Mismatched dimensionalities!");
753 }
754
755 if (printDebug) {
756 BESDEBUG(debugChannel,
757 "Printing constraints on fromArray name= " << fromArray.name() << " before transfer..." << endl);
758 printConstraintsToDebugChannel(debugChannel, fromArray);
759 }
760
761 // Only real way to the constraints is with the iterator,
762 // so we'll iterator on the fromArray and move
763 // to toarray iterator in sync.
764 Array::Dim_iter fromArrIt = fromArray.dim_begin();
765 Array::Dim_iter fromArrEndIt = fromArray.dim_end();
766 Array::Dim_iter toArrIt = pToArray->dim_begin();
767 for (; fromArrIt != fromArrEndIt; ++fromArrIt) {
768 if (skipFirstFromDim && (fromArrIt == fromArray.dim_begin())) {
769 // If we skip first to array as well, increment
770 // before the next call.
771 if (skipFirstToDim) {
772 ++toArrIt;
773 }
774 continue;
775 }
776// If aggregates with renaming dimensions do not match each other. SK 07/26/18
777// NCML_ASSERT_MSG(fromArrIt->name == toArrIt->name, "GAggregationUtil::transferArrayConstraints: "
778// "Expected the dimensions to have the same name but they did not.");
779 pToArray->add_constraint(toArrIt, fromArrIt->start, fromArrIt->stride, fromArrIt->stop);
780 ++toArrIt;
781 }
782
783 if (printDebug) {
784 BESDEBUG(debugChannel, "Printing constrains on pToArray after transfer..." << endl);
785 printConstraintsToDebugChannel(debugChannel, *pToArray);
786 }
787}
788
789BaseType*
790AggregationUtil::getVariableNoRecurse(const libdap::DDS& ddsConst, const std::string& name)
791{
792 BaseType* ret = 0;
793 DDS& dds = const_cast<DDS&>(ddsConst); // semantically const
794 DDS::Vars_iter endIt = dds.var_end();
795 DDS::Vars_iter it;
796 for (it = dds.var_begin(); it != endIt; ++it) {
797 BaseType* var = *it;
798 if (var && var->name() == name) {
799 ret = var;
800 break;
801 }
802 }
803 return ret;
804}
805
806// Ugh cut and pasted from the other...
807// TODO REFACTOR DDS and Constructor really need a common abstract interface,
808// like IVariableContainer that declares the iterators and associated methods.
809BaseType*
810AggregationUtil::getVariableNoRecurse(const libdap::Constructor& varContainerConst, const std::string& name)
811{
812 BaseType* ret = 0;
813 Constructor& varContainer = const_cast<Constructor&>(varContainerConst); // semantically const
814 Constructor::Vars_iter endIt = varContainer.var_end();
815 Constructor::Vars_iter it;
816 for (it = varContainer.var_begin(); it != endIt; ++it) {
817 BaseType* var = *it;
818 if (var && var->name() == name) {
819 ret = var;
820 break;
821 }
822 }
823 return ret;
824}
825
826/*static*/
827Array*
829{
830 if (!pBT) {
831 return 0;
832 }
833
834 // After switch():
835 // if Array, will be cast to Array.
836 // if Grid, will be cast data Array member of Grid.
837 // Other types, will be null.
838 libdap::Array* pArray(0);
839 switch (pBT->type()) {
840 case libdap::dods_array_c:
841 pArray = static_cast<libdap::Array*>(pBT);
842 break;
843
844 case libdap::dods_grid_c:
845 pArray = static_cast<Grid*>(pBT)->get_array();
846 break;
847
848 default:
849 pArray = 0;
850 break;
851 }
852 return pArray;
853}
854
855const Array*
856AggregationUtil::findMapByName(const libdap::Grid& inGrid, const string& findName)
857{
858 Grid& grid = const_cast<Grid&>(inGrid);
859 Array* pRet = 0;
860 Grid::Map_iter it;
861 Grid::Map_iter endIt = grid.map_end();
862 for (it = grid.map_begin(); it != endIt; ++it) {
863 if ((*it)->name() == findName) {
864 pRet = static_cast<Array*>(*it);
865 break;
866 }
867 }
868 return pRet;
869}
870
871Array* AggregationUtil::readDatasetArrayDataForAggregation(const Array& constrainedTemplateArray,
872 const std::string& varName, AggMemberDataset& dataset, const ArrayGetterInterface& arrayGetter,
873 const std::string& debugChannel)
874{
875 BESStopWatch sw;
876 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("AggregationUtil::readDatasetArrayDataForAggregation", "");
877
878 const libdap::DDS* pDDS = dataset.getDDS();
879 NCML_ASSERT_MSG(pDDS, "GridAggregateOnOuterDimension::read(): Got a null DataDDS "
880 "while loading dataset = " + dataset.getLocation());
881
882 // Grab the Array from the DataDDS with the getter
883 Array* pDatasetArray = arrayGetter.readAndGetArray(varName, *pDDS, &constrainedTemplateArray, debugChannel);
884 NCML_ASSERT_MSG(pDatasetArray, "In aggregation member dataset, failed to get the array! "
885 "Dataset location = " + dataset.getLocation());
886
887 // Make sure that the data was read in or I dunno what went on.
888 if (!pDatasetArray->read_p()) {
889 NCML_ASSERT_MSG(pDatasetArray->read_p(),
890 "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: pDatasetArray was not read_p()!");
891 }
892
893 // Make sure it matches the prototype or somthing went wrong
894 if (!AggregationUtil::doTypesMatch(constrainedTemplateArray, *pDatasetArray)) {
896 "Invalid aggregation! "
897 "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
898 "We found the aggregation variable name=" + varName
899 + " but it was not of the same type as the prototype variable!");
900 }
901
902 // Make sure the subshapes match! (true means check dimension names too... debate this)
903 if (!AggregationUtil::doShapesMatch(constrainedTemplateArray, *pDatasetArray, true)) {
905 "Invalid aggregation! "
906 "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
907 "We found the aggregation variable name=" + varName
908 + " but it was not of the same shape as the prototype!");
909 }
910
911 // Make sure the length of the data array also matches the proto
912 if (constrainedTemplateArray.length() != pDatasetArray->length()) {
913 NCML_ASSERT_MSG(constrainedTemplateArray.length() == pDatasetArray->length(),
914 "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
915 "The prototype array and the loaded dataset array length()'s were not equal, even "
916 "though their shapes matched. Logic problem.");
917 }
918
919 return pDatasetArray;
920}
921
922void AggregationUtil::addDatasetArrayDataToAggregationOutputArray(libdap::Array& oOutputArray, unsigned int atIndex,
923 const Array& constrainedTemplateArray, const std::string& varName, AggMemberDataset& dataset,
924 const ArrayGetterInterface& arrayGetter, const std::string& debugChannel)
925{
926 BESStopWatch sw;
927 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("AggregationUtil::addDatasetArrayDataToAggregationOutputArray", "");
928
929 libdap::Array* pDatasetArray = readDatasetArrayDataForAggregation(constrainedTemplateArray, varName, dataset, arrayGetter,
930 debugChannel);
931
932 // FINALLY, we get to stream the data!
933 oOutputArray.set_value_slice_from_row_major_vector(*pDatasetArray, atIndex);
934
935 // Now that we have copied it - let the memory go! Free! Let the bytes be freed! - ndp 08/12/2015
936 pDatasetArray->clear_local_data();
937}
938
939void AggregationUtil::gatherMetadataChangesFrom(BaseType* pIntoVar, const BaseType& fromVarC)
940{
941 BaseType& fromVar = const_cast<BaseType&>(fromVarC); //semantic const
942 // The output will end up here.
943 AttrTable finalAT;
944
945 // First, seed it with the changes in the fromVar.
946 unionAttrsInto(&finalAT, fromVar.get_attr_table());
947
948 // Then union in the items from the to var
949 unionAttrsInto(&finalAT, pIntoVar->get_attr_table());
950
951 // HACK BUG In the set_attr_table call through AttrTable operator=
952 // means we keep bad memory around. Workaround until fixed!
953 pIntoVar->get_attr_table().erase();
954
955 // Finally, replace the output var's table with the constructed one!
956 pIntoVar->set_attr_table(finalAT);
957}
958
959} // namespace agg_util
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
virtual const libdap::DDS * getDDS()=0
const std::string & getLocation() const
static void addDatasetArrayDataToAggregationOutputArray(libdap::Array &oOutputArray, unsigned int atIndex, const libdap::Array &constrainedTemplateArray, const string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const string &debugChannel)
static void printConstraintsToDebugChannel(const std::string &debugChannel, const libdap::Array &fromArray)
static void gatherMetadataChangesFrom(libdap::BaseType *pIntoVar, const libdap::BaseType &fromVar)
static LibdapType * findTypedVariableAtDDSTopLevel(const libdap::DDS &dds, const string &name)
static bool doTypesMatch(const libdap::Array &lhs, const libdap::Array &rhs)
static unsigned int collectVariableArraysInOrder(std::vector< libdap::Array * > &varArrays, const std::string &collectVarName, const ConstDDSList &datasetsInOrder)
static void transferArrayConstraints(libdap::Array *pToArray, const libdap::Array &fromArray, bool skipFirstFromDim, bool skipFirstToDim, bool printDebug=false, const std::string &debugChannel="agg_util")
static void printDimensions(std::ostream &os, const libdap::Array &fromArray)
static libdap::Array * readDatasetArrayDataForAggregation(const libdap::Array &constrainedTemplateArray, const std::string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const std::string &debugChannel)
static void performUnionAggregation(libdap::DDS *pOutputUnion, const ConstDDSList &datasetsInOrder)
static libdap::BaseType * getVariableNoRecurse(const libdap::DDS &dds, const std::string &name)
static void unionAllVariablesInto(libdap::DDS *pOutputUnion, const ConstDDSList &datasetsInOrder)
static libdap::Array * getAsArrayIfPossible(libdap::BaseType *pBT)
static bool couldBeCoordinateVariable(libdap::BaseType *pBT)
static const libdap::Array * findMapByName(const libdap::Grid &inGrid, const std::string &findName)
static bool addCopyOfVariableIfNameIsAvailable(libdap::DDS *pOutDDS, const libdap::BaseType &varProto, bool add_at_top=false)
static void addOrReplaceVariableForName(libdap::DDS *pOutDDS, const libdap::BaseType &varProto)
static void unionAttrsInto(libdap::AttrTable *pOut, const libdap::AttrTable &fromTable)
static bool doShapesMatch(const libdap::Array &lhs, const libdap::Array &rhs, bool checkDimNames)
static bool findAttribute(const libdap::AttrTable &inTable, const string &name, libdap::AttrTable::Attr_iter &attr)
static void resetCVInsertionPosition()
static bool validateArrayTypesAndShapesMatch(const std::vector< libdap::Array * > &arrays, bool enforceMatchingDimNames)
static void printConstraints(std::ostream &os, const libdap::Array &fromArray)
static libdap::BaseType * findVariableAtDDSTopLevel(const libdap::DDS &dds, const string &name)
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
virtual libdap::Array * readAndGetArray(const std::string &name, const libdap::DDS &dds, const libdap::Array *const pConstraintTemplate, const std::string &debugChannel) const =0
virtual libdap::Array * readAndGetArray(const std::string &name, const libdap::DDS &dds, const libdap::Array *const pConstraintTemplate, const std::string &debugChannel) const
virtual TopLevelGridDataArrayGetter * clone() const
virtual TopLevelGridMapArrayGetter * clone() const
virtual libdap::Array * readAndGetArray(const std::string &name, const libdap::DDS &dds, const libdap::Array *const pConstraintTemplate, const std::string &debugChannel) const