31#include "AggregationUtil.h"
34#include "AggMemberDataset.h"
35#include "AggregationException.h"
39#include <libdap/Array.h>
40#include <libdap/AttrTable.h>
41#include <libdap/BaseType.h>
42#include <libdap/DataDDS.h>
43#include <libdap/DDS.h>
44#include <libdap/Grid.h>
46#include "BESStopWatch.h"
52using libdap::AttrTable;
53using libdap::BaseType;
54using libdap::Constructor;
65int AggregationUtil::d_last_added_cv_position = 0;
71ArrayGetterInterface::~ArrayGetterInterface()
78TopLevelArrayGetter::TopLevelArrayGetter() :
79 ArrayGetterInterface()
84TopLevelArrayGetter::~TopLevelArrayGetter()
90TopLevelArrayGetter::clone()
const
97TopLevelArrayGetter::readAndGetArray(
const std::string& name,
const libdap::DDS& dds,
98 const libdap::Array*
const pConstraintTemplate,
const std::string& debugChannel)
const
105 BaseType* pBT = AggregationUtil::getVariableNoRecurse(dds, name);
110 "Did not find a variable named \"" + name +
"\" at the top-level of the DDS!");
115 if (pBT->type() != libdap::dods_array_c) {
117 "The top-level DDS variable named \"" + name +
"\" was not of the expected type!"
118 " Expected:Array Found:" + pBT->type_name());
121 libdap::Array* pDatasetArray =
static_cast<libdap::Array*
>(pBT);
124 if (pConstraintTemplate) {
126 *pConstraintTemplate,
129 !(debugChannel.empty()),
134 pDatasetArray->set_send_p(
true);
135 pDatasetArray->set_in_selection(
true);
136 pDatasetArray->read();
138 return pDatasetArray;
144TopLevelGridDataArrayGetter::TopLevelGridDataArrayGetter() :
150TopLevelGridDataArrayGetter::~TopLevelGridDataArrayGetter()
155TopLevelGridDataArrayGetter*
164 const libdap::Array*
const pConstraintTemplate,
const std::string& debugChannel)
const
167 if (
BESDebug::IsSet(TIMING_LOG_KEY)) sw.
start(
"TopLevelGridDataArrayGetter::readAndGetArray",
"");
175 "Did not find a variable named \"" + name +
"\" at the top-level of the DDS!");
180 if (pBT->type() != libdap::dods_grid_c) {
182 "The top-level DDS variable named \"" + name +
"\" was not of the expected type!"
183 " Expected:Grid Found:" + pBT->type_name());
187 Grid* pDataGrid =
static_cast<Grid*
>(pBT);
188 Array* pDataArray =
static_cast<Array*
>(pDataGrid->array_var());
191 "The data Array var for variable name=\"" + name +
"\" was unexpectedly null!");
195 if (pConstraintTemplate) {
197 *pConstraintTemplate,
200 !(debugChannel.empty()),
210 pDataGrid->set_send_p(
true);
211 pDataGrid->set_in_selection(
true);
215 if (!pDataArray->read_p()) {
216 pDataArray->set_send_p(
true);
217 pDataArray->set_in_selection(
true);
227TopLevelGridMapArrayGetter::TopLevelGridMapArrayGetter(
const std::string& gridName) :
233TopLevelGridMapArrayGetter::~TopLevelGridMapArrayGetter()
238TopLevelGridMapArrayGetter*
247 const libdap::Array*
const pConstraintTemplate,
const std::string& debugChannel)
const
251 if (
BESDebug::IsSet(TIMING_LOG_KEY)) sw.
start(
"TopLevelGridMapArrayGetter::readAndGetArray",
"");
258 throw AggregationException(
"Did not find a variable named \"" + _gridName +
"\" at the top-level of the DDS!");
263 if (pBT->type() != libdap::dods_grid_c) {
265 "The top-level DDS variable named \"" + _gridName +
"\" was not of the expected type!"
266 " Expected:Grid Found:" + pBT->type_name());
270 Grid* pDataGrid =
static_cast<Grid*
>(pBT);
272 NCML_ASSERT_MSG(pMap,
273 "Expected to find the map with name " + arrayName +
" within the Grid " + _gridName +
" but failed to find it!");
276 pMap->set_send_p(
true);
277 pMap->set_in_selection(
true);
280 if (pConstraintTemplate) {
282 *pConstraintTemplate,
285 !(debugChannel.empty()),
300 VALID_PTR(pOutputUnion);
309 vector<const DDS*>::const_iterator endIt = datasetsInOrder.end();
310 vector<const DDS*>::const_iterator it;
311 for (it = datasetsInOrder.begin(); it != endIt; ++it) {
312 const DDS* pDDS = *it;
318 const_cast<DDS*
>(pDDS)->get_attr_table());
329 AttrTable& fromTable =
const_cast<AttrTable&
>(fromTableIn);
330 AttrTable::Attr_iter endIt = fromTable.attr_end();
331 AttrTable::Attr_iter it;
332 for (it = fromTable.attr_begin(); it != endIt; ++it) {
333 const string& name = fromTable.get_name(it);
334 AttrTable::Attr_iter attrInOut;
339 "Union of AttrTable: an attribute named " << name <<
" already exist in output, skipping it..." << endl);
345 if (fromTable.is_container(it)) {
346 AttrTable* pOrigAttrContainer = fromTable.get_attr_table(it);
347 NCML_ASSERT_MSG(pOrigAttrContainer,
348 "AggregationUtil::mergeAttrTables(): expected non-null AttrTable for the attribute container: "
350 AttrTable* pClonedAttrContainer =
new AttrTable(*pOrigAttrContainer);
351 VALID_PTR(pClonedAttrContainer);
352 pOut->append_container(pClonedAttrContainer, name);
354 "Union of AttrTable: adding a deep copy of attribute=" << name <<
" to the merged output." << endl);
358 string type = fromTable.get_type(it);
359 vector<string>* pAttrTokens = fromTable.get_attr_vector(it);
360 VALID_PTR(pAttrTokens);
362 pOut->append_attr(name, type, pAttrTokens);
370 AttrTable& inTableSemanticConst =
const_cast<AttrTable&
>(inTable);
371 attr = inTableSemanticConst.simple_find(name);
372 return (attr != inTableSemanticConst.attr_end());
377 ConstDDSList::const_iterator endIt = datasetsInOrder.end();
378 ConstDDSList::const_iterator it;
379 for (it = datasetsInOrder.begin(); it != endIt; ++it) {
386 DDS& dds =
const_cast<DDS&
>(fromDDS);
387 DDS::Vars_iter endIt = dds.var_end();
389 for (it = dds.var_begin(); it != endIt; ++it) {
394 BESDEBUG(
"ncml",
"Variable name=" << var->name() <<
" wasn't in the union yet and was added." << endl);
398 "Variable name=" << var->name() <<
" was already in the union and was skipped." << endl);
410 d_last_added_cv_position = 0;
420 BESDEBUG(
"ncml2",
"AggregationUtil::addCopyOfVariableIfNameIsAvailable: " << varProto.name() << endl);
430 DDS::Vars_iter pos = pOutDDS->var_begin() + d_last_added_cv_position;
432 pOutDDS->insert_var(pos,
const_cast<BaseType*
>(&varProto));
434 ++d_last_added_cv_position;
437 pOutDDS->add_var(
const_cast<BaseType*
>(&varProto));
451 pOutDDS->del_var(varProto.name());
455 pOutDDS->add_var(
const_cast<BaseType*
>(&varProto));
462 DDS& dds =
const_cast<DDS&
>(dds_const);
463 DDS::Vars_iter endIt = dds.var_end();
465 for (it = dds.var_begin(); it != endIt; ++it) {
467 if (var && var->name() == name) {
475template<
class LibdapType>
481 return dynamic_cast<LibdapType*
>(pBT);
489void AggregationUtil::produceOuterDimensionJoinedArray(Array* pJoinedArray,
const std::string& joinedArrayName,
490 const std::string& newOuterDimName,
const std::vector<libdap::Array*>& fromVars,
bool copyData)
492 string funcName =
"AggregationUtil::createOuterDimensionJoinedArray:";
494 NCML_ASSERT_MSG(fromVars.size() > 0, funcName +
"Must be at least one Array in input!");
498 throw AggregationException(
499 funcName +
" The input arrays must all have the same data type and dimensions but do not!");
503 Array* templateArray = fromVars[0];
504 VALID_PTR(templateArray);
505 BaseType* templateVar = templateArray->var();
506 NCML_ASSERT_MSG(templateVar, funcName +
"Expected a non-NULL prototype BaseType in the first Array!");
509 pJoinedArray->add_var(templateVar);
511 pJoinedArray->set_name(joinedArrayName);
514 pJoinedArray->set_attr_table(templateArray->get_attr_table());
518 pJoinedArray->append_dim(fromVars.size(), newOuterDimName);
521 for (Array::Dim_iter it = templateArray->dim_begin(); it != templateArray->dim_end(); ++it) {
522 int dimSize = templateArray->dimension_size(it);
523 string dimName = templateArray->dimension_name(it);
524 pJoinedArray->append_dim(dimSize, dimName);
529 pJoinedArray->reserve_value_capacity(pJoinedArray->length());
531 joinArrayData(pJoinedArray, fromVars,
false,
538 bool enforceMatchingDimNames)
540 NCML_ASSERT(arrays.size() > 0);
542 Array* pTemplate = 0;
543 for (vector<Array*>::const_iterator it = arrays.begin(); it != arrays.end(); ++it) {
547 VALID_PTR(pTemplate);
563 Array& lhs =
const_cast<Array&
>(lhsC);
564 Array& rhs =
const_cast<Array&
>(rhsC);
565 return (lhs.var() && rhs.var() && lhs.var()->type() == rhs.var()->type());
571 Array& lhs =
const_cast<Array&
>(lhsC);
572 Array& rhs =
const_cast<Array&
>(rhsC);
576 if (lhs.dimensions() != rhs.dimensions()) {
581 Array::Dim_iter rhsIt = rhs.dim_begin();
582 for (Array::Dim_iter lhsIt = lhs.dim_begin(); lhsIt != lhs.dim_end(); (++lhsIt, ++rhsIt)) {
583 valid = (valid && (lhs.dimension_size(lhsIt) == rhs.dimension_size(rhsIt)));
586 valid = (valid && (lhs.dimension_name(lhsIt) == rhs.dimension_name(rhsIt)));
594 const std::string& collectVarName,
const ConstDDSList& datasetsInOrder)
596 unsigned int count = 0;
597 ConstDDSList::const_iterator endIt = datasetsInOrder.end();
598 ConstDDSList::const_iterator it;
599 for (it = datasetsInOrder.begin(); it != endIt; ++it) {
600 DDS* pDDS =
const_cast<DDS*
>(*it);
604 varArrays.push_back(pVar);
613 Array* pArr =
dynamic_cast<Array*
>(pBT);
614 if (pArr && (pArr->dimensions() == 1)) {
616 Array::Dim_iter it = pArr->dim_begin();
617 bool matches = (pArr->dimension_name(it) == pArr->name());
618 NCML_ASSERT_MSG((++it == pArr->dim_end()),
619 "Logic error: NCMLUtil::isCoordinateVariable(): expected one dimension from Array, but got more!");
628void AggregationUtil::joinArrayData(Array* pAggArray,
const std::vector<Array*>& varArrays,
629 bool reserveStorage,
bool clearDataAfterUse)
632 VALID_PTR(pAggArray->var());
633 NCML_ASSERT_MSG(pAggArray->var()->is_simple_type(),
634 "AggregationUtil::joinArrayData: the output Array is not of a simple type! Can't aggregate!");
637 if (reserveStorage) {
639 unsigned int totalLength = 0;
641 vector<Array*>::const_iterator it;
642 vector<Array*>::const_iterator endIt = varArrays.end();
643 for (it = varArrays.begin(); it != endIt; ++it) {
646 totalLength += pArr->length();
650 pAggArray->reserve_value_capacity(totalLength);
654 unsigned int nextElt = 0;
655 vector<Array*>::const_iterator it;
656 vector<Array*>::const_iterator endIt = varArrays.end();
657 for (it = varArrays.begin(); it != endIt; ++it) {
660 NCML_ASSERT_MSG(pArr->var() && (pArr->var()->type() == pAggArray->var()->type()),
661 "AggregationUtil::joinArrayData: one of the arrays to join has different type than output! Can't aggregate!");
664 if (!pArr->read_p()) {
669 nextElt += pAggArray->set_value_slice_from_row_major_vector(*pArr, nextElt);
671 if (clearDataAfterUse) {
672 pArr->clear_local_data();
694 os <<
"Array dimensions: " << endl;
695 Array& theArray =
const_cast<Array&
>(fromArray);
697 Array::Dim_iter endIt = theArray.dim_end();
698 for (it = theArray.dim_begin(); it != endIt; ++it) {
699 Array::dimension d = *it;
700 os <<
"Dim = {" << endl;
701 os <<
"name=" << d.name << endl;
702 os <<
"size=" << d.size << endl;
705 os <<
"End Array dimensions." << endl;
710 os <<
"Array constraints: " << endl;
711 Array& theArray =
const_cast<Array&
>(rcArray);
713 Array::Dim_iter endIt = theArray.dim_end();
714 for (it = theArray.dim_begin(); it != endIt; ++it) {
715 Array::dimension d = *it;
716 os <<
"Dim = {" << endl;
717 os <<
"name=" << d.name << endl;
718 os <<
"start=" << d.start << endl;
719 os <<
"stride=" << d.stride << endl;
720 os <<
"stop=" << d.stop << endl;
723 os <<
"End Array constraints" << endl;
729 BESDEBUG(debugChannel,
"Printing constraints for Array: " << fromArray.name() <<
": " << oss.str() << endl);
731 BESDEBUG(debugChannel, oss.str() << endl);
735 bool skipFirstToDim,
bool printDebug ,
const std::string& debugChannel )
738 Array& fromArray =
const_cast<Array&
>(fromArrayConst);
741 pToArray->reset_constraint();
744 int skipDelta = ((skipFirstFromDim) ? (1) : (0));
747 if (skipFirstToDim) {
750 if (skipDelta < 0 || (pToArray->dimensions() + skipDelta !=
const_cast<Array&
>(fromArrayConst).dimensions())) {
752 "Mismatched dimensionalities!");
756 BESDEBUG(debugChannel,
757 "Printing constraints on fromArray name= " << fromArray.name() <<
" before transfer..." << endl);
764 Array::Dim_iter fromArrIt = fromArray.dim_begin();
765 Array::Dim_iter fromArrEndIt = fromArray.dim_end();
766 Array::Dim_iter toArrIt = pToArray->dim_begin();
767 for (; fromArrIt != fromArrEndIt; ++fromArrIt) {
768 if (skipFirstFromDim && (fromArrIt == fromArray.dim_begin())) {
771 if (skipFirstToDim) {
779 pToArray->add_constraint(toArrIt, fromArrIt->start, fromArrIt->stride, fromArrIt->stop);
784 BESDEBUG(debugChannel,
"Printing constrains on pToArray after transfer..." << endl);
793 DDS& dds =
const_cast<DDS&
>(ddsConst);
794 DDS::Vars_iter endIt = dds.var_end();
796 for (it = dds.var_begin(); it != endIt; ++it) {
798 if (var && var->name() == name) {
813 Constructor& varContainer =
const_cast<Constructor&
>(varContainerConst);
814 Constructor::Vars_iter endIt = varContainer.var_end();
815 Constructor::Vars_iter it;
816 for (it = varContainer.var_begin(); it != endIt; ++it) {
818 if (var && var->name() == name) {
838 libdap::Array* pArray(0);
839 switch (pBT->type()) {
840 case libdap::dods_array_c:
841 pArray =
static_cast<libdap::Array*
>(pBT);
844 case libdap::dods_grid_c:
845 pArray =
static_cast<Grid*
>(pBT)->get_array();
858 Grid& grid =
const_cast<Grid&
>(inGrid);
861 Grid::Map_iter endIt = grid.map_end();
862 for (it = grid.map_begin(); it != endIt; ++it) {
863 if ((*it)->name() == findName) {
864 pRet =
static_cast<Array*
>(*it);
873 const std::string& debugChannel)
876 if (
BESDebug::IsSet(TIMING_LOG_KEY)) sw.
start(
"AggregationUtil::readDatasetArrayDataForAggregation",
"");
878 const libdap::DDS* pDDS = dataset.
getDDS();
879 NCML_ASSERT_MSG(pDDS,
"GridAggregateOnOuterDimension::read(): Got a null DataDDS "
880 "while loading dataset = " + dataset.
getLocation());
883 Array* pDatasetArray = arrayGetter.
readAndGetArray(varName, *pDDS, &constrainedTemplateArray, debugChannel);
884 NCML_ASSERT_MSG(pDatasetArray,
"In aggregation member dataset, failed to get the array! "
888 if (!pDatasetArray->read_p()) {
889 NCML_ASSERT_MSG(pDatasetArray->read_p(),
890 "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: pDatasetArray was not read_p()!");
896 "Invalid aggregation! "
897 "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
898 "We found the aggregation variable name=" + varName
899 +
" but it was not of the same type as the prototype variable!");
905 "Invalid aggregation! "
906 "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
907 "We found the aggregation variable name=" + varName
908 +
" but it was not of the same shape as the prototype!");
912 if (constrainedTemplateArray.length() != pDatasetArray->length()) {
913 NCML_ASSERT_MSG(constrainedTemplateArray.length() == pDatasetArray->length(),
914 "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
915 "The prototype array and the loaded dataset array length()'s were not equal, even "
916 "though their shapes matched. Logic problem.");
919 return pDatasetArray;
923 const Array& constrainedTemplateArray,
const std::string& varName,
AggMemberDataset& dataset,
927 if (
BESDebug::IsSet(TIMING_LOG_KEY)) sw.
start(
"AggregationUtil::addDatasetArrayDataToAggregationOutputArray",
"");
933 oOutputArray.set_value_slice_from_row_major_vector(*pDatasetArray, atIndex);
936 pDatasetArray->clear_local_data();
941 BaseType& fromVar =
const_cast<BaseType&
>(fromVarC);
953 pIntoVar->get_attr_table().erase();
956 pIntoVar->set_attr_table(finalAT);
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
virtual bool start(std::string name)
virtual const libdap::DDS * getDDS()=0
const std::string & getLocation() const
static void addDatasetArrayDataToAggregationOutputArray(libdap::Array &oOutputArray, unsigned int atIndex, const libdap::Array &constrainedTemplateArray, const string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const string &debugChannel)
static void printConstraintsToDebugChannel(const std::string &debugChannel, const libdap::Array &fromArray)
static void gatherMetadataChangesFrom(libdap::BaseType *pIntoVar, const libdap::BaseType &fromVar)
static LibdapType * findTypedVariableAtDDSTopLevel(const libdap::DDS &dds, const string &name)
static bool doTypesMatch(const libdap::Array &lhs, const libdap::Array &rhs)
static unsigned int collectVariableArraysInOrder(std::vector< libdap::Array * > &varArrays, const std::string &collectVarName, const ConstDDSList &datasetsInOrder)
static void transferArrayConstraints(libdap::Array *pToArray, const libdap::Array &fromArray, bool skipFirstFromDim, bool skipFirstToDim, bool printDebug=false, const std::string &debugChannel="agg_util")
static void printDimensions(std::ostream &os, const libdap::Array &fromArray)
static libdap::Array * readDatasetArrayDataForAggregation(const libdap::Array &constrainedTemplateArray, const std::string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const std::string &debugChannel)
static void performUnionAggregation(libdap::DDS *pOutputUnion, const ConstDDSList &datasetsInOrder)
static libdap::BaseType * getVariableNoRecurse(const libdap::DDS &dds, const std::string &name)
static void unionAllVariablesInto(libdap::DDS *pOutputUnion, const ConstDDSList &datasetsInOrder)
static libdap::Array * getAsArrayIfPossible(libdap::BaseType *pBT)
static bool couldBeCoordinateVariable(libdap::BaseType *pBT)
static const libdap::Array * findMapByName(const libdap::Grid &inGrid, const std::string &findName)
static bool addCopyOfVariableIfNameIsAvailable(libdap::DDS *pOutDDS, const libdap::BaseType &varProto, bool add_at_top=false)
static void addOrReplaceVariableForName(libdap::DDS *pOutDDS, const libdap::BaseType &varProto)
static void unionAttrsInto(libdap::AttrTable *pOut, const libdap::AttrTable &fromTable)
static bool doShapesMatch(const libdap::Array &lhs, const libdap::Array &rhs, bool checkDimNames)
static bool findAttribute(const libdap::AttrTable &inTable, const string &name, libdap::AttrTable::Attr_iter &attr)
static void resetCVInsertionPosition()
static bool validateArrayTypesAndShapesMatch(const std::vector< libdap::Array * > &arrays, bool enforceMatchingDimNames)
static void printConstraints(std::ostream &os, const libdap::Array &fromArray)
static libdap::BaseType * findVariableAtDDSTopLevel(const libdap::DDS &dds, const string &name)
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
virtual libdap::Array * readAndGetArray(const std::string &name, const libdap::DDS &dds, const libdap::Array *const pConstraintTemplate, const std::string &debugChannel) const =0
virtual libdap::Array * readAndGetArray(const std::string &name, const libdap::DDS &dds, const libdap::Array *const pConstraintTemplate, const std::string &debugChannel) const
virtual TopLevelGridDataArrayGetter * clone() const
virtual TopLevelGridMapArrayGetter * clone() const
virtual libdap::Array * readAndGetArray(const std::string &name, const libdap::DDS &dds, const libdap::Array *const pConstraintTemplate, const std::string &debugChannel) const