36#include "AggregationElement.h"
37#include "AggMemberDatasetUsingLocationRef.h"
38#include "AggMemberDatasetSharedDDSWrapper.h"
39#include "AggregationUtil.h"
40#include "ArrayAggregateOnOuterDimension.h"
41#include "ArrayJoinExistingAggregation.h"
42#include "GridAggregateOnOuterDimension.h"
43#include "GridJoinExistingAggregation.h"
44#include "AggMemberDatasetDimensionCache.h"
46#include <libdap/AttrTable.h>
47#include <libdap/Array.h>
48#include <libdap/AttrTable.h>
49#include "DDSAccessInterface.h"
51#include "DimensionElement.h"
52#include <libdap/Grid.h>
53#include "MyBaseTypeFactory.h"
54#include "NCMLBaseArray.h"
56#include "NCMLParser.h"
57#include "NetcdfElement.h"
58#include "ScanElement.h"
60#include "BESStopWatch.h"
64using agg_util::AMDList;
73const string AggregationElement::_sTypeName =
"aggregation";
75const vector<string> AggregationElement::_sValidAttrs = getValidAttributes();
77AggregationElement::AggregationElement() :
78 NCMLElement(0), _type(
""), _dimName(
""), _recheckEvery(
""), _parent(0), _datasets(), _scanners(), _aggVars(), _gotVariableAggElement(
79 false), _wasAggregatedMapAddedForJoinExistingGrid(
false), _coordinateAxisType(
"")
84 RCObjectInterface(),
NCMLElement(proto), _type(proto._type), _dimName(proto._dimName), _recheckEvery(
85 proto._recheckEvery), _parent(proto._parent)
88 , _aggVars(proto._aggVars), _gotVariableAggElement(
false), _wasAggregatedMapAddedForJoinExistingGrid(
false), _coordinateAxisType(
93 if (!proto._datasets.empty()) {
95 "WARNING: AggregationElement copy ctor is deep copying all contained datasets! This might be memory and time intensive!");
99 _datasets.reserve(proto._datasets.size());
100 for (vector<NetcdfElement*>::const_iterator it = proto._datasets.begin(); it != proto._datasets.end(); ++it) {
102 addChildDataset(elt->
clone());
104 NCML_ASSERT(_datasets.size() == proto._datasets.size());
106 _scanners.reserve(proto._scanners.size());
107 for (vector<ScanElement*>::const_iterator it = proto._scanners.begin(); it != proto._scanners.end(); ++it) {
109 addScanElement(elt->
clone());
111 NCML_ASSERT(_scanners.size() == proto._scanners.size());
114AggregationElement::~AggregationElement()
116 BESDEBUG(
"ncml:memory",
"~AggregationElement called...");
121 _wasAggregatedMapAddedForJoinExistingGrid =
false;
124 while (!_datasets.empty()) {
126 _datasets.pop_back();
131 while (!_scanners.empty()) {
133 _scanners.pop_back();
164 if (BESISDEBUG( TIMING_LOG_KEY ))
165 sw.
start(
"AggregationElement::handleBegin",
"");
168 NCML_ASSERT(!getParentDataset());
171 if (!_parser->isScopeNetcdf()) {
173 "Got an <aggregation> = " +
toString()
174 +
" at incorrect parse location. They can only be direct children of <netcdf>. Scope="
175 + _parser->getScopeString());
179 NCML_ASSERT_MSG(dataset,
180 "We expected a non-noll current dataset while processing AggregationElement::handleBegin() for " +
toString());
184 "Got <aggregation> = " +
toString() +
" but the enclosing dataset = " + dataset->
toString()
185 +
" already had an aggregation set! There can be only one!");
192void AggregationElement::handleContent(
const string& content)
197 "Got non-whitespace for content and didn't expect it. Element=" +
toString() +
" content=\"" + content
209 BESDEBUG(
"ncml",
"AggregationElement::handleEnd() - Processing the aggregation!!" << endl);
211 if (isUnionAggregation()) {
212 BESDEBUG(
"ncml2",
"AggregationElement::handleEnd() - isUnionAggregation" << endl);
215 else if (isJoinNewAggregation()) {
216 BESDEBUG(
"ncml2",
"AggregationElement::handleEnd() - isJoinNewAggregation" << endl);
219 else if (isJoinExistingAggregation()) {
220 BESDEBUG(
"ncml2",
"AggregationElement::handleEnd() - isJoinExistingAggregation" << endl);
221 processJoinExisting();
223 else if (_type ==
"forecastModelRunCollection" || _type ==
"forecastModelSingleRunCollection") {
225 "Sorry, we do not implement the forecastModelRunCollection aggregations in this version of the NCML Module!");
229 "Unknown aggregation type=" + _type +
" at scope=" + _parser->getScopeString());
239bool AggregationElement::isJoinNewAggregation()
const
241 return (_type ==
"joinNew");
244bool AggregationElement::isUnionAggregation()
const
246 return (_type ==
"union");
249bool AggregationElement::isJoinExistingAggregation()
const
251 return (_type ==
"joinExisting");
257 BESDEBUG(
"ncml",
"AggregationElement: adding child dataset: " << pDataset->
toString() << endl);
261 _datasets.push_back(pDataset);
271 "Tried to add an aggregation variable twice: name=" + name +
" at scope=" + _parser->getScopeString());
274 _aggVars.push_back(name);
275 BESDEBUG(
"ncml",
"Added aggregation variable name=" + name << endl);
282 AggVarIter endIt = endAggVarIter();
283 AggVarIter it = beginAggVarIter();
284 for (; it != endIt; ++it) {
293string AggregationElement::printAggregationVariables()
const
296 AggVarIter endIt = endAggVarIter();
297 AggVarIter it = beginAggVarIter();
298 for (; it != endIt; ++it) {
306AggregationElement::AggVarIter AggregationElement::beginAggVarIter()
const
308 return _aggVars.begin();
311AggregationElement::AggVarIter AggregationElement::endAggVarIter()
const
313 return _aggVars.end();
318 return _gotVariableAggElement;
323 _gotVariableAggElement =
true;
329 _scanners.push_back(pScanner);
336 BESDEBUG(
"ncml",
"AggregationElement::processParentDatasetComplete() called..." << endl);
338 if (_type ==
"joinNew") {
339 processParentDatasetCompleteForJoinNew();
341 else if (_type ==
"joinExisting") {
342 processParentDatasetCompleteForJoinExisting();
357void AggregationElement::processUnion()
359 BESDEBUG(
"ncml",
"Processing a union aggregation..." << endl);
368 vector<const DDS*> datasetsInOrder;
371 collectDatasetsInOrder(datasetsInOrder);
373 if (getParentDataset()) {
374 pUnion = getParentDataset()->
getDDS();
376 AggregationUtil::performUnionAggregation(pUnion, datasetsInOrder);
379void AggregationElement::processJoinNew()
385 processAnyScanElements();
388 "AggregationElement::processJoinNew() - beginning joinNew on the following aggVars=" + printAggregationVariables() << endl);
391 BESDEBUG(
"ncml",
"Merging dimensions from children into aggregated dataset..." << endl);
395 unsigned int newDimSize = _datasets.size();
399 if (_datasets.empty()) {
400 THROW_NCML_PARSE_ERROR(
line(),
"In joinNew aggregation we cannot have zero datasets specified!");
404 DDS* pAggDDS = getParentDataset()->
getDDS();
406 DDS* pTemplateDDS = _datasets[0]->getDDS();
407 NCML_ASSERT_MSG(pTemplateDDS,
"AggregationElement::processJoinNew() - NULL template dataset!");
410 AggregationUtil::unionAttrsInto(&(pAggDDS->get_attr_table()), pTemplateDDS->get_attr_table());
415 vector<string>::const_iterator endIt = _aggVars.end();
416 for (vector<string>::const_iterator it = _aggVars.begin(); it != endIt; ++it) {
417 const string& varName = *it;
419 "AggregationElement::processJoinNew() - Aggregating with joinNew on variable=" << varName <<
"..." << endl);
420 processJoinNewOnAggVar(pAggDDS, varName, *pTemplateDDS);
428 AggregationUtil::resetCVInsertionPosition();
431 AggregationUtil::unionAllVariablesInto(pAggDDS, *pTemplateDDS,
true);
441doAllScannersSpecifyNCoords(
const vector<ScanElement*>& scanners)
444 for (vector<ScanElement*>::const_iterator it = scanners.begin();
445 it != scanners.end();
449 if ((*it)->ncoords().empty())
459void AggregationElement::processJoinExisting()
461 BESDEBUG(
"ncml:2",
"Called AggregationElement::processJoinExisting()...");
464 processAnyScanElements();
467 if (_datasets.empty()) {
468 THROW_NCML_PARSE_ERROR(
line(),
"In joinExisting aggregation we cannot have zero datasets specified!");
478 granuleList.reserve(_datasets.size());
479 fillDimensionCacheForJoinExistingDimension(granuleList, _dimName);
483 addNewDimensionForJoinExisting(granuleList);
488 BESDEBUG(
"ncml:2",
"Merging dimensions from children into aggregated dataset..." << endl);
489 mergeDimensions(
true, _dimName);
492 DDS* pAggDDS = getParentDataset()->
getDDS();
495 DDS* pTemplateDDS = _datasets[0]->getDDS();
496 NCML_ASSERT_MSG(pTemplateDDS,
"AggregationElement::processJoinExisting(): NULL template dataset!");
499 AggregationUtil::unionAttrsInto(&(pAggDDS->get_attr_table()), pTemplateDDS->get_attr_table());
502 decideWhichVariablesToJoinExist(*pTemplateDDS);
506 vector<string>::const_iterator endIt = _aggVars.end();
507 for (vector<string>::const_iterator it = _aggVars.begin(); it != endIt; ++it) {
508 const string& varName = *it;
509 BESDEBUG(
"ncml",
"Aggregating with joinExisting on variable=" << varName <<
"..." << endl);
510 processJoinExistingOnAggVar(pAggDDS, varName, *pTemplateDDS);
516 unionAddAllRequiredNonAggregatedVariablesFrom(*pTemplateDDS);
519void AggregationElement::unionAddAllRequiredNonAggregatedVariablesFrom(
const DDS& templateDDS)
526 AggregationUtil::resetCVInsertionPosition();
529 if (isJoinExistingAggregation()) {
531 AggregationUtil::unionAllVariablesInto(getParentDataset()->getDDS(), templateDDS,
true);
538 else if (isJoinNewAggregation())
541 AggregationUtil::unionAllVariablesInto(getParentDataset()->getDDS(), templateDDS,
true);
545void AggregationElement::decideWhichVariablesToJoinExist(
const DDS& templateDDS)
548 if (_aggVars.empty()) {
550 "Searching the the template DDS for variables with outer " "dimension matching the join dimension = " << _dimName <<
" in order to add them to the aggregation output list." << endl);
554 vector<string> matchingVars;
555 findVariablesWithOuterDimensionName(matchingVars, templateDDS, _dimName);
556 for (vector<string>::const_iterator it = matchingVars.begin(); it != matchingVars.end(); ++it) {
563 "joinExist aggregation had variableAgg specified... " "Validating these variables have outer dimension named " << _dimName << endl);
565 for (vector<string>::const_iterator it = _aggVars.begin(); it != _aggVars.end(); ++it) {
566 BaseType* pVar = AggregationUtil::findVariableAtDDSTopLevel(templateDDS, *it);
570 std::ostringstream msg;
571 msg <<
"Error validating the variableAgg list. The variable named " << *it
572 <<
" was not found in the top-level DDS!";
573 THROW_NCML_PARSE_ERROR(
line(), msg.str());
577 Array* pArray = AggregationUtil::getAsArrayIfPossible(pVar);
579 std::ostringstream msg;
580 msg <<
"The declared variableAgg aggregation variable named " << *it
581 <<
" was not of a type able to be aggregated!";
582 THROW_NCML_PARSE_ERROR(
line(), msg.str());
586 if (pArray->dimension_name(pArray->dim_begin()) != _dimName) {
587 std::ostringstream msg;
588 msg <<
"The declared variableAgg variable named " << *it <<
" did not match the outer dimension name "
589 << _dimName <<
" for this joinExisting aggregation!";
590 THROW_NCML_PARSE_ERROR(
line(), msg.str());
594 std::ostringstream msg;
595 msg <<
"The variable named " << *it <<
" is a valid joinExisting variable. Will be added to output.";
596 BESDEBUG(
"ncml", msg.str() << endl);
602void AggregationElement::fillDimensionCacheForJoinExistingDimension(AMDList& granuleList,
608 vector<NetcdfElement*>::iterator endIt = _datasets.end();
609 for (vector<NetcdfElement*>::iterator it = _datasets.begin(); it != endIt; ++it) {
610 granuleList.push_back((*it)->getAggMemberDataset());
615 if (doesFirstGranuleSpecifyNcoords()) {
617 if (!doAllGranulesSpecifyNcoords()) {
618 THROW_NCML_PARSE_ERROR(-1,
"In a joinExisting aggregation we found that the first "
619 "granule specified an ncoords but not all of the others "
620 "did. Either all or none of them should have ncoords specified.");
624 seedDimensionCacheFromUserSpecs(granuleList);
634 AMDList::iterator endIt = granuleList.end();
635 for (AMDList::iterator it = granuleList.begin(); it != endIt; ++it) {
638 BESDEBUG(
"ncml",
"AggregationElement::fillDimensionCacheForJoinExistingDimension() - Loading dimension cache for: " << (*it)->getLocation() <<
"..." << endl);
642 BESDEBUG(
"ncml",
"AggregationElement::fillDimensionCacheForJoinExistingDimension() - " <<
643 "WARNING NcML Dimension Caching is not configured or is not working! Loading dimensions from DDS for dataset: " <<
644 (*it)->getLocation() <<
"" << endl);
655bool AggregationElement::doesFirstGranuleSpecifyNcoords()
const
657 if (_datasets.size() > 0) {
658 return _datasets.at(0)->hasNcoords();
665bool AggregationElement::doAllGranulesSpecifyNcoords()
const
668 vector<NetcdfElement*>::const_iterator endIt = _datasets.end();
669 for (vector<NetcdfElement*>::const_iterator it = _datasets.begin(); it != endIt; ++it) {
670 success = success && (*it)->hasNcoords();
678void AggregationElement::seedDimensionCacheFromUserSpecs(agg_util::AMDList& rGranuleList)
const
680 NCML_ASSERT(_datasets.size() == rGranuleList.size());
682 vector<NetcdfElement*>::const_iterator datasetIt;
683 AMDList::iterator amdIt;
684 for (datasetIt = _datasets.begin(), amdIt = rGranuleList.begin(); datasetIt != _datasets.end();
685 ++datasetIt, ++amdIt) {
687 const NetcdfElement* pDataset = *datasetIt;
688 if (!pDataset->hasNcoords()) {
690 THROW_NCML_INTERNAL_ERROR(
"Expected netcdf element member of a joinExisting "
691 "aggregation to have the ncoords attribute specified "
694 unsigned int ncoords = pDataset->getNcoordsAsUnsignedInt();
696 VALID_PTR(pAMD.get());
700 pAMD->setDimensionCacheFor(dim,
true);
702 NCML_ASSERT_MSG((pAMD->isDimensionCached(dim.name) && pAMD->getCachedDimensionSize(dim.name) == dim.size),
703 "Dimension cache bug");
706 NCML_ASSERT(amdIt == rGranuleList.end());
711void AggregationElement::addNewDimensionForJoinExisting(
const agg_util::AMDList& rGranuleList)
714 unsigned int aggDimSize = 0;
715 for (AMDList::const_iterator it = rGranuleList.begin(); it != rGranuleList.end(); ++it) {
716 NCML_ASSERT((*it)->isDimensionCached(_dimName));
717 aggDimSize += (*it)->getCachedDimensionSize(_dimName);
721 NCML_ASSERT(getParentDataset());
722 NCML_ASSERT_MSG(!(getParentDataset()->getDimensionInLocalScope(_dimName)),
723 "AggregationElement::addNewDimensionForJoinExisting() found a dimension "
724 "named " + _dimName +
" already but did not expect it!");
731 oss <<
"Added joinExisting aggregation dimension "
732 " name=" << _dimName <<
" with aggregated size= " << aggDimSize;
733 BESDEBUG(
"ncml:2", oss.str());
736void AggregationElement::findVariablesWithOuterDimensionName(vector<string>& oMatchingVars,
const DDS& templateDDS,
737 const string& outerDimName)
const
739 for (DDS::Vars_iter it =
const_cast<DDS&
>(templateDDS).var_begin(); it !=
const_cast<DDS&
>(templateDDS).var_end();
741 Array* pArray = AggregationUtil::getAsArrayIfPossible(*it);
743 if (pArray && outerDimName == pArray->dimension_name(pArray->dim_begin())) {
744 oMatchingVars.push_back(pArray->name());
749void AggregationElement::getParamsForJoinAggOnVariable(JoinAggParams* pOutParams,
const DDS& ,
750 const std::string& varName,
const DDS& templateDDS)
752 VALID_PTR(pOutParams);
755 pOutParams->_pAggVarTemplate = AggregationUtil::getVariableNoRecurse(templateDDS, varName);
756 if (!(pOutParams->_pAggVarTemplate)) {
757 THROW_NCML_PARSE_ERROR(
line(),
758 " We could not find a template for the specified aggregation variable=" + varName
759 +
" so we cannot continue the aggregation.");
764 NCML_ASSERT_MSG(pDim,
"Didn't find a DimensionElement with the aggregation dimName=" + _dimName);
765 pOutParams->_pAggDim = &(pDim->getDimension());
775 BaseType* pExists = AggregationUtil::getVariableNoRecurse(aggOutputDDS, varName);
776 NCML_ASSERT_MSG(!pExists,
777 "Failed since the name of the new variable to add (name="
779 +
") already exists in the "
780 " output aggregation DDS! What happened?!");
785 collectAggMemberDatasets(pOutParams->_memberDatasets);
788void AggregationElement::processJoinNewOnAggVar(DDS* pAggDDS,
const std::string& varName,
const DDS& templateDDS)
791 if (
BESDebug::IsSet(TIMING_LOG_KEY)) sw.
start(
"AggregationElement::processJoinNewOnAggVar",
"");
794 JoinAggParams joinAggParams;
795 getParamsForJoinAggOnVariable(&joinAggParams,
796 *pAggDDS, varName, templateDDS);
799 BaseType* pAggVarTemplate = joinAggParams._pAggVarTemplate;
800 if (pAggVarTemplate->type() == dods_array_c) {
801 processAggVarJoinNewForArray(*pAggDDS, *(
static_cast<Array*
>(pAggVarTemplate)), *(joinAggParams._pAggDim),
802 joinAggParams._memberDatasets);
804 else if (pAggVarTemplate->type() == dods_grid_c) {
805 processAggVarJoinNewForGrid(*pAggDDS, *(
static_cast<Grid*
>(pAggVarTemplate)), *(joinAggParams._pAggDim),
806 joinAggParams._memberDatasets);
809 THROW_NCML_PARSE_ERROR(
line(),
810 "Got an aggregation variable not of type Array or Grid, but of: " + pAggVarTemplate->type_name()
811 +
" which we cannot aggregate!");
816void AggregationElement::processJoinExistingOnAggVar(DDS* pAggDDS,
const std::string& varName,
const DDS& templateDDS)
820 if (
BESDebug::IsSet(TIMING_LOG_KEY)) sw.
start(
"AggregationElement::processJoinExistingOnAggVar",
"");
823 JoinAggParams joinAggParams;
824 getParamsForJoinAggOnVariable(&joinAggParams,
825 *pAggDDS, varName, templateDDS);
828 BaseType* pAggVarTemplate = joinAggParams._pAggVarTemplate;
829 if (pAggVarTemplate->type() == dods_array_c) {
830 processAggVarJoinExistingForArray(*pAggDDS, *(
static_cast<Array*
>(pAggVarTemplate)), *(joinAggParams._pAggDim),
831 joinAggParams._memberDatasets);
833 else if (pAggVarTemplate->type() == dods_grid_c) {
834 processAggVarJoinExistingForGrid(*pAggDDS, *(
static_cast<Grid*
>(pAggVarTemplate)), *(joinAggParams._pAggDim),
835 joinAggParams._memberDatasets);
838 THROW_NCML_PARSE_ERROR(
line(),
839 "Got an aggregation variable not of type Array or Grid, but of: " + pAggVarTemplate->type_name()
840 +
" which we cannot aggregate!");
845void AggregationElement::processAggVarJoinNewForArray(DDS& aggDDS,
const libdap::Array& arrayTemplate,
849 if (
BESDebug::IsSet(TIMING_LOG_KEY)) sw.
start(
"AggregationElement::processJoinExistingOnAggVar",
"");
854 unique_ptr<ArrayAggregateOnOuterDimension> pAggArray(
858 NCML_ASSERT_MSG(!(arrayGetter.get()),
"Expected unique_ptr owner xfer, failed!");
864 "Adding new ArrayAggregateOnOuterDimension with name=" << arrayTemplate.name() <<
" to aggregated dataset!" << endl);
872 aggDDS.add_var_nocopy(pAggArray.release());
875void AggregationElement::processAggVarJoinNewForGrid(DDS& aggDDS,
const Grid& gridTemplate,
879 if (
BESDebug::IsSet(TIMING_LOG_KEY)) sw.
start(
"AggregationElement::processAggVarJoinNewForGrid",
"");
881 unique_ptr<GridAggregateOnOuterDimension> pAggGrid(
887 "Adding new GridAggregateOnOuterDimension with name=" << gridTemplate.name() <<
" to aggregated dataset!" << endl);
889 aggDDS.add_var_nocopy(pAggGrid.release());
892void AggregationElement::processAggVarJoinExistingForArray(DDS& aggDDS,
const libdap::Array& arrayTemplate,
897 if (
BESDebug::IsSet(TIMING_LOG_KEY)) sw.
start(
"AggregationElement::processAggVarJoinExistingForArray",
"");
902 unique_ptr<ArrayJoinExistingAggregation> pAggArray(
907 NCML_ASSERT_MSG(!(arrayGetter.get()),
"Expected unique_ptr owner xfer, failed!");
913 "Adding new ArrayJoinExistingAggregation with name=" << arrayTemplate.name() <<
" to aggregated dataset!" << endl);
915 aggDDS.add_var_nocopy(pAggArray.release());
918void AggregationElement::processAggVarJoinExistingForGrid(DDS& aggDDS,
const Grid& gridTemplate,
923 if (
BESDebug::IsSet(TIMING_LOG_KEY)) sw.
start(
"AggregationElement::processAggVarJoinExistingForGrid",
"");
925 unique_ptr<GridJoinExistingAggregation> pAggGrid(
929 "Adding new GridJoinExistingAggregation with name=" << gridTemplate.name() <<
" to aggregated dataset!" << endl);
931 aggDDS.add_var_nocopy(pAggGrid.release());
934void AggregationElement::processParentDatasetCompleteForJoinNew()
937 if (
BESDebug::IsSet(TIMING_LOG_KEY)) sw.
start(
"AggregationElement::processParentDatasetCompleteForJoinNew",
"");
939 NetcdfElement* pParentDataset = getParentDataset();
940 VALID_PTR(pParentDataset);
941 DDS* pParentDDS = pParentDataset->getDDS();
942 VALID_PTR(pParentDDS);
945 NCML_ASSERT_MSG(pDim,
" AggregationElement::processParentDatasetCompleteForJoinNew(): "
946 " didn't find a DimensionElement with the joinNew dimName=" + _dimName);
950 BaseType* pBT = AggregationUtil::getVariableNoRecurse(*pParentDDS, dim.name);
955 pCV = createAndAddCoordinateVariableForNewDimension(*pParentDDS, dim);
956 NCML_ASSERT_MSG(pCV,
"processParentDatasetCompleteForJoinNew(): "
957 "failed to create a new coordinate variable for dim=" + dim.name);
963 VariableElement* pVarElt = pParentDataset->findVariableElementForLibdapVar(pBT);
968 pCV = ensureVariableIsProperNewCoordinateVariable(pBT, dim,
true);
973 pCV = processDeferredCoordinateVariable(pBT, dim);
982 if (!_coordinateAxisType.empty()) {
983 addCoordinateAxisType(*pCV, _coordinateAxisType);
990 AggVarIter endIt = endAggVarIter();
991 for (it = beginAggVarIter(); it != endIt; ++it) {
992 const string& aggVar = *it;
993 BaseType* pBT = AggregationUtil::getVariableNoRecurse(*pParentDDS, aggVar);
997 pGrid->prepend_map(pCV,
true);
1002void AggregationElement::processParentDatasetCompleteForJoinExisting()
1005 if (
BESDebug::IsSet(TIMING_LOG_KEY)) sw.
start(
"AggregationElement::processParentDatasetCompleteForJoinExisting",
"");
1007 NetcdfElement* pParentDataset = getParentDataset();
1008 VALID_PTR(pParentDataset);
1009 DDS* pAggDDS = pParentDataset->getDDS();
1013 NCML_ASSERT_MSG(pDim,
" Didn't find a DimensionElement with the joinExisting dimName=" + _dimName);
1017 BaseType* pDimNameVar = AggregationUtil::getVariableNoRecurse(*pAggDDS, dim.name);
1019 bool placeholderExists =
false;
1025 VariableElement* pVarElt = pParentDataset->findVariableElementForLibdapVar(pDimNameVar);
1030 pCV = ensureVariableIsProperNewCoordinateVariable(pDimNameVar, dim,
true);
1032 placeholderExists =
false;
1037 placeholderExists =
true;
1045 unique_ptr<ArrayJoinExistingAggregation> pNewMap;
1051 auto endIt = endAggVarIter();
1052 for (
auto it = beginAggVarIter(); it != endIt; ++it) {
1053 const string& aggVar = *it;
1054 BaseType* pAggVar = AggregationUtil::getVariableNoRecurse(*pAggDDS, aggVar);
1062 if (!pCV || placeholderExists) {
1064 VALID_PTR(pNewMap.get());
1069 if (placeholderExists) {
1070 processPlaceholderCoordinateVariableForJoinExisting(*pDimNameVar, pNewMap.get());
1074 AggregationUtil::addOrReplaceVariableForName(pAggDDS, *(pNewMap.get()));
1077 pCV = pNewMap.get();
1081 NCML_ASSERT_MSG(pCV,
"Expected a coordinate variable since a Grid exists... what happened?");
1084 pGrid->prepend_map(pCV,
true);
1089void AggregationElement::processPlaceholderCoordinateVariableForJoinExisting(
const libdap::BaseType& placeholderVar,
1090 libdap::Array* pNewVar)
1095 BaseType* pNewEltProto = pNewVar->var();
1096 VALID_PTR(pNewEltProto);
1097 if (placeholderVar.type() != pNewEltProto->type()) {
1098 THROW_NCML_PARSE_ERROR(
line(),
1099 " We expected the type of the placeholder coordinate variable to be the same "
1100 " as that created by the aggregation. Expected type=" + pNewEltProto->type_name()
1101 + +
" but placeholder has type=" + placeholderVar.type_name()
1102 +
" Please make sure these match in the input file!");
1106 AggregationUtil::gatherMetadataChangesFrom(pNewVar, placeholderVar);
1115 _coordinateAxisType = cat;
1121 return _coordinateAxisType;
1125AggregationElement::ensureVariableIsProperNewCoordinateVariable(libdap::BaseType* pBT,
const agg_util::Dimension& dim,
1126 bool throwOnInvalidCV)
const
1132 if (AggregationUtil::couldBeCoordinateVariable(pBT)) {
1134 Array* pArr =
static_cast<Array*
>(pBT);
1135 if (pArr->length() ==
static_cast<int>(dim.size)) {
1142 oss << string(
"In the aggregation for dimension=") << dim.name
1143 <<
": The coordinate variable we found does NOT have the same dimensionality as the"
1144 "aggregated dimension! We expected dimensionality=" << dim.size
1145 <<
" but the coordinate variable had dimensionality=" << pArr->length();
1146 BESDEBUG(
"ncml", oss.str() << endl);
1147 if (throwOnInvalidCV) {
1148 THROW_NCML_PARSE_ERROR(
line(), oss.str());
1155 std::ostringstream msg;
1156 msg <<
"Aggregation found a variable matching aggregated dimension name=" << dim.name
1157 <<
" but it was not a coordinate variable. "
1158 " It must be a 1D array whose dimension name is the same as its name. ";
1159 BESDEBUG(
"ncml",
"AggregationElement::ensureVariableIsProperNewCoordinateVariable: " + msg.str() << endl);
1160 if (throwOnInvalidCV) {
1161 THROW_NCML_PARSE_ERROR(
line(), msg.str())
1169AggregationElement::findMatchingCoordinateVariable(
const DDS& dds,
const agg_util::Dimension& dim,
1170 bool throwOnInvalidCV)
const
1172 BaseType* pBT = AggregationUtil::getVariableNoRecurse(dds, dim.name);
1179 return ensureVariableIsProperNewCoordinateVariable(pBT, dim, throwOnInvalidCV);
1195AggregationElement::processDeferredCoordinateVariable(libdap::BaseType* pBT,
const agg_util::Dimension& dim)
1200 "Processing the placeholder coordinate variable (no values) for the " "current aggregation to add placeholder metadata to the generated values..." << endl);
1205 unique_ptr<Array> pNewArrCV = createCoordinateVariableForNewDimension(dim);
1206 NCML_ASSERT_MSG(pNewArrCV.get(),
" createCoordinateVariableForNewDimension()"
1210 BaseType* pNewEltProto = pNewArrCV->var();
1211 VALID_PTR(pNewEltProto);
1212 if (pBT->type() != pNewEltProto->type()) {
1213 THROW_NCML_PARSE_ERROR(
line(),
1214 " We expected the type of the placeholder coordinate variable to be the same "
1215 " as that created by the aggregation. Expected type=" + pNewEltProto->type_name()
1216 + +
" but placeholder has type=" + pBT->type_name()
1217 +
" Please make sure these match in the input file!");
1225 pNewArrCV->get_attr_table() = pBT->get_attr_table();
1228 DDS* pDDS = getParentDataset()->
getDDS();
1230 pDDS->del_var(pBT->name());
1234 BESDEBUG(
"ncml",
"Adding CV: " << pNewArrCV->name() << endl);
1236 pDDS->add_var(pNewArrCV.get());
1238 pDDS->add_var_nocopy(pNewArrCV.release());
1241 Array* pArrCV =
static_cast<Array*
>(AggregationUtil::getVariableNoRecurse(*pDDS, dim.name));
1246unique_ptr<libdap::Array> AggregationElement::createCoordinateVariableForNewDimension(
1250 NCML_ASSERT(_datasets.size() > 0);
1251 bool hasCoordValue = !(_datasets[0]->coordValue().empty());
1252 if (hasCoordValue) {
1253 return createCoordinateVariableForNewDimensionUsingCoordValue(dim);
1256 return createCoordinateVariableForNewDimensionUsingLocation(dim);
1261AggregationElement::createAndAddCoordinateVariableForNewDimension(DDS& dds,
const agg_util::Dimension& dim)
1263 unique_ptr<libdap::Array> pNewCV = createCoordinateVariableForNewDimension(dim);
1266 NCML_ASSERT_MSG(pNewCV.get(),
1267 "AgregationElement::createCoordinateVariableForNewDimension() failed to create a coordinate variable!");
1275 BESDEBUG(
"ncml2",
"AggregationElement::createAndAddCoordinateVariableForNewDimension: " << pNewCV->name());
1277 dds.add_var(pNewCV.get());
1286 static int last_added = 0;
1287 DDS::Vars_iter pos = dds.var_begin();
1288 for (
int i = 0; i < last_added; ++i)
1291 dds.insert_var(pos, pNewCV.get());
1295 Array* pCV =
static_cast<Array*
>(AggregationUtil::getVariableNoRecurse(dds, dim.name));
1297 NCML_ASSERT_MSG(pCV,
"Logic Error: tried to add a new coordinate variable while processing joinNew"
1298 " but we couldn't locate it!");
1302unique_ptr<libdap::Array> AggregationElement::createCoordinateVariableForNewDimensionUsingCoordValue(
1305 NCML_ASSERT(_datasets.size() > 0);
1306 NCML_ASSERT_MSG(_datasets.size() == dim.size,
"Logic error: Number of datasets doesn't match dimension!");
1308 double doubleVal = 0;
1309 if (_datasets[0]->getCoordValueAsDouble(doubleVal)) {
1310 return createCoordinateVariableForNewDimensionUsingCoordValueAsDouble(dim);
1313 return createCoordinateVariableForNewDimensionUsingCoordValueAsString(dim);
1317unique_ptr<libdap::Array> AggregationElement::createCoordinateVariableForNewDimensionUsingCoordValueAsDouble(
1320 vector<dods_float64> coords;
1321 coords.reserve(dim.size);
1322 double doubleVal = 0;
1324 for (
unsigned int i = 0; i < _datasets.size(); ++i) {
1325 const NetcdfElement* pDataset = _datasets[i];
1326 if (!pDataset->getCoordValueAsDouble(doubleVal)) {
1327 THROW_NCML_PARSE_ERROR(
line(),
1328 "In creating joinNew coordinate variable from coordValue, expected a coordValue of type double"
1329 " but failed! coordValue=" + pDataset->coordValue() +
" which was in the dataset location="
1330 + pDataset->location() +
" with title=\"" + pDataset->title() +
"\"");
1334 coords.push_back(
static_cast<dods_float64
>(doubleVal));
1341 NCML_ASSERT_MSG(pNewCV.get(),
"createCoordinateVariableForNewDimensionUsingCoordValueAsDouble: failed to create"
1342 " the new Array<Float64> for variable: " + dim.name);
1343 pNewCV->append_dim(dim.size, dim.name);
1344 pNewCV->set_value(coords, coords.size());
1348unique_ptr<libdap::Array> AggregationElement::createCoordinateVariableForNewDimensionUsingCoordValueAsString(
1352 vector<string> coords;
1353 coords.reserve(dim.size);
1354 for (
unsigned int i = 0; i < _datasets.size(); ++i) {
1355 const NetcdfElement* pDataset = _datasets[i];
1356 if (pDataset->coordValue().empty()) {
1357 int parseLine =
line();
1358 THROW_NCML_PARSE_ERROR(parseLine,
1359 "In creating joinNew coordinate variable from coordValue, expected a coordValue of type string"
1360 " but it was empty! dataset location=" + pDataset->location() +
" with title=\"" + pDataset->title()
1365 coords.push_back(pDataset->coordValue());
1371 NCML_ASSERT_MSG(pNewCV.get(),
"createCoordinateVariableForNewDimensionUsingCoordValueAsString: failed to create"
1372 " the new Array<String> for variable: " + dim.name);
1373 pNewCV->append_dim(dim.size, dim.name);
1374 pNewCV->set_value(coords, coords.size());
1378unique_ptr<libdap::Array> AggregationElement::createCoordinateVariableForNewDimensionUsingLocation(
1382 vector<string> coords;
1383 coords.reserve(dim.size);
1384 for (
unsigned int i = 0; i < _datasets.size(); ++i) {
1385 const NetcdfElement* pDataset = _datasets[i];
1386 string location(
"");
1387 if (pDataset->location().empty()) {
1388 std::ostringstream oss;
1389 oss <<
"Virtual_Dataset_" << i;
1390 location = oss.str();
1394 location = pDataset->location();
1396 coords.push_back(location);
1401 NCML_ASSERT_MSG(pNewCV.get(),
1402 "createCoordinateVariableForNewDimensionUsingCoordValueUsingLocation: failed to create"
1403 " the new Array<String> for variable: " + dim.name);
1405 pNewCV->append_dim(dim.size, dim.name);
1406 pNewCV->set_value(coords, coords.size());
1410void AggregationElement::collectDatasetsInOrder(vector<const DDS*>& ddsList)
const
1413 ddsList.reserve(_datasets.size());
1414 vector<NetcdfElement*>::const_iterator endIt = _datasets.end();
1415 vector<NetcdfElement*>::const_iterator it;
1416 for (it = _datasets.begin(); it != endIt; ++it) {
1417 const NetcdfElement* elt = *it;
1419 const DDS* pDDS = elt->getDDS();
1421 ddsList.push_back(pDDS);
1425void AggregationElement::collectAggMemberDatasets(AMDList& rMemberDatasets)
const
1427 rMemberDatasets.resize(0);
1428 rMemberDatasets.reserve(_datasets.size());
1430 for (vector<NetcdfElement*>::const_iterator it = _datasets.begin(); it != _datasets.end(); ++it) {
1433 VALID_PTR(pAGM.get());
1436 if (!((*it)->ncoords().empty()) && !_dimName.empty()) {
1437 if (!(pAGM->isDimensionCached(_dimName))) {
1438 unsigned int ncoords = (*it)->getNcoordsAsUnsignedInt();
1445 rMemberDatasets.push_back(pAGM);
1449void AggregationElement::processAnyScanElements()
1451 if (_scanners.size() > 0) {
1452 BESDEBUG(
"ncml",
"Started to process " << _scanners.size() <<
" scan elements..." << endl);
1455 vector<ScanElement*>::iterator it;
1456 vector<ScanElement*>::iterator endIt = _scanners.end();
1457 vector<NetcdfElement*> scannedDatasets;
1458 for (it = _scanners.begin(); it != endIt; ++it) {
1459 BESDEBUG(
"ncml",
"Processing scan element = " << (*it)->toString() <<
" ..." << endl);
1463 (*it)->getDatasetList(scannedDatasets);
1468 vector<NetcdfElement*>::iterator datasetIt;
1469 vector<NetcdfElement*>::iterator datasetEndIt = scannedDatasets.end();
1470 for (datasetIt = scannedDatasets.begin(); datasetIt != datasetEndIt; ++datasetIt) {
1472 _parser->addChildDatasetToCurrentDataset(*datasetIt);
1474 (*datasetIt)->unref();
1477 scannedDatasets.clear();
1481void AggregationElement::mergeDimensions(
bool checkDimensionMismatch,
const std::string& dimToSkip)
1483 NetcdfElement* pParent = getParentDataset();
1485 vector<NetcdfElement*>::const_iterator datasetsEndIt = _datasets.end();
1486 vector<NetcdfElement*>::const_iterator datasetsIt;
1487 for (datasetsIt = _datasets.begin(); datasetsIt != datasetsEndIt; ++datasetsIt) {
1489 const NetcdfElement* dataset = *datasetsIt;
1491 const vector<DimensionElement*>& dimensions = dataset->getDimensionElements();
1492 vector<DimensionElement*>::const_iterator dimEndIt = dimensions.end();
1493 vector<DimensionElement*>::const_iterator dimIt;
1494 for (dimIt = dimensions.begin(); dimIt != dimEndIt; ++dimIt) {
1495 const DimensionElement* pDim = *dimIt;
1498 if (!dimToSkip.empty() && (pDim->name() == dimToSkip)) {
1502 const DimensionElement* pUnionDim = pParent->getDimensionInLocalScope(pDim->name());
1505 if (!pUnionDim->checkDimensionsMatch(*pDim)) {
1506 string msg = string(
"The union aggregation already had a dimension=") + pUnionDim->toString()
1507 +
" but we found another with different cardinality: " + pDim->toString()
1508 +
" This is likely an error and could cause a later exception.";
1509 BESDEBUG(
"ncml",
"WARNING: " + msg);
1510 if (checkDimensionMismatch) {
1512 msg +
" Scope=" + _parser->getScopeString());
1520 "Dimension name=" << pDim->name() <<
" was not found in the union yet, so adding it. The full elt is: " << pDim->toString() << endl);
1521 pParent->addDimension(
const_cast<DimensionElement*
>(pDim));
1527#define COORDINATE_AXIS_TYPE_ATTR "_CoordinateAxisType"
1528void AggregationElement::addCoordinateAxisType(libdap::Array& rCV,
const std::string& cat)
1530 AttrTable& rAT = rCV.get_attr_table();
1531 AttrTable::Attr_iter foundIt = rAT.simple_find(COORDINATE_AXIS_TYPE_ATTR);
1533 if (foundIt != rAT.attr_end()) {
1534 rAT.del_attr(COORDINATE_AXIS_TYPE_ATTR);
1538 "Adding attribute to the aggregation variable " << rCV.name() <<
" Attr is " << COORDINATE_AXIS_TYPE_ATTR <<
" = " << cat << endl);
1541 rAT.append_attr(COORDINATE_AXIS_TYPE_ATTR,
"String", cat);
1544vector<string> AggregationElement::getValidAttributes()
1546 vector<string> attrs;
1547 attrs.push_back(
"type");
1548 attrs.push_back(
"dimName");
1549 attrs.push_back(
"recheckEvery");
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
virtual bool start(std::string name)
void loadDimensionCache(AggMemberDataset *amd)
static AggMemberDatasetDimensionCache * get_instance()
virtual void fillDimensionCacheByUsingDDS()=0
unique_ptr< ArrayJoinExistingAggregation > makeAggregatedOuterMapVector() const
virtual int unref() const
A reference to an RCObject which automatically ref() and deref() on creation and destruction.
virtual string toString() const
const std::string & getAggregationVariableCoordinateAxisType() const
void setAggregationVariableCoordinateAxisType(const std::string &cat)
void setVariableAggElement()
void addChildDataset(NetcdfElement *pDataset)
NetcdfElement * setParentDataset(NetcdfElement *parent)
Private Impl.
void processParentDatasetComplete()
void addScanElement(ScanElement *pScanner)
bool isAggregationVariable(const string &name) const
bool gotVariableAggElement() const
virtual void setAttributes(const XMLAttributeMap &attrs)
virtual void handleBegin()
virtual const string & getTypeName() const
void addAggregationVariable(const string &name)
virtual AggregationElement * clone() const
static std::unique_ptr< libdap::Array > makeArrayTemplateVariable(const std::string &type, const std::string &name, bool addTemplateVar)
Base class for NcML element concrete classes.
virtual bool validateAttributes(const XMLAttributeMap &attrs, const std::vector< std::string > &validAttrs, std::vector< std::string > *pInvalidAttrs=0, bool printInvalid=true, bool throwOnError=true)
static std::string printAttributeIfNotEmpty(const std::string &attrName, const std::string &attrValue)
int getParseLineNumber() const
static bool isAllWhitespace(const std::string &str)
Concrete class for NcML <netcdf> element.
void addDimension(DimensionElement *dim)
virtual const libdap::DDS * getDDS() const
virtual NetcdfElement * clone() const
void setChildAggregation(AggregationElement *agg, bool throwIfExists=true)
const DimensionElement * getDimensionInLocalScope(const std::string &name) const
void setVariableGotValues(libdap::BaseType *pVarToValidate, bool removeEntry)
virtual std::string toString() const
void setParentAggregation(AggregationElement *parent)
AggregationElement * getChildAggregation() const
virtual ScanElement * clone() const
void setParent(AggregationElement *pParent)
const std::string getValueForLocalNameOrDefault(const std::string &localname, const std::string &defVal="") const
NcML Parser for adding/modifying/removing metadata (attributes) to existing local datasets using NcML...