bes Updated for version 3.20.13
ArrayJoinExistingAggregation.cc
1
2// This file is part of the "NcML Module" project, a BES module designed
3// to allow NcML files to be used to be used as a wrapper to add
4// AIS to existing datasets of any format.
5//
6// Copyright (c) 2010 OPeNDAP, Inc.
7// Author: Michael Johnson <m.johnson@opendap.org>
8//
9// For more information, please also see the main website: http://opendap.org/
10//
11// This library is free software; you can redistribute it and/or
12// modify it under the terms of the GNU Lesser General Public
13// License as published by the Free Software Foundation; either
14// version 2.1 of the License, or (at your option) any later version.
15//
16// This library is distributed in the hope that it will be useful,
17// but WITHOUT ANY WARRANTY; without even the implied warranty of
18// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19// Lesser General Public License for more details.
20//
21// You should have received a copy of the GNU Lesser General Public
22// License along with this library; if not, write to the Free Software
23// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24//
25// Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26//
27// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29
30#include "config.h"
31
32#include <sstream>
33
34#include <libdap/Marshaller.h>
35
36#include "BESDebug.h"
37#include "BESStopWatch.h"
38
39#include "ArrayJoinExistingAggregation.h"
40
41#include "AggregationException.h" // agg_util
42#include "AggregationUtil.h" // agg_util
43#include "NCMLDebug.h"
44
45#define DEBUG_CHANNEL "ncml:2"
46
47namespace agg_util {
48
50 AMDList memberDatasets, std::unique_ptr<ArrayGetterInterface> arrayGetter, const Dimension& joinDim) :
51 ArrayAggregationBase(granuleTemplate, std::move(memberDatasets), std::move(arrayGetter)),
52 _joinDim(joinDim)
53{
54 BESDEBUG_FUNC(DEBUG_CHANNEL, "Making the aggregated outer dimension be: " + joinDim.toString() + "\n");
55
56 // We created the array with the given granule prototype, but the resulting
57 // outer dimension size must be calculated according to the
58 // value in the passed in dimension object.
59 libdap::Array::dimension& rOuterDim = *(dim_begin());
60 NCML_ASSERT_MSG(rOuterDim.name == joinDim.name, "The outer dimension name of this is not the expected "
61 "outer dimension name! Broken precondition: This ctor cannot be called "
62 "without this being true!");
63 rOuterDim.size = joinDim.size;
64 // Force it to recompute constraints since we changed size.
65 reset_constraint();
66
67 ostringstream oss;
69
70 BESDEBUG_FUNC(DEBUG_CHANNEL, "Constrained Dims after set are: " + oss.str());
71}
72
74 ArrayAggregationBase(rhs), _joinDim(rhs._joinDim)
75{
76 duplicate(rhs);
77}
78
79/* virtual */
80ArrayJoinExistingAggregation::~ArrayJoinExistingAggregation()
81{
82 cleanup();
83}
84
85ArrayJoinExistingAggregation&
86ArrayJoinExistingAggregation::operator=(const ArrayJoinExistingAggregation& rhs)
87{
88 if (this != &rhs) {
89 cleanup();
90 ArrayAggregationBase::operator=(rhs);
91 duplicate(rhs);
92 }
93 return *this;
94}
95
96/* virtual */
97ArrayJoinExistingAggregation*
99{
100 return new ArrayJoinExistingAggregation(*this);
101}
102
103// Set this to 0 to get the old behavior where the entire response
104// (for this variable) is built in memory and then sent to the client.
105#define PIPELINING 1
106
107/* virtual */
108// begin modifying here for the double buffering
109// see notes about how this was written marked with '***'
110// Following this method is an older version of serialize that
111// provides no new functionality but does get run instead of the
112// more general implementation in libdap::Array.
113bool ArrayJoinExistingAggregation::serialize(libdap::ConstraintEvaluator &eval, libdap::DDS &dds, libdap::Marshaller &m,
114 bool ce_eval)
115{
116 BESStopWatch sw;
117 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("ArrayJoinExistingAggregation::serialize", "");
118
119 // *** This serialize() implementation was made by starting with a simple version that
120 // *** tested read_p(), calling read() if needed and tsting send_p() and is_in_selection(),
121 // *** returning true if the data did not need to be sent. I moved that test here.
122
123 // Only continue if we are supposed to serialize this object at all.
124 if (!(send_p() || is_in_selection())) {
125 BESDEBUG_FUNC(DEBUG_CHANNEL, "Object not in output, skipping... name=" << name() << endl);
126 return true;
127 }
128
129 // *** Add status so that we can do our magic _or_ pass off the call to libdap
130 // *** and collect the result either way.
131 bool status = false;
132
133 if (!read_p()) {
134 // *** copy lines from AggregationBase::read() into here in place
135 // *** of the call to read()
136
137 // call subclass impl
139
140 // *** Inserted code from readConstrainedGranuleArraysAndAggregateDataHook here
141
142 // outer one is the first in iteration
143 const Array::dimension& outerDim = *(dim_begin());
144 BESDEBUG("ncml",
145 "Aggregating datasets array with outer dimension constraints: " << " start=" << outerDim.start << " stride=" << outerDim.stride << " stop=" << outerDim.stop << endl);
146
147 try {
148#if PIPELINING
149 // assumes the constraints are already set properly on this
150 m.put_vector_start(length());
151#else
152 reserve_value_capacity();
153#endif
154
155 // Start the iteration state for the granule.
156 const AMDList& datasets = getDatasetList(); // the list
157 NCML_ASSERT(!datasets.empty());
158 int currDatasetIndex = 0; // index into datasets
159 const AggMemberDataset* pCurrDataset = (datasets[currDatasetIndex]).get();
160
161 int outerDimIndexOfCurrDatasetHead = 0;
162 int currDatasetSize = int(pCurrDataset->getCachedDimensionSize(_joinDim.name));
163 bool currDatasetWasRead = false;
164
165 // where in this output array we are writing next
166 unsigned int nextOutputBufferElementIndex = 0;
167
168 // Traverse the outer dimension constraints,
169 // Keeping track of which dataset we need to
170 // be inside for the given values of the constraint.
171 for (int outerDimIndex = outerDim.start; outerDimIndex <= outerDim.stop && outerDimIndex < outerDim.size;
172 outerDimIndex += outerDim.stride) {
173 // Figure out where the given outer index maps into in local granule space
174 int localGranuleIndex = outerDimIndex - outerDimIndexOfCurrDatasetHead;
175
176 // if this is beyond the dataset end, move state to the next dataset
177 // and try again until we're in the proper interval, with proper dataset.
178 while (localGranuleIndex >= currDatasetSize) {
179 localGranuleIndex -= currDatasetSize;
180 outerDimIndexOfCurrDatasetHead += currDatasetSize;
181 ++currDatasetIndex;
182 NCML_ASSERT(currDatasetIndex < int(datasets.size()));
183 pCurrDataset = datasets[currDatasetIndex].get();
184 currDatasetSize = pCurrDataset->getCachedDimensionSize(_joinDim.name);
185 currDatasetWasRead = false;
186
187 BESDEBUG_FUNC(DEBUG_CHANNEL,
188 "The constraint traversal passed a granule boundary " << "on the outer dimension and is stepping forward into " << "granule index=" << currDatasetIndex << endl);
189 }
190
191 // If we haven't read in this granule yet (we passed a boundary)
192 // then do it now. Map constraints into the local granule space.
193 if (!currDatasetWasRead) {
194 BESDEBUG_FUNC(DEBUG_CHANNEL,
195 " Current granule dataset was traversed but not yet " "read and copied into output. Mapping constraints " "and calling read()..." << endl);
196
197 // Set up a constraint object for the actual granule read
198 // so that it only loads the data values in which we are
199 // interested.
200 Array& granuleConstraintTemplate = getGranuleTemplateArray();
201
202 // The inner dim constraints were set up in the containing read() call.
203 // The outer dim was left open for us to fix now...
204 Array::Dim_iter outerDimIt = granuleConstraintTemplate.dim_begin();
205
206 // modify the outerdim size to match the dataset we need to
207 // load. The inners MUST match so we can let those get
208 //checked later...
209 outerDimIt->size = currDatasetSize;
210 outerDimIt->c_size = currDatasetSize; // this will get recalc below?
211
212 // find the mapped endpoint
213 // Basically, the fullspace endpoint mapped to local offset,
214 // clamped into the local granule size.
215 int granuleStopIndex = std::min(outerDim.stop - outerDimIndexOfCurrDatasetHead,
216 currDatasetSize - 1);
217
218 // we must clamp the stride to the interval of the
219 // dataset in order to avoid an exception in
220 // add_constraint on stride being larger than dataset.
221 int clampedStride = std::min(outerDim.stride, currDatasetSize);
222 // mapped endpoint clamped within this granule
223 granuleConstraintTemplate.add_constraint(outerDimIt, localGranuleIndex, clampedStride,
224 granuleStopIndex);
226 getGranuleTemplateArray(), name(), const_cast<AggMemberDataset&>(*pCurrDataset),
227 getArrayGetterInterface(), DEBUG_CHANNEL);
228#if PIPELINING
229 m.put_vector_part(pDatasetArray->get_buf(), getGranuleTemplateArray().length(), var()->width(),
230 var()->type());
231#else
232 this->set_value_slice_from_row_major_vector(*pDatasetArray, nextOutputBufferElementIndex);
233#endif
234
235 pDatasetArray->clear_local_data();
236
237 // Jump output buffer index forward by the amount we added.
238 nextOutputBufferElementIndex += getGranuleTemplateArray().length();
239 currDatasetWasRead = true;
240
241 BESDEBUG_FUNC(DEBUG_CHANNEL,
242 " The granule index " << currDatasetIndex << " was read with constraints and copied into the aggregation output." << endl);
243 } // !currDatasetWasRead
244 } // for loop over outerDim
245 } // end of try
246 catch (AggregationException& ex) {
247 THROW_NCML_PARSE_ERROR(-1, ex.what());
248 }
249
250 // *** end of code inserted from readConstrainedGranuleArraysAndAggregateDataHook
251
252#if PIPELINING
253 m.put_vector_end();
254 status = true;
255#else
256 set_read_p(true);
257 status = libdap::Array::serialize(eval, dds, m, ce_eval);
258#endif
259 }
260 else {
261 status = libdap::Array::serialize(eval, dds, m, ce_eval);
262 }
263
264 return status;
265}
266
268// Private Impl Below
269
270void ArrayJoinExistingAggregation::duplicate(const ArrayJoinExistingAggregation& rhs)
271{
272 _joinDim = rhs._joinDim;
273}
274
275void ArrayJoinExistingAggregation::cleanup() noexcept
276{
277}
278
279/* virtual */
281{
282 // transfer the constraints from this object into the subArray template
283 // skipping our first dim which is the join dim we need to do specially every
284 // granule in the read hook.
286 *this, // from this
287 true, // skip first dim in the copy since we handle it special
288 true, // also skip it in the toArray for the same reason.
289 true, // print debug
290 DEBUG_CHANNEL); // on this channel
291}
292
294{
295 BESStopWatch sw;
296 if (BESDebug::IsSet(TIMING_LOG_KEY))
297 sw.start("ArrayJoinExistingAggregation::readConstrainedGranuleArraysAndAggregateDataHook", "");
298
299 // outer one is the first in iteration
300 const Array::dimension& outerDim = *(dim_begin());
301 BESDEBUG("ncml",
302 "Aggregating datasets array with outer dimension constraints: " << " start=" << outerDim.start << " stride=" << outerDim.stride << " stop=" << outerDim.stop << endl);
303
304 try {
305 // assumes the constraints are already set properly on this
306 reserve_value_capacity();
307
308 // Start the iteration state for the granule.
309 const AMDList& datasets = getDatasetList(); // the list
310 NCML_ASSERT(!datasets.empty());
311 int currDatasetIndex = 0; // index into datasets
312 const AggMemberDataset* pCurrDataset = (datasets[currDatasetIndex]).get();
313
314 int outerDimIndexOfCurrDatasetHead = 0;
315 int currDatasetSize = int(pCurrDataset->getCachedDimensionSize(_joinDim.name));
316 bool currDatasetWasRead = false;
317
318 // where in this output array we are writing next
319 unsigned int nextOutputBufferElementIndex = 0;
320
321 // Traverse the outer dimension constraints,
322 // Keeping track of which dataset we need to
323 // be inside for the given values of the constraint.
324 for (int outerDimIndex = outerDim.start; outerDimIndex <= outerDim.stop && outerDimIndex < outerDim.size;
325 outerDimIndex += outerDim.stride) {
326 // Figure out where the given outer index maps into in local granule space
327 int localGranuleIndex = outerDimIndex - outerDimIndexOfCurrDatasetHead;
328
329 // if this is beyond the dataset end, move state to the next dataset
330 // and try again until we're in the proper interval, with proper dataset.
331 while (localGranuleIndex >= currDatasetSize) {
332 localGranuleIndex -= currDatasetSize;
333 outerDimIndexOfCurrDatasetHead += currDatasetSize;
334 ++currDatasetIndex;
335 NCML_ASSERT(currDatasetIndex < int(datasets.size()));
336 pCurrDataset = datasets[currDatasetIndex].get();
337 currDatasetSize = pCurrDataset->getCachedDimensionSize(_joinDim.name);
338 currDatasetWasRead = false;
339
340 BESDEBUG_FUNC(DEBUG_CHANNEL,
341 "The constraint traversal passed a granule boundary " << "on the outer dimension and is stepping forward into " << "granule index=" << currDatasetIndex << endl);
342 }
343
344 // If we haven't read in this granule yet (we passed a boundary)
345 // then do it now. Map constraints into the local granule space.
346 if (!currDatasetWasRead) {
347 BESDEBUG_FUNC(DEBUG_CHANNEL,
348 " Current granule dataset was traversed but not yet " "read and copied into output. Mapping constraints " "and calling read()..." << endl);
349
350 // Set up a constraint object for the actual granule read
351 // so that it only loads the data values in which we are
352 // interested.
353 Array& granuleConstraintTemplate = getGranuleTemplateArray();
354
355 // The inner dim constraints were set up in the containing read() call.
356 // The outer dim was left open for us to fix now...
357 Array::Dim_iter outerDimIt = granuleConstraintTemplate.dim_begin();
358
359 // modify the outerdim size to match the dataset we need to
360 // load. The inners MUST match so we can let those get
361 //checked later...
362 outerDimIt->size = currDatasetSize;
363 outerDimIt->c_size = currDatasetSize; // this will get recalc below?
364
365 // find the mapped endpoint
366 // Basically, the fullspace endpoint mapped to local offset,
367 // clamped into the local granule size.
368 int granuleStopIndex = std::min(outerDim.stop - outerDimIndexOfCurrDatasetHead, currDatasetSize - 1);
369
370 // we must clamp the stride to the interval of the
371 // dataset in order to avoid an exception in
372 // add_constraint on stride being larger than dataset.
373 int clampedStride = std::min(outerDim.stride, currDatasetSize);
374 // mapped endpoint clamped within this granule
375 granuleConstraintTemplate.add_constraint(outerDimIt, localGranuleIndex, clampedStride, granuleStopIndex);
376
377 // Do the constrained read and copy it into this output buffer
378 agg_util::AggregationUtil::addDatasetArrayDataToAggregationOutputArray(*this, // into the output buffer of this object
379 nextOutputBufferElementIndex, // into the next open slice
380 getGranuleTemplateArray(), // constraints we just setup
381 name(), // aggvar name
382 const_cast<AggMemberDataset&>(*pCurrDataset), // Dataset who's DDS should be searched
383 getArrayGetterInterface(), DEBUG_CHANNEL);
384
385 // Jump output buffer index forward by the amount we added.
386 nextOutputBufferElementIndex += getGranuleTemplateArray().length();
387 currDatasetWasRead = true;
388
389 BESDEBUG_FUNC(DEBUG_CHANNEL,
390 " The granule index " << currDatasetIndex << " was read with constraints and copied into the aggregation output." << endl);
391 } // !currDatasetWasRead
392 } // for loop over outerDim
393 } // try
394
395 catch (AggregationException& ex) {
396 THROW_NCML_PARSE_ERROR(-1, ex.what());
397 }
398
399}
400
401} // namespace agg_util
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
virtual unsigned int getCachedDimensionSize(const std::string &dimName) const =0
static void addDatasetArrayDataToAggregationOutputArray(libdap::Array &oOutputArray, unsigned int atIndex, const libdap::Array &constrainedTemplateArray, const string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const string &debugChannel)
static void transferArrayConstraints(libdap::Array *pToArray, const libdap::Array &fromArray, bool skipFirstFromDim, bool skipFirstToDim, bool printDebug=false, const std::string &debugChannel="agg_util")
static void printDimensions(std::ostream &os, const libdap::Array &fromArray)
static libdap::Array * readDatasetArrayDataForAggregation(const libdap::Array &constrainedTemplateArray, const std::string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const std::string &debugChannel)
const AMDList & getDatasetList() const
const ArrayGetterInterface & getArrayGetterInterface() const
virtual ArrayJoinExistingAggregation * ptr_duplicate()
ArrayJoinExistingAggregation(const libdap::Array &granuleTemplate, AMDList memberDatasets, std::unique_ptr< ArrayGetterInterface > arrayGetter, const Dimension &joinDim)
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
std::string toString() const
Definition: Dimension.cc:45