bes Updated for version 3.20.13
BESDapResponseBuilder.cc
1// -*- mode: c++; c-basic-offset:4 -*-
2
3// This file is part of libdap, A C++ implementation of the OPeNDAP Data
4// Access Protocol.
5
6// Copyright (c) 2011 OPeNDAP, Inc.
7// Author: James Gallagher <jgallagher@opendap.org>
8//
9// This library is free software; you can redistribute it and/or
10// modify it under the terms of the GNU Lesser General Public
11// License as published by the Free Software Foundation; either
12// version 2.1 of the License, or (at your option) any later version.
13//
14// This library is distributed in the hope that it will be useful,
15// but WITHOUT ANY WARRANTY; without even the implied warranty of
16// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17// Lesser General Public License for more details.
18//
19// You should have received a copy of the GNU Lesser General Public
20// License along with this library; if not, write to the Free Software
21// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22//
23// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24
25#include "config.h"
26
27#include <signal.h>
28#include <unistd.h>
29#include <sys/stat.h>
30
31#ifdef HAVE_UUID_UUID_H
32#include <uuid/uuid.h> // used to build CID header value for data ddx
33#elif defined(HAVE_UUID_H)
34#include <uuid.h>
35#else
36#error "Could not find UUID library header"
37#endif
38
39
40#ifndef WIN32
41#include <sys/wait.h>
42#else
43#include <io.h>
44#include <fcntl.h>
45#include <process.h>
46#endif
47
48#include <iostream>
49#include <string>
50#include <sstream>
51#include <fstream>
52
53#include <cstring>
54#include <ctime>
55
56//#define DODS_DEBUG
57#define CLEAR_LOCAL_DATA
58#undef USE_LOCAL_TIMEOUT_SCHEME
59
60#include <libdap/DAS.h>
61#include <libdap/DDS.h>
62#include <libdap/Structure.h>
63#include <libdap/ConstraintEvaluator.h>
64#include <libdap/DDXParserSAX2.h>
65#include <libdap/Ancillary.h>
66#include <libdap/XDRStreamMarshaller.h>
67#include <libdap/XDRFileUnMarshaller.h>
68
69#include <libdap/DMR.h>
70#include <libdap/D4Group.h>
71#include <libdap/D4Attributes.h>
72#include <libdap/XMLWriter.h>
73#include <libdap/D4AsyncUtil.h>
74#include <libdap/D4StreamMarshaller.h>
75#include <libdap/chunked_ostream.h>
76#include <libdap/chunked_istream.h>
77#include <libdap/D4ConstraintEvaluator.h>
78#include <libdap/D4FunctionEvaluator.h>
79#include <libdap/D4BaseTypeFactory.h>
80
81#include <libdap/ServerFunctionsList.h>
82
83#include <libdap/mime_util.h> // for last_modified_time() and rfc_822_date()
84#include <libdap/escaping.h>
85#include <libdap/util.h>
86// #include <d4_function/D4FunctionEvaluator.h>
87
88#include "DapUtils.h"
89
90#if USE_LOCAL_TIMEOUT_SCHEME
91#ifndef WIN32
92#include <libdap/SignalHandler.h>
93#include <libdap/EventHandler.h>
94#include <libdap/AlarmHandler.h>
95#endif
96#endif
97
98#include "TheBESKeys.h"
99#include "BESDapResponseBuilder.h"
100#include "BESContextManager.h"
101#include "BESDapFunctionResponseCache.h"
102#include "BESStoredDapResultCache.h"
103
104
105#include "BESResponseObject.h"
106#include "BESDDSResponse.h"
107#include "BESDataDDSResponse.h"
108#include "BESDMRResponse.h"
109#include "BESDataHandlerInterface.h"
110#include "BESInternalFatalError.h"
111#include "BESSyntaxUserError.h"
112#include "BESDataNames.h"
113
114#include "BESRequestHandler.h"
115#include "BESRequestHandlerList.h"
116#include "BESNotFoundError.h"
117
118#include "BESUtil.h"
119#include "BESDebug.h"
120#include "BESLog.h"
121#include "BESStopWatch.h"
122#include "DapFunctionUtils.h"
123#include "RequestServiceTimer.h"
124
125using namespace std;
126using namespace libdap;
127
128const string CRLF = "\r\n"; // Change here, expr-test.cc
129const string BES_KEY_TIMEOUT_CANCEL = "BES.CancelTimeoutOnSend";
130
131#define MODULE "dap"
132#define prolog std::string("BESDapResponseBuilder::").append(__func__).append("() - ")
133
134BESDapResponseBuilder::~BESDapResponseBuilder()
135{
136#if USE_LOCAL_TIMEOUT_SCHEME
137 // If an alarm was registered, delete it. The register code in SignalHandler
138 // always deletes the old alarm handler object, so only the one returned by
139 // remove_handler needs to be deleted at this point.
140 delete dynamic_cast<AlarmHandler*>(SignalHandler::instance()->remove_handler(SIGALRM));
141#endif
142}
143
151{
152 return d_dap2ce;
153}
154
166{
167 d_dap2ce = www2id(_ce, "%", "%20");
168}
169
174{
175 return d_dap4ce;
176}
177
189{
190 d_dap4ce = www2id(_ce, "%", "%20");
191}
192
197{
198 return d_dap4function;
199}
200
213{
214 d_dap4function = www2id(_func, "%", "%20");
215}
216
217std::string BESDapResponseBuilder::get_store_result() const
218{
219 return d_store_result;
220}
221
222void BESDapResponseBuilder::set_store_result(std::string _sr)
223{
224 d_store_result = _sr;
225 BESDEBUG(MODULE, prolog << "store_result: " << _sr << endl);
226}
227
228std::string BESDapResponseBuilder::get_async_accepted() const
229{
230 return d_async_accepted;
231}
232
233void BESDapResponseBuilder::set_async_accepted(std::string _aa)
234{
235 d_async_accepted = _aa;
236 BESDEBUG(MODULE, prolog << "set_async_accepted() - async_accepted: " << _aa << endl);
237}
238
248{
249 return d_dataset;
250}
251
263{
264 d_dataset = www2id(ds, "%", "%20");
265}
266
281static string::size_type find_closing_paren(const string &ce, string::size_type pos)
282{
283 // Iterate over the string finding all ( or ) characters until the matching ) is found.
284 // For each ( found, increment count. When a ) is found and count is zero, it is the
285 // matching closing paren, otherwise, decrement count and keep looking.
286 int count = 1;
287 do {
288 pos = ce.find_first_of("()", pos + 1);
289 if (pos == string::npos){
290 stringstream msg;
291 msg << "Expected to find a matching closing parenthesis in: " << ce;
292 throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
293 }
294
295 if (ce[pos] == '(')
296 ++count;
297 else
298 --count; // must be ')'
299
300 } while (count > 0);
301
302 return pos;
303}
304
311void BESDapResponseBuilder::split_ce(ConstraintEvaluator &eval, const string &expr)
312{
313 BESDEBUG(MODULE, prolog << "source expression: " << expr << endl);
314
315 string ce;
316 if (!expr.empty())
317 ce = expr;
318 else
319 ce = d_dap2ce;
320
321 string btp_function_ce = "";
322 string::size_type pos = 0;
323
324 // This hack assumes that the functions are listed first. Look for the first
325 // open paren and the last closing paren to accommodate nested function calls
326 string::size_type first_paren = ce.find("(", pos);
327 string::size_type closing_paren = string::npos;
328 if (first_paren != string::npos) closing_paren = find_closing_paren(ce, first_paren); //ce.find(")", pos);
329
330 while (first_paren != string::npos && closing_paren != string::npos) {
331 // Maybe a BTP function; get the name of the potential function
332 string name = ce.substr(pos, first_paren - pos);
333
334 // is this a BTP function
335 btp_func f;
336 if (eval.find_function(name, &f)) {
337 // Found a BTP function
338 if (!btp_function_ce.empty()) btp_function_ce += ",";
339 btp_function_ce += ce.substr(pos, closing_paren + 1 - pos);
340 ce.erase(pos, closing_paren + 1 - pos);
341 if (ce[pos] == ',') ce.erase(pos, 1);
342 }
343 else {
344 pos = closing_paren + 1;
345 // exception?
346 if (pos < ce.length() && ce.at(pos) == ',') ++pos;
347 }
348
349 first_paren = ce.find("(", pos);
350 closing_paren = ce.find(")", pos);
351 }
352
353 d_dap2ce = ce;
354 d_btp_func_ce = btp_function_ce;
355
356 BESDEBUG(MODULE, prolog << "Modified constraint: " << d_dap2ce << endl);
357 BESDEBUG(MODULE, prolog << "BTP Function part: " << btp_function_ce << endl);
358 BESDEBUG(MODULE, prolog << "END" << endl);
359}
360
367static void
368throw_if_dap2_response_too_big(DDS *dds)
369{
370 if (dds->too_big()) {
371#if 0
372 stringstream msg;
373 msg << "The Request for " << request_size / 1024 << " kilobytes is too large; ";
374 msg << "requests on this server are limited to "
375 + long_to_string(dds->get_response_limit() /1024) + "KB.";
376 throw Error(msg.str());
377#endif
378 stringstream msg;
379 msg << "The submitted DAP2 request will generate a " << dds->get_request_size_kb(true);
380 msg << " kilobyte response, which is too large. ";
381 msg << "The maximum response size for this server is limited to " << dds->get_response_limit_kb();
382 msg << " kilobytes.";
383 throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
384 }
385}
386
387static void
388throw_if_dap4_response_too_big(DMR &dmr)
389{
390 if (dmr.too_big()) {
391 stringstream msg;
392 msg << "The submitted DAP4 request will generate a " << dmr.request_size_kb(true);
393 msg << " kilobyte response, which is too large. ";
394 msg << "The maximum response size for this server is limited to " << dmr.response_limit_kb();
395 msg << " kilobytes.";
396 throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
397 }
398}
399
414void BESDapResponseBuilder::send_das(ostream &out, DAS &das, bool with_mime_headers) const
415{
416 if (with_mime_headers) set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), "2.0");
417
418 das.print(out);
419
420 out << flush;
421}
422
440void BESDapResponseBuilder::send_das(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool constrained,
441 bool with_mime_headers)
442{
443#if USE_LOCAL_TIMEOUT_SCHEME
444 // Set up the alarm.
445 establish_timeout(out);
446 dds.set_timeout(d_timeout);
447#endif
448
449 // Verify the request hasn't exceeded bes_timeout.
450 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog +" ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
451
452 if (!constrained) {
453 if (with_mime_headers) set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), "2.0");
454
456
457 (*dds)->print_das(out);
458 out << flush;
459
460 return;
461 }
462
463 split_ce(eval);
464
465 // If there are functions, parse them and eval.
466 // Use that DDS and parse the non-function ce
467 // Serialize using the second ce and the second dds
468 if (!d_btp_func_ce.empty()) {
469 ConstraintEvaluator func_eval;
470 BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
471
472 DDS *fdds = 0; // nullptr
473 if (responseCache && responseCache->can_be_cached(*dds, get_btp_func_ce())) {
474 fdds = responseCache->get_or_cache_dataset(*dds, get_btp_func_ce());
475 }
476 else {
477 func_eval.parse_constraint(get_btp_func_ce(), **dds);
478 fdds = func_eval.eval_function_clauses(**dds);
479 }
480
481 delete *dds; *dds = 0;
482 *dds = fdds;
483
484 if (with_mime_headers)
485 set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
486
487 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog +"ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
489
490 (*dds)->print_das(out);
491 }
492 else {
493 eval.parse_constraint(d_dap2ce, **dds); // Throws Error if the ce doesn't parse.
494
495 if (with_mime_headers)
496 set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
497
498 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog +"ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
500
501 (*dds)->print_das(out);
502 }
503
504 out << flush;
505}
506
507
526void BESDapResponseBuilder::send_dds(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool constrained,
527 bool with_mime_headers)
528{
529 if (!constrained) {
530 if (with_mime_headers)
531 set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
532
533 // Verify the request hasn't exceeded bes_timeout, and disable timeout if allowed.
534 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog +"ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
536
537 (*dds)->print(out);
538 out << flush;
539 return;
540 }
541
542#if USE_LOCAL_TIMEOUT_SCHEME
543 // Set up the alarm.
544 establish_timeout(out);
545 dds.set_timeout(d_timeout);
546#endif
547
548 // Split constraint into two halves
549 split_ce(eval);
550
551 // If there are functions, parse them and eval.
552 // Use that DDS and parse the non-function ce
553 // Serialize using the second ce and the second dds
554 if (!d_btp_func_ce.empty()) {
555 ConstraintEvaluator func_eval;
556
557 BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
558
559 DDS *fdds = 0; // nulll_ptr
560 if (responseCache && responseCache->can_be_cached(*dds, get_btp_func_ce())) {
561 fdds = responseCache->get_or_cache_dataset(*dds, get_btp_func_ce());
562 }
563 else {
564 func_eval.parse_constraint(get_btp_func_ce(), **dds);
565 fdds = func_eval.eval_function_clauses(**dds);
566 }
567
568 delete *dds; *dds = 0;
569 *dds = fdds;
570
571 // Server functions might mark variables to use their read()
572 // methods. Clear that so the CE in d_dap2ce will control what is
573 // sent. If that is empty (there was only a function call) all
574 // of the variables in the intermediate DDS (i.e., the function
575 // result) will be sent.
576 (*dds)->mark_all(false);
577
578 // This next step utilizes a well known static method (so really it's a function;),
579 // promote_function_output_structures() to look for
580 // one or more top level Structures whose name indicates (by way of ending with
581 // "_unwrap") that their contents should be promoted (aka moved) to the top level.
582 // This is in support of a hack around the current API where server side functions
583 // may only return a single DAP object and not a collection of objects. The name suffix
584 // "_unwrap" is used as a signal from the function to the the various response
585 // builders and transmitters that the representation needs to be altered before
586 // transmission, and that in fact is what happens in our friend
587 // promote_function_output_structures()
588 promote_function_output_structures(*dds);
589
590 eval.parse_constraint(d_dap2ce, **dds);
591
592 if (with_mime_headers)
593 set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
594
595 // Verify the request hasn't exceeded bes_timeout, and disable timeout if allowed.
596 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog +"ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
598
599 (*dds)->print_constrained(out);
600 }
601 else {
602 eval.parse_constraint(d_dap2ce, **dds); // Throws Error if the ce doesn't parse.
603
604 if (with_mime_headers)
605 set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset),(*dds)->get_dap_version());
606
607 // Verify the request hasn't exceeded bes_timeout, and disable timeout if allowed.
608 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog +"ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
610
611 (*dds)->print_constrained(out);
612 }
613
614 out << flush;
615}
616
617#ifdef DAP2_STORED_RESULTS
632bool BESDapResponseBuilder::store_dap2_result(ostream &out, DDS &dds, ConstraintEvaluator &eval)
633{
634 if (get_store_result().empty()) return false;
635
636 string serviceUrl = get_store_result();
637
638 XMLWriter xmlWrtr;
639 D4AsyncUtil d4au;
640
641 // FIXME Keys should be read in initialize(). Also, I think the D4AsyncUtil should
642 // be removed from libdap - it is much more about how the BES processes these kinds
643 // of operations. Change this when working on the response caching for ODSIP. But...
644 // do we really need to put the style sheet in the bes.conf file? Should it be baked
645 // into the code (because we don't want people to change it)?
646 bool found;
647 string *stylesheet_ref = 0, ss_ref_value;
648 TheBESKeys::TheKeys()->get_value(D4AsyncUtil::STYLESHEET_REFERENCE_KEY, ss_ref_value, found);
649 if (found && !ss_ref_value.empty()) {
650 stylesheet_ref = &ss_ref_value;
651 }
652
654 if (resultCache == NULL) {
655
661 string msg = "The Stored Result request cannot be serviced. ";
662 msg += "Unable to acquire StoredResultCache instance. ";
663 msg += "This is most likely because the StoredResultCache is not (correctly) configured.";
664
665 BESDEBUG(MODULE, prolog << "[WARNING] " << msg << endl);
666
667 d4au.writeD4AsyncResponseRejected(xmlWrtr, UNAVAILABLE, msg, stylesheet_ref);
668 out << xmlWrtr.get_doc();
669 out << flush;
670
671 BESDEBUG(MODULE,prolog << "Sent AsyncRequestRejected" << endl);
672 }
673 else if (get_async_accepted().length() != 0) {
674
678 BESDEBUG(MODULE, prolog << "serviceUrl="<< serviceUrl << endl);
679
681 string storedResultId = "";
682 storedResultId = resultCache->store_dap2_result(dds, get_ce(), this, &eval);
683
684 BESDEBUG(MODULE, prolog << "storedResultId='"<< storedResultId << "'" << endl);
685
686 string targetURL = BESUtil::assemblePath(serviceUrl, storedResultId);
687 BESDEBUG(MODULE, prolog << "targetURL='"<< targetURL << "'" << endl);
688
689 XMLWriter xmlWrtr;
690 d4au.writeD4AsyncAccepted(xmlWrtr, 0, 0, targetURL, stylesheet_ref);
691 out << xmlWrtr.get_doc();
692 out << flush;
693
694 BESDEBUG(MODULE, prolog << "Sent DAP4 AsyncAccepted response" << endl);
695 }
696 else {
701 d4au.writeD4AsyncRequired(xmlWrtr, 0, 0, stylesheet_ref);
702 out << xmlWrtr.get_doc();
703 out << flush;
704
705 BESDEBUG(MODULE, prolog << "Sent DAP4 AsyncRequired response" << endl);
706 }
707
708 return true;
709
710}
711#endif
712
716void BESDapResponseBuilder::serialize_dap2_data_dds(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool ce_eval)
717{
718 BESStopWatch sw;
719 if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
720
721 // Verify the request hasn't exceeded bes_timeout, and disable timeout if allowed.
722 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog +"ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
724
725 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
726
727 (*dds)->print_constrained(out);
728 out << "Data:\n";
729 out << flush;
730
731 XDRStreamMarshaller m(out);
732
733 // Send all variables in the current projection (send_p())
734 for (DDS::Vars_iter i = (*dds)->var_begin(); i != (*dds)->var_end(); i++) {
735 if ((*i)->send_p()) {
736 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog +"ERROR: bes-timeout expired before transmit " + (*i)->name(), __FILE__, __LINE__);
737 (*i)->serialize(eval, **dds, m, ce_eval);
738#ifdef CLEAR_LOCAL_DATA
739 (*i)->clear_local_data();
740#endif
741 }
742 }
743
744 BESDEBUG(MODULE, prolog << "END" << endl);
745}
746
747#ifdef DAP2_STORED_RESULTS
756void BESDapResponseBuilder::serialize_dap2_data_ddx(ostream &out, DDS **dds, ConstraintEvaluator &eval,
757 const string &boundary, const string &start, bool ce_eval)
758{
759 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
760
761 // Write the MPM headers for the DDX (text/xml) part of the response
762 libdap::set_mime_ddx_boundary(out, boundary, start, dods_ddx, x_plain);
763
764 // Make cid
765 uuid_t uu;
766 uuid_generate(uu);
767 char uuid[37];
768 uuid_unparse(uu, uuid.data());
769 char domain[256];
770 if (getdomainname(domain, 255) != 0 || strlen(domain) == 0) strncpy(domain, "opendap.org", 255);
771
772 string cid = string(uuid.data()) + "@" + string(domain.data());
773
774 // Send constrained DDX with a data blob reference.
775 // Note: CID passed but ignored jhrg 10/20/15
776 (*dds)->print_xml_writer(out, true, cid);
777
778 // write the data part mime headers here
779 set_mime_data_boundary(out, boundary, cid, dods_data_ddx /* old value dap4_data*/, x_plain);
780
781 XDRStreamMarshaller m(out);
782
783 conditional_timeout_cancel();
784
785
786 // Send all variables in the current projection (send_p()).
787 for (DDS::Vars_iter i = (*dds)->var_begin(); i != (*dds)->var_end(); i++) {
788 if ((*i)->send_p()) {
789 (*i)->serialize(eval, **dds, m, ce_eval);
790#ifdef CLEAR_LOCAL_DATA
791 (*i)->clear_local_data();
792#endif
793 }
794 }
795
796 BESDEBUG(MODULE, prolog << "END" << endl);
797}
798#endif
799
816#if 0
817void BESDapResponseBuilder::remove_timeout() const
818{
819#if USE_LOCAL_TIMEOUT_SCHEME
820 alarm(0);
821#endif
822}
823#endif
824
838libdap::DDS *
840{
841 BESDEBUG(MODULE, prolog << "BEGIN"<< endl);
842
843 dhi.first_container();
844
845 BESDDSResponse *bdds = dynamic_cast<BESDDSResponse *>(obj);
846 if (!bdds) throw BESInternalFatalError("Expected a BESDDSResponse instance", __FILE__, __LINE__);
847
848 DDS *dds = bdds->get_dds();
849
850 set_dataset_name(dds->filename());
851 set_ce(dhi.data[POST_CONSTRAINT]);
852 set_async_accepted(dhi.data[ASYNC]);
853 set_store_result(dhi.data[STORE_RESULT]);
854
855 ConstraintEvaluator &eval = bdds->get_ce();
856
857 // Split constraint into two halves
858 split_ce(eval);
859
860 // If there are functions, parse them and eval.
861 // Use that DDS and parse the non-function ce
862 // Serialize using the second ce and the second dds
863 if (!d_btp_func_ce.empty()) {
864 BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
865
866 ConstraintEvaluator func_eval;
867 DDS *fdds = 0; // nulll_ptr
868 if (responseCache && responseCache->can_be_cached(dds, get_btp_func_ce())) {
869 fdds = responseCache->get_or_cache_dataset(dds, get_btp_func_ce());
870 }
871 else {
872 func_eval.parse_constraint(get_btp_func_ce(), *dds);
873 fdds = func_eval.eval_function_clauses(*dds);
874 }
875
876 delete dds; // Delete so that we can ...
877 bdds->set_dds(fdds); // Transfer management responsibility
878 dds = fdds;
879
880 dds->mark_all(false);
881
882 promote_function_output_structures(dds);
883 }
884
885 eval.parse_constraint(d_dap2ce, *dds); // Throws Error if the ce doesn't parse.
886 BESDEBUG(MODULE, prolog << "END"<< endl);
887
888 return dds;
889}
890
908libdap::DDS *
910{
911 BESStopWatch sw;
912 if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
913
914 BESDEBUG(MODULE, prolog << "BEGIN"<< endl);
915
916 dhi.first_container();
917
918 BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(obj);
919 if (!bdds) throw BESInternalFatalError("Expected a BESDataDDSResponse instance", __FILE__, __LINE__);
920
921 DDS *dds = bdds->get_dds();
922
923 set_dataset_name(dds->filename());
924 set_ce(dhi.data[POST_CONSTRAINT]);
925 set_async_accepted(dhi.data[ASYNC]);
926 set_store_result(dhi.data[STORE_RESULT]);
927
928
929 // This function is used by all fileout modules and they need to include the attributes in data access.
930 // So obtain the attributes if necessary. KY 2019-10-30
931 if(bdds->get_ia_flag() == false) {
932 BESRequestHandler *besRH = BESRequestHandlerList::TheList()->find_handler(dhi.container->get_container_type());
933 besRH->add_attributes(dhi);
934 }
935
936 ConstraintEvaluator &eval = bdds->get_ce();
937
938 // Split constraint into two halves; stores the function and non-function parts in this instance.
939 split_ce(eval);
940
941 // If there are functions, parse them and eval.
942 // Use that DDS and parse the non-function ce
943 // Serialize using the second ce and the second dds
944 if (!get_btp_func_ce().empty()) {
945 BESDEBUG(MODULE,prolog << "Found function(s) in CE: " << get_btp_func_ce() << endl);
946
947 BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
948
949 ConstraintEvaluator func_eval;
950 DDS *fdds = nullptr;
951 if (responseCache && responseCache->can_be_cached(dds, get_btp_func_ce())) {
952 fdds = responseCache->get_or_cache_dataset(dds, get_btp_func_ce());
953 }
954 else {
955 func_eval.parse_constraint(get_btp_func_ce(), *dds);
956 fdds = func_eval.eval_function_clauses(*dds);
957 }
958
959 delete dds; // Delete so that we can ...
960 bdds->set_dds(fdds); // Transfer management responsibility
961 dds = fdds;
962
963 // Server functions might mark (i.e. setting send_p) so variables will use their read()
964 // methods. Clear that so the CE in d_dap2ce will control what is
965 // sent. If that is empty (there was only a function call) all
966 // of the variables in the intermediate DDS (i.e., the function
967 // result) will be sent.
968 dds->mark_all(false);
969
970 // Look for one or more top level Structures whose name indicates (by way of ending with
971 // "_uwrap") that their contents should be moved to the top level.
972 //
973 // This is in support of a hack around the current API where server side functions
974 // may only return a single DAP object and not a collection of objects. The name suffix
975 // "_unwrap" is used as a signal from the function to the the various response
976 // builders and transmitters that the representation needs to be altered before
977 // transmission, and that in fact is what happens in our friend
978 // promote_function_output_structures()
979 promote_function_output_structures(dds);
980 }
981
982 // evaluate the rest of the CE - the part that follows the function calls.
983 eval.parse_constraint(get_ce(), *dds);
984
985 dds->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
986
987 throw_if_dap2_response_too_big(dds);
988
989 // Iterate through the variables in the DataDDS and read
990 // in the data if the variable has the send flag set.
991 for (DDS::Vars_iter i = dds->var_begin(), e = dds->var_end(); i != e; ++i) {
992 if ((*i)->send_p()) {
993 try {
994 (*i)->intern_data(eval, *dds);
995 }
996 catch(std::exception &e) {
997 throw BESSyntaxUserError(string("Caught a C++ standard exception while working on '") + (*i)->name() + "' The error was: " + e.what(), __FILE__, __LINE__);
998 }
999 }
1000 }
1001
1002 BESDEBUG(MODULE, prolog << "END"<< endl);
1003
1004 return dds;
1005}
1006
1019void BESDapResponseBuilder::send_dap2_data(ostream &data_stream, DDS **dds, ConstraintEvaluator &eval,
1020 bool with_mime_headers)
1021{
1022 BESDEBUG(MODULE, prolog << "BEGIN"<< endl);
1023
1024#if USE_LOCAL_TIMEOUT_SCHEME
1025 // Set up the alarm.
1026 establish_timeout(data_stream);
1027 dds.set_timeout(d_timeout);
1028#endif
1029
1030 // Split constraint into two halves
1031 split_ce(eval);
1032
1033 // If there are functions, parse them and eval.
1034 // Use that DDS and parse the non-function ce
1035 // Serialize using the second ce and the second dds
1036 if (!get_btp_func_ce().empty()) {
1037 BESDEBUG(MODULE,prolog << "Found function(s) in CE: " << get_btp_func_ce() << endl);
1038
1039 BESDapFunctionResponseCache *response_cache = BESDapFunctionResponseCache::get_instance();
1040
1041 ConstraintEvaluator func_eval;
1042 DDS *fdds = 0; // nulll_ptr
1043 if (response_cache && response_cache->can_be_cached(*dds, get_btp_func_ce())) {
1044 fdds = response_cache->get_or_cache_dataset(*dds, get_btp_func_ce());
1045 }
1046 else {
1047 func_eval.parse_constraint(get_btp_func_ce(), **dds);
1048 fdds = func_eval.eval_function_clauses(**dds);
1049 }
1050
1051 delete *dds; *dds = 0;
1052 *dds = fdds;
1053
1054 (*dds)->mark_all(false);
1055
1056 promote_function_output_structures(*dds);
1057
1058 // evaluate the rest of the CE - the part that follows the function calls.
1059 eval.parse_constraint(get_ce(), **dds);
1060
1061 (*dds)->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1062
1063 throw_if_dap2_response_too_big(*dds);
1064
1065 if (with_mime_headers)
1066 set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1067
1068#if STORE_DAP2_RESULT_FEATURE
1069 // This means: if we are not supposed to store the result, then serialize it.
1070 if (!store_dap2_result(data_stream, **dds, eval)) {
1071 serialize_dap2_data_dds(data_stream, dds, eval, true /* was 'false'. jhrg 3/10/15 */);
1072 }
1073#else
1074 serialize_dap2_data_dds(data_stream, dds, eval, true /* was 'false'. jhrg 3/10/15 */);
1075#endif
1076
1077 }
1078 else {
1079 BESDEBUG(MODULE, prolog << "Simple constraint" << endl);
1080
1081 eval.parse_constraint(get_ce(), **dds); // Throws Error if the ce doesn't parse.
1082
1083 (*dds)->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1084
1085 throw_if_dap2_response_too_big(*dds);
1086
1087 if (with_mime_headers)
1088 set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1089
1090#if STORE_DAP2_RESULT_FEATURE
1091 // This means: if we are not supposed to store the result, then serialize it.
1092 if (!store_dap2_result(data_stream, **dds, eval)) {
1093 serialize_dap2_data_dds(data_stream, dds, eval);
1094 }
1095#else
1096 serialize_dap2_data_dds(data_stream, dds, eval);
1097#endif
1098 }
1099
1100 dap_utils::log_request_and_memory_size(dds);
1101
1102#if 0
1103 // This change addresses an issue on OSX where the size returned by getrusage() is in bytes, not kb.
1104 // The BESUtil::get_current_memory_usage() function sorts the problem. jhrg 4/6/22
1105 struct rusage usage;
1106 int usage_val;
1107 usage_val = getrusage(RUSAGE_SELF, &usage);
1108
1109 if (usage_val == 0){ //if getrusage() was successful, out put both sizes
1110 long mem_size = usage.ru_maxrss; // get the max size (man page says it is in kilobytes)
1111 INFO_LOG(prolog << "request size: "<< req_size << "KB -|- memory used by process: " << mem_size << "KB" << endl);
1112 }
1113 else { //if the getrusage() wasn't successful, only output the request size
1114 INFO_LOG(prolog << "request size: "<< req_size << "KB" << endl );
1115 }
1116#endif
1117
1118 data_stream << flush;
1119
1120 BESDEBUG(MODULE, prolog << "END"<< endl);
1121
1122}
1123
1124void BESDapResponseBuilder::send_dap2_data(BESDataHandlerInterface &dhi, DDS **dds, ConstraintEvaluator &eval,
1125 bool with_mime_headers)
1126{
1127 BESDEBUG(MODULE, prolog << "BEGIN"<< endl);
1128
1129 ostream & data_stream = dhi.get_output_stream();
1130#if USE_LOCAL_TIMEOUT_SCHEME
1131 // Set up the alarm.
1132 establish_timeout(data_stream);
1133 dds.set_timeout(d_timeout);
1134#endif
1135
1136 // Split constraint into two halves
1137 split_ce(eval);
1138
1139 // If there are functions, parse them and eval.
1140 // Use that DDS and parse the non-function ce
1141 // Serialize using the second ce and the second dds
1142 if (!get_btp_func_ce().empty()) {
1143 BESDEBUG(MODULE, prolog << "Found function(s) in CE: " << get_btp_func_ce() << endl);
1144
1145 // Server-side functions need to include the attributes in data access.
1146 // So obtain the attributes if necessary. KY 2019-10-30
1147 {
1148 BESResponseObject *response = dhi.response_handler->get_response_object();
1149 auto *bdds = dynamic_cast<BESDataDDSResponse *> (response);
1150 if (!bdds)
1151 throw BESInternalError("cast error", __FILE__, __LINE__);
1152
1153 if(!bdds->get_ia_flag()) {
1154 BESRequestHandler *besRH = BESRequestHandlerList::TheList()->find_handler(dhi.container->get_container_type());
1155 besRH->add_attributes(dhi);
1156 }
1157 }
1158
1159 BESDapFunctionResponseCache *response_cache = BESDapFunctionResponseCache::get_instance();
1160 ConstraintEvaluator func_eval;
1161 DDS *fdds = nullptr;
1162 if (response_cache && response_cache->can_be_cached(*dds, get_btp_func_ce())) {
1163 fdds = response_cache->get_or_cache_dataset(*dds, get_btp_func_ce());
1164 }
1165 else {
1166 func_eval.parse_constraint(get_btp_func_ce(), **dds);
1167 fdds = func_eval.eval_function_clauses(**dds);
1168 }
1169
1170 delete *dds;
1171 *dds = nullptr;
1172 *dds = fdds;
1173
1174 (*dds)->mark_all(false);
1175
1176 promote_function_output_structures(*dds);
1177
1178 // evaluate the rest of the CE - the part that follows the function calls.
1179 eval.parse_constraint(get_ce(), **dds);
1180
1181 (*dds)->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1182
1183 throw_if_dap2_response_too_big(*dds);
1184
1185 if (with_mime_headers)
1186 set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1187
1188#if STORE_DAP2_RESULT_FEATURE
1189 // This means: if we are not supposed to store the result, then serialize it.
1190 if (!store_dap2_result(data_stream, **dds, eval)) {
1191 serialize_dap2_data_dds(data_stream, dds, eval, true /* was 'false'. jhrg 3/10/15 */);
1192 }
1193#else
1194 serialize_dap2_data_dds(data_stream, dds, eval, true /* was 'false'. jhrg 3/10/15 */);
1195#endif
1196
1197 }
1198 else {
1199 BESDEBUG(MODULE, prolog << "Simple constraint" << endl);
1200
1201 eval.parse_constraint(get_ce(), **dds); // Throws Error if the ce doesn't parse.
1202
1203 (*dds)->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1204
1205 throw_if_dap2_response_too_big(*dds);
1206
1207 if (with_mime_headers)
1208 set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1209
1210#if STORE_DAP2_RESULT_FEATURE
1211 // This means: if we are not supposed to store the result, then serialize it.
1212 if (!store_dap2_result(data_stream, **dds, eval)) {
1213 serialize_dap2_data_dds(data_stream, dds, eval);
1214 }
1215#else
1216 serialize_dap2_data_dds(data_stream, dds, eval);
1217#endif
1218 }
1219
1220 dap_utils::log_request_and_memory_size(dds);
1221
1222#if 0
1223 struct rusage usage;
1224 int usage_val;
1225 usage_val = getrusage(RUSAGE_SELF, &usage);
1226
1227 if (usage_val == 0){ //if getrusage() was successful, out put both sizes
1228 long mem_size = usage.ru_maxrss; // get the max size (man page says it is in kilobytes)
1229 INFO_LOG(prolog << "request size: "<< req_size << "KB -|- memory used by process: " << mem_size << "KB" << endl );
1230 }
1231 else { //if the getrusage() wasn't successful, only output the request size
1232 INFO_LOG(prolog << "request size: "<< req_size << "KB" << endl );
1233 }
1234#endif
1235
1236 data_stream << flush;
1237
1238 BESDEBUG(MODULE, prolog << "END"<< endl);
1239
1240}
1254void BESDapResponseBuilder::send_ddx(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool with_mime_headers)
1255{
1256 // Verify the request hasn't exceeded bes_timeout, and disable timeout if allowed.
1257 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog +"ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
1259
1260 if (d_dap2ce.empty()) {
1261 if (with_mime_headers)
1262 set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1263
1264 (*dds)->print_xml_writer(out, false /*constrained */, "");
1265 //dds.print(out);
1266 out << flush;
1267 return;
1268 }
1269
1270#if USE_LOCAL_TIMEOUT_SCHEME
1271 // Set up the alarm.
1272 establish_timeout(out);
1273 dds.set_timeout(d_timeout);
1274#endif
1275
1276 // Split constraint into two halves
1277 split_ce(eval);
1278
1279 // If there are functions, parse them and eval.
1280 // Use that DDS and parse the non-function ce
1281 // Serialize using the second ce and the second dds
1282 if (!d_btp_func_ce.empty()) {
1283 BESDapFunctionResponseCache *response_cache = BESDapFunctionResponseCache::get_instance();
1284
1285 ConstraintEvaluator func_eval;
1286 DDS *fdds = 0; // nulll_ptr
1287 if (response_cache && response_cache->can_be_cached(*dds, get_btp_func_ce())) {
1288 fdds = response_cache->get_or_cache_dataset(*dds, get_btp_func_ce());
1289 }
1290 else {
1291 func_eval.parse_constraint(get_btp_func_ce(), **dds);
1292 fdds = func_eval.eval_function_clauses(**dds);
1293 }
1294
1295 delete *dds; *dds = 0;
1296 *dds = fdds;
1297
1298 (*dds)->mark_all(false);
1299
1300 promote_function_output_structures(*dds);
1301
1302 eval.parse_constraint(d_dap2ce, **dds);
1303
1304 if (with_mime_headers)
1305 set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1306
1307 // Verify the request hasn't exceeded bes_timeout, and disable timeout if allowed.
1308 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog +"ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
1310
1311 (*dds)->print_xml_writer(out, true, "");
1312 }
1313 else {
1314 eval.parse_constraint(d_dap2ce, **dds); // Throws Error if the ce doesn't parse.
1315
1316 if (with_mime_headers)
1317 set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1318
1319 // Verify the request hasn't exceeded bes_timeout, and disable timeout if allowed.
1320 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog +"ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
1322
1323 // dds.print_constrained(out);
1324 (*dds)->print_xml_writer(out, true, "");
1325 }
1326
1327 out << flush;
1328}
1329
1330void BESDapResponseBuilder::send_dmr(ostream &out, DMR &dmr, bool with_mime_headers)
1331{
1332 // If the CE is not empty, parse it. The projections, etc., are set as a side effect.
1333 // If the parser returns false, the expression did not parse. The parser may also
1334 // throw Error
1335 if (!d_dap4ce.empty()) {
1336
1337 BESDEBUG(MODULE, prolog << "Parsing DAP4 constraint: '"<< d_dap4ce << "'"<< endl);
1338
1339 D4ConstraintEvaluator parser(&dmr);
1340 bool parse_ok = parser.parse(d_dap4ce);
1341 if (!parse_ok){
1342 stringstream msg;
1343 msg << "Failed to parse the provided DAP4 server-side function expression: " << d_dap4function;
1344 throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
1345 }
1346 }
1347 // with an empty CE, send everything. Even though print_dap4() and serialize()
1348 // don't need this, other code may depend on send_p being set. This may change
1349 // if DAP4 has a separate function evaluation phase. jhrg 11/25/13
1350 else {
1351 dmr.root()->set_send_p(true);
1352 }
1353
1354 if (with_mime_headers) set_mime_text(out, dap4_dmr, x_plain, last_modified_time(d_dataset), dmr.dap_version());
1355
1356 // Verify the request hasn't exceeded bes_timeout, and disable timeout if allowed.
1357 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog +"ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
1359
1360 BESDEBUG(MODULE, prolog << "dmr.request_xml_base(): '"<< dmr.request_xml_base() << "' (dmr: " << (void *) &dmr << ")" << endl);
1361
1362 XMLWriter xml;
1363 dmr.print_dap4(xml, /*constrained &&*/!d_dap4ce.empty() /* true == constrained */);
1364 out << xml.get_doc() << flush;
1365}
1366
1367void BESDapResponseBuilder::send_dap4_data_using_ce(ostream &out, DMR &dmr, bool with_mime_headers)
1368{
1369 if (!d_dap4ce.empty()) {
1370 BESDEBUG(MODULE , "BESDapResponseBuilder::send_dap4_data_using_ce() - expression constraint is not empty. " <<endl);
1371 D4ConstraintEvaluator parser(&dmr);
1372 bool parse_ok = parser.parse(d_dap4ce);
1373 if (!parse_ok){
1374 stringstream msg;
1375 msg << "Failed to parse the provided DAP4 server-side function expression: " << d_dap4function;
1376 throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
1377 }
1378 }
1379 // with an empty CE, send everything. Even though print_dap4() and serialize()
1380 // don't need this, other code may depend on send_p being set. This may change
1381 // if DAP4 has a separate function evaluation phase. jhrg 11/25/13
1382 else {
1383 dmr.set_ce_empty(true);
1384 dmr.root()->set_send_p(true);
1385 }
1386
1387 dap_utils::log_request_and_memory_size(dmr);
1388 throw_if_dap4_response_too_big(dmr);
1389
1390 // The following block is for debugging purpose. KY 05/13/2020
1391#if !NDEBUG
1392 for (auto i = dmr.root()->var_begin(), e = dmr.root()->var_end(); i != e; ++i) {
1393 BESDEBUG(MODULE , prolog << (*i)->name() << endl);
1394 if ((*i)->send_p()) {
1395 BESDEBUG(MODULE , prolog << "Obtain data- " << (*i)->name() << endl);
1396 D4Attributes *d4_attrs = (*i)->attributes();
1397 BESDEBUG(MODULE , prolog << "Number of attributes " << d4_attrs << endl);
1398 for (auto ii = d4_attrs->attribute_begin(), ee = d4_attrs->attribute_end(); ii != ee; ++ii) {
1399 BESDEBUG(MODULE ,prolog << "Attribute name is " << (*ii)->name() << endl);
1400 }
1401 }
1402 }
1403#endif
1404
1405 if (!store_dap4_result(out, dmr)) {
1406 serialize_dap4_data(out, dmr, with_mime_headers);
1407 }
1408}
1409
1419{
1420 BESStopWatch sw;
1421 if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
1422 if (!d_dap4ce.empty()) {
1423 BESDEBUG(MODULE , prolog << "Expression constraint is not empty. " <<endl);
1424 D4ConstraintEvaluator parser(&dmr);
1425 bool parse_ok = parser.parse(d_dap4ce);
1426 if (!parse_ok){
1427 stringstream msg;
1428 msg << "Failed to parse the provided DAP4 server-side function expression: " << d_dap4function;
1429 throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
1430 }
1431 }
1432 // with an empty CE, send everything. Even though print_dap4() and serialize()
1433 // don't need this, other code may depend on send_p being set. This may change
1434 // if DAP4 has a separate function evaluation phase. jhrg 11/25/13
1435 else {
1436 dmr.set_ce_empty(true);
1437 dmr.root()->set_send_p(true);
1438 }
1439 throw_if_dap4_response_too_big(dmr);
1440}
1441
1442void BESDapResponseBuilder::send_dap4_data(ostream &out, DMR &dmr, bool with_mime_headers)
1443{
1444 // If a function was passed in with this request, evaluate it and use that DMR
1445 // for the remainder of this request.
1446 // TODO Add caching for these function invocations
1447 if (!d_dap4function.empty()) {
1448 D4BaseTypeFactory d4_factory;
1449 DMR function_result(&d4_factory, "function_results");
1450
1451 // Function modules load their functions onto this list. The list is
1452 // part of libdap, not the BES.
1453 if (!ServerFunctionsList::TheList()) {
1454 stringstream msg;
1455 msg << "The function expression could not be evaluated because ";
1456 msg << "there are no server-side functions defined on this server.";
1457 throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
1458 }
1459
1460 D4FunctionEvaluator parser(&dmr, ServerFunctionsList::TheList());
1461 bool parse_ok = parser.parse(d_dap4function);
1462 if (!parse_ok){
1463 stringstream msg;
1464 msg << "Failed to parse the provided DAP4 server-side function expression: " << d_dap4function;
1465 throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
1466 }
1467 parser.eval(&function_result);
1468
1469 // Now use the results of running the functions for the remainder of the
1470 // send_data operation.
1471 send_dap4_data_using_ce(out, function_result, with_mime_headers);
1472 }
1473 else {
1474 send_dap4_data_using_ce(out, dmr, with_mime_headers);
1475 }
1476}
1477
1481void BESDapResponseBuilder::serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers)
1482{
1483 BESStopWatch sw;
1484 if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
1485
1486 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
1487
1488 if (with_mime_headers) set_mime_binary(out, dap4_data, x_plain, last_modified_time(d_dataset), dmr.dap_version());
1489
1490 BESDEBUG(MODULE, prolog << "dmr.request_xml_base(): \"" << dmr.request_xml_base() << "\""<< endl);
1491
1492 // Write the DMR
1493 XMLWriter xml;
1494 dmr.print_dap4(xml, !d_dap4ce.empty());
1495
1496 // now make the chunked output stream; set the size to be at least chunk_size
1497 // but make sure that the whole of the xml plus the CRLF can fit in the first
1498 // chunk. (+2 for the CRLF bytes).
1499 chunked_ostream cos(out, max((unsigned int) CHUNK_SIZE, xml.get_doc_size() + 2));
1500
1501 // Verify the request hasn't exceeded bes_timeout, and disable timeout if allowed.
1502 RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog +"ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
1504
1505 // using flush means that the DMR and CRLF are in the first chunk.
1506 cos << xml.get_doc() << CRLF << flush;
1507
1508 // Write the data, chunked with checksums
1509 D4StreamMarshaller m(cos);
1510 dmr.root()->serialize(m, dmr, !d_dap4ce.empty());
1511#ifdef CLEAR_LOCAL_DATA
1512 dmr.root()->clear_local_data();
1513#endif
1514 cos << flush;
1515
1516 BESDEBUG(MODULE, prolog << "END" << endl);
1517}
1518
1533bool BESDapResponseBuilder::store_dap4_result(ostream &out, libdap::DMR &dmr)
1534{
1535 if (get_store_result().length() != 0) {
1536 string serviceUrl = get_store_result();
1537
1538 D4AsyncUtil d4au;
1539 XMLWriter xmlWrtr;
1540
1541 // FIXME See above comment for store dap2 result
1542 bool found;
1543 string *stylesheet_ref = 0, ss_ref_value;
1544 TheBESKeys::TheKeys()->get_value(D4AsyncUtil::STYLESHEET_REFERENCE_KEY, ss_ref_value, found);
1545 if (found && !ss_ref_value.empty()) {
1546 stylesheet_ref = &ss_ref_value;
1547 }
1548
1550 if (resultCache == NULL) {
1551
1557 string msg = "The Stored Result request cannot be serviced. ";
1558 msg += "Unable to acquire StoredResultCache instance. ";
1559 msg += "This is most likely because the StoredResultCache is not (correctly) configured.";
1560
1561 BESDEBUG(MODULE, prolog << "[WARNING] " << msg << endl);
1562 d4au.writeD4AsyncResponseRejected(xmlWrtr, UNAVAILABLE, msg, stylesheet_ref);
1563 out << xmlWrtr.get_doc();
1564 out << flush;
1565 BESDEBUG(MODULE, prolog << "Sent AsyncRequestRejected" << endl);
1566
1567 return true;
1568 }
1569
1570 if (get_async_accepted().length() != 0) {
1571
1575 BESDEBUG(MODULE, prolog << "serviceUrl="<< serviceUrl << endl);
1576
1577 string storedResultId = "";
1578 storedResultId = resultCache->store_dap4_result(dmr, get_ce(), this);
1579
1580 BESDEBUG(MODULE,prolog << "storedResultId='"<< storedResultId << "'" << endl);
1581
1582 string targetURL = BESUtil::assemblePath(serviceUrl, storedResultId);
1583 BESDEBUG(MODULE, prolog << "targetURL='"<< targetURL << "'" << endl);
1584
1585 d4au.writeD4AsyncAccepted(xmlWrtr, 0, 0, targetURL, stylesheet_ref);
1586 out << xmlWrtr.get_doc();
1587 out << flush;
1588 BESDEBUG(MODULE, prolog << "Sent AsyncAccepted" << endl);
1589
1590 }
1591 else {
1596 d4au.writeD4AsyncRequired(xmlWrtr, 0, 0, stylesheet_ref);
1597 out << xmlWrtr.get_doc();
1598 out << flush;
1599 BESDEBUG(MODULE, prolog << "Sent AsyncAccepted" << endl);
1600 }
1601
1602 return true;
1603 }
1604
1605 return false;
1606}
1607
1626libdap::DMR *
1628{
1629 BESStopWatch sw;
1630 if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
1631 BESDEBUG(MODULE , prolog << "BEGIN" << endl);
1632
1633 unique_ptr<DMR> dmr = setup_dap4_intern_data(obj, dhi);
1634
1635 intern_dap4_data_grp(dmr->root());
1636
1637 return dmr.release();
1638}
1639
1640unique_ptr<DMR>
1641BESDapResponseBuilder::setup_dap4_intern_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
1642{
1643 dhi.first_container();
1644
1645 BESDMRResponse *bdmr = dynamic_cast<BESDMRResponse *>(obj);
1646 if (!bdmr) throw BESInternalFatalError("Expected a BESDMRResponse instance", __FILE__, __LINE__);
1647
1648 unique_ptr<DMR> dmr(bdmr->get_dmr());
1649 // TL;DR Set the DMR managed by the BESResponseObject to nullptr to avoid calling ~DMR
1650 // twice on the same object.
1651 bdmr->set_dmr(nullptr);
1652 // Why this is here: In the past we designed the BESResponseObject class hierarchy to
1653 // manage the response object, which effectively means delete it when the BES is done
1654 // processing the request. We pass nullptr to set_dmr so that the BESResponseObject
1655 // does not call ~DMR since unique_ptr<> will do that for us.
1656
1657 // Set the correct context by following intern_dap2_data()
1658 set_dataset_name(dmr->filename());
1659 set_dap4ce(dhi.data[DAP4_CONSTRAINT]);
1660 set_dap4function(dhi.data[DAP4_FUNCTION]);
1661 set_async_accepted(dhi.data[ASYNC]);
1662 set_store_result(dhi.data[STORE_RESULT]);
1663
1664 if (!d_dap4function.empty()) {
1665 D4BaseTypeFactory d4_factory;
1666 unique_ptr<DMR> function_result(new DMR(&d4_factory, "function_results"));
1667
1668 // Function modules load their functions onto this list. The list is
1669 // part of libdap, not the BES.
1670 if (!ServerFunctionsList::TheList()) {
1671 stringstream msg;
1672 msg << "The function expression could not be evaluated because ";
1673 msg << "there are no server-side functions defined on this server.";
1674 throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
1675 }
1676
1677 D4FunctionEvaluator parser(dmr.get(), ServerFunctionsList::TheList());
1678 bool parse_ok = parser.parse(d_dap4function);
1679 if (!parse_ok){
1680 stringstream msg;
1681 msg << "Failed to parse the provided DAP4 server-side function expression: " << d_dap4function;
1682 throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
1683 }
1684
1685 parser.eval(function_result.get());
1686
1687 // Now use the results of running the functions for the remainder of the
1688 // send_data operation.
1689 dap4_process_ce_for_intern_data(*function_result);
1690
1691 return function_result;
1692 }
1693 else {
1694 BESDEBUG(MODULE , prolog << "Processing the constraint expression. " << endl);
1696 return dmr;
1697 }
1698}
1699
1700void BESDapResponseBuilder::intern_dap4_data_grp(libdap::D4Group* grp) {
1701 for (D4Group::Vars_iter i = grp->var_begin(), e = grp->var_end(); i != e; ++i) {
1702 BESDEBUG(MODULE , "BESDapResponseBuilder::intern_dap4_data() - "<< (*i)->name() <<endl);
1703 if ((*i)->send_p()) {
1704 BESDEBUG(MODULE , "BESDapResponseBuilder::intern_dap4_data() Obtain data- "<< (*i)->name() <<endl);
1705 (*i)->intern_data();
1706 }
1707 }
1708
1709 for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
1710 BESDEBUG(MODULE , "BESDapResponseBuilder::intern_dap4_data() group- "<< (*gi)->name() <<endl);
1711 intern_dap4_data_grp(*gi);
1712 }
1713}
std::string get_container_type() const
retrieve the type of data this container holds, such as cedar or netcdf.
Definition: BESContainer.h:232
Holds a DDS object within the BES.
libdap::ConstraintEvaluator & get_ce()
void set_dds(libdap::DDS *ddsIn)
libdap::DDS * get_dds()
Represents an OPeNDAP DMR DAP4 data object within the BES.
Cache the results from server functions.
virtual libdap::DDS * get_or_cache_dataset(libdap::DDS *dds, const std::string &constraint)
Return a DDS loaded with data that can be serialized back to a client.
virtual void set_dataset_name(const std::string _dataset)
Set the dataset pathname.
virtual std::string get_dataset_name() const
Get the dataset name.
virtual libdap::DMR * intern_dap4_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
std::string d_dap4function
DAP4 Constraint expression.
virtual std::string get_dap4function() const
Get the DAP4 server side function expression.
virtual void split_ce(libdap::ConstraintEvaluator &eval, const std::string &expr="")
virtual std::string get_ce() const
Get the constraint expression.
std::string d_dap2ce
Name of the dataset/database.
virtual void set_dap4ce(std::string _ce)
virtual libdap::DDS * process_dap2_dds(BESResponseObject *obj, BESDataHandlerInterface &dhi)
Transmit data.
virtual void serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers=true)
virtual libdap::DDS * intern_dap2_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
virtual void send_dds(std::ostream &out, libdap::DDS **dds, libdap::ConstraintEvaluator &eval, bool constrained=false, bool with_mime_headers=true)
Transmit a DDS.
virtual std::string get_dap4ce() const
Get the DAP4 constraint expression.
virtual void dap4_process_ce_for_intern_data(libdap::DMR &dmr)
Parse the DAP4 CE and throw if the request is too large.
std::string d_async_accepted
Version string for the library's default protocol version.
virtual void send_ddx(std::ostream &out, libdap::DDS **dds, libdap::ConstraintEvaluator &eval, bool with_mime_headers=true)
virtual void serialize_dap2_data_dds(std::ostream &out, libdap::DDS **dds, libdap::ConstraintEvaluator &eval, bool ce_eval=true)
virtual void set_dap4function(std::string _func)
virtual bool store_dap4_result(std::ostream &out, libdap::DMR &dmr)
virtual void set_ce(std::string _ce)
std::string d_btp_func_ce
DAP4 Server Side Function expression.
std::string d_dap4ce
DAP2 Constraint expression.
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
void set_dds(libdap::DDS *ddsIn)
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
void first_container()
set the container pointer to the first container in the containers list
BESContainer * container
pointer to current container in this interface
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
virtual BESRequestHandler * find_handler(const std::string &handler_name)
find and return the specified request handler
Represents a specific data type request handler.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
virtual string store_dap4_result(libdap::DMR &dmr, const string &constraint, BESDapResponseBuilder *rb)
static BESStoredDapResultCache * get_instance()
error thrown if there is a user syntax error in the request or any other user error
static void conditional_timeout_cancel()
Checks if the timeout alarm should be canceled based on the value of the BES key BES....
Definition: BESUtil.cc:895
static std::string assemblePath(const std::string &firstPart, const std::string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
Definition: BESUtil.cc:801
static RequestServiceTimer * TheTimer()
Return a pointer to a singleton timer instance. If an instance does not exist it will create and init...
void throw_if_timeout_expired(const std::string &message, const std::string &file, const int line)
Checks the RequestServiceTimer to determine if the time spent servicing the request at this point has...
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71