bes Updated for version 3.20.13
FONcTransmitter.cc
1// FONcTransmitter.cc
2
3// This file is part of BES Netcdf File Out Module
4
5// Copyright (c) 2004,2005 University Corporation for Atmospheric Research
6// Author: Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
7//
8// This library is free software; you can redistribute it and/or
9// modify it under the terms of the GNU Lesser General Public
10// License as published by the Free Software Foundation; either
11// version 2.1 of the License, or (at your option) any later version.
12//
13// This library is distributed in the hope that it will be useful,
14// but WITHOUT ANY WARRANTY; without even the implied warranty of
15// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16// Lesser General Public License for more details.
17//
18// You should have received a copy of the GNU Lesser General Public
19// License along with this library; if not, write to the Free Software
20// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21//
22// You can contact University Corporation for Atmospheric Research at
23// 3080 Center Green Drive, Boulder, CO 80301
24
25// (c) COPYRIGHT University Corporation for Atmospheric Research 2004-2005
26// Please read the full copyright statement in the file COPYRIGHT_UCAR.
27//
28// Authors:
29// pwest Patrick West <pwest@ucar.edu>
30// jgarcia Jose Garcia <jgarcia@ucar.edu>
31// kyang Kent Yang <myang6@hdfgroup.org> (for DAP4/netCDF-4 enhancement)
32
33#include "config.h"
34
35
36#ifdef HAVE_UNISTD_H
37#include <unistd.h>
38#endif
39
40#include <exception>
41#include <sstream> // std::stringstream
42#include <thread>
43#include <future>
44
45#include <libdap/D4Group.h>
46#include <libdap/D4Attributes.h>
47#include <libdap/BaseType.h>
48#include <libdap/escaping.h>
49
50#include <BESContextManager.h>
51#include <BESDataDDSResponse.h>
52#include <BESDapNames.h>
53#include <BESDataNames.h>
54#include <BESDebug.h>
55#include <BESUtil.h>
56#include <TempFile.h>
57#include <RequestServiceTimer.h>
58
59#include <BESLog.h>
60#include <BESError.h>
61#include <BESInternalFatalError.h>
62#include <BESDapError.h>
63#include "BESDMRResponse.h"
64#include <stringbuffer.h>
65
66#include "FONcBaseType.h"
67#include "FONcRequestHandler.h"
68#include "FONcTransmitter.h"
69#include "FONcTransform.h"
70
71using namespace libdap;
72using namespace std;
73using namespace rapidjson;
74
75#define MODULE "fonc"
76#define prolog string("FONcTransmitter::").append(__func__).append("() - ")
77
78
79#if 0 // Moved to BESUtil.cc
80// size of the buffer used to read from the temporary file built on disk and
81// send data to the client over the network connection (socket/stream)
82// #define OUTPUT_FILE_BLOCK_SIZE 4096
83#endif
84
98{
99 add_method(DATA_SERVICE, FONcTransmitter::send_dap2_data);
100 add_method(DAP4DATA_SERVICE, FONcTransmitter::send_dap4_data);
101}
102
103
121{
122 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
123
124 try { // Expanded try block so all DAP errors are caught. ndp 12/23/2015
125 auto bdds = dynamic_cast<BESDataDDSResponse *>(obj);
126 if (!bdds) throw BESInternalFatalError("Expected a BESDataDDSResponse instance", __FILE__, __LINE__);
127 auto dds = bdds->get_dds();
128
129 string base_name = dds->filename().substr(dds->filename().find_last_of("/\\") + 1);
130
131 // This object closes the file when it goes out of scope.
132 bes::TempFile temp_file;
133 string temp_file_name = temp_file.create(FONcRequestHandler::temp_dir, "dap2_nc_"+base_name);
134
135 BESDEBUG(MODULE, prolog << "Building response file " << temp_file_name << endl);
136
137 ostream &strm = dhi.get_output_stream();
138 if (!strm) throw BESInternalError("Output stream is not set, can not return as", __FILE__, __LINE__);
139
140 BESDEBUG(MODULE, prolog << "Transmitting temp file " << temp_file_name << endl);
141
142 // Note that 'RETURN_CMD' is the same as the string that determines the file type:
143 // netcdf 3 or netcdf 4. Hack. jhrg 9/7/16
144 FONcTransform ft(obj, &dhi, temp_file_name, dhi.data[RETURN_CMD]);
145
146#if 0
147 // This is used to signal the BESUtil::file_to_stream_task() this code is done
148 // writing to the file. WIP jhrg 6/4/21
149 atomic<bool> file_write_done(false);
150
151 // Calling the 'packaged_task' here blocks, but we could have run the task in a thread.
152 // See: https://stackoverflow.com/questions/18143661/what-is-the-difference-between-packaged-task-and-async
153 // jhrg 6/4/21
154
155 std::packaged_task<uint64_t(const string &, atomic<bool>&, ostream&)> task(BESUtil::file_to_stream_task);
156 std::future<uint64_t> result = task.get_future();
157 task(temp_file.get_name(), file_write_done, strm);
158#endif
159
160#define TOGGLE_TASK 0
161 // TOGGLE_TASK 1 besstandalone -c bes.nc4.conf -i mem-pressure-tests/bescmd.xml > tmp2.nc4
162 // 151.13s user 8.75s system 98% cpu 2:41.69 total
163 // TOGGLE_TASK 0 besstandalone -c bes.nc4.conf -i mem-pressure-tests/bescmd.xml > tmp2.nc4
164 // 154.71s user 8.99s system 99% cpu 2:45.27 total
165 // TOGGLE_TASK 0 as above, but using BESUtil::file_to_stream(temp_file.get_name(), strm);
166 // and not ESUtil::file_to_stream_task(temp_file.get_name(), file_write_done, strm);
167 // 148.61s user 7.54s system 99% cpu 2:36.35 total
168#if TOGGLE_TASK
169 // This code works without the sleep(1) hack in BESUtil::file_to_stream_task().
170 // Because it is marked as deferred, the task does not start until the future's
171 // get() method is run, after transform() has written all the data. jhrg 6/4/21
172 future<uint64_t> result = async(launch::deferred, &BESUtil::file_to_stream_task, temp_file.get_name(),
173 std::ref(file_write_done), std::ref(strm));
174#endif
175 ft.transform_dap2(strm);
176
177#if 0
178 file_write_done = true;
179 uint64_t tcount = result.get();
180#endif
181
182 //original call before the 'task' hack was added:
183 //BESUtil::file_to_stream(temp_file.get_name(),strm);
184 //jhrg 6/4/21
185
186#if 0
187 // The task can be called like this right here
188 uint64_t tcount = BESUtil::file_to_stream_task(temp_file.get_name(), file_write_done, strm);
189
190 // Or it can be run like this...
191 std::packaged_task<uint64_t(const string &, atomic<bool>&, ostream&)> task(BESUtil::file_to_stream_task);
192 std::future<uint64_t> result = task.get_future();
193 task(temp_file.get_name(), file_write_done, strm);
194 uint64_t tcount = result.get();
195#endif
196 //BESDEBUG(MODULE, prolog << "NetCDF file bytes written " << tcount << endl);
197 }
198 catch (Error &e) {
199 throw BESDapError("Failed to read data: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
200 }
201 catch (BESError &e) {
202 throw;
203 }
204 catch (std::exception &e) {
205 throw BESInternalError("Failed to read data: STL Error: " + string(e.what()), __FILE__, __LINE__);
206 }
207 catch (...) {
208 throw BESInternalError("Failed to get read data: Unknown exception caught", __FILE__, __LINE__);
209 }
210
211 BESDEBUG(MODULE, prolog << "END Transmitted as netcdf" << endl);
212}
213
232{
233 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
234
235 try { // Expanded try block so all DAP errors are caught. ndp 12/23/2015
236
237 auto bdmr = dynamic_cast<BESDMRResponse *>(obj);
238 if (!bdmr) throw BESInternalFatalError("Expected a BESDMRResponse instance", __FILE__, __LINE__);
239 auto dmr = bdmr->get_dmr();
240
241 string base_name = dmr->filename().substr(dmr->filename().find_last_of("/\\") + 1);
242
243 // This object closes the file when it goes out of scope.
244 bes::TempFile temp_file;
245 string temp_file_name = temp_file.create(FONcRequestHandler::temp_dir, "dap4_nc_"+base_name);
246
247 BESDEBUG(MODULE, prolog << "Building response file " << temp_file_name << endl);
248 // Note that 'RETURN_CMD' is the same as the string that determines the file type:
249 // netcdf 3 or netcdf 4. Hack. jhrg 9/7/16
250 // FONcTransform ft(loaded_dmr, dhi, temp_file.get_name(), dhi.data[RETURN_CMD]);
251 FONcTransform ft(obj, &dhi, temp_file_name, dhi.data[RETURN_CMD]);
252
253 // Call the transform function for DAP4.
254 ft.transform_dap4();
255
256 ostream &strm = dhi.get_output_stream();
257
258#if !NDEBUG
259 stringstream msg;
260 msg << prolog << "Using ostream: " << (void *) &strm << endl;
261 BESDEBUG(MODULE, msg.str());
262 INFO_LOG( msg.str());
263#endif
264
265 if (!strm) throw BESInternalError("Output stream is not set, can not return as", __FILE__, __LINE__);
266
267 // Verify the request hasn't exceeded bes_timeout, and disable timeout if allowed.
268 RequestServiceTimer::TheTimer()->throw_if_timeout_expired("ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
270
271 BESDEBUG(MODULE, prolog << "Transmitting temp file " << temp_file_name << endl);
272
273 // FONcTransmitter::write_temp_file_to_stream(temp_file.get_fd(), strm); //, loaded_dds->filename(), ncVersion);
274 BESUtil::file_to_stream(temp_file_name,strm);
275 }
276 catch (Error &e) {
277 throw BESDapError("Failed to read data: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
278 }
279 catch (BESError &e) {
280 throw;
281 }
282 catch (std::exception &e) {
283 throw BESInternalError("Failed to read data: STL Error: " + string(e.what()), __FILE__, __LINE__);
284 }
285 catch (...) {
286 throw BESInternalError("Failed to get read data: Unknown exception caught", __FILE__, __LINE__);
287 }
288
289 BESDEBUG(MODULE, prolog << "END Transmitted as netcdf" << endl);
290}
291
292
293
294
295
296
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
Base exception class for the BES with basic string message.
Definition: BESError.h:59
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Abstract base class representing a specific set of information in response to a request to the BES.
static uint64_t file_to_stream_task(const std::string &file_name, std::atomic< bool > &file_write_done, std::ostream &o_strm)
Definition: BESUtil.cc:1303
static void conditional_timeout_cancel()
Checks if the timeout alarm should be canceled based on the value of the BES key BES....
Definition: BESUtil.cc:895
static void file_to_stream(const std::string &file_name, std::ostream &o_strm)
Copies the contents of the file identified by file_name to the stream o_strm.
Definition: BESUtil.cc:1132
Transformation object that converts an OPeNDAP DataDDS to a netcdf file.
Definition: FONcTransform.h:59
virtual void transform_dap2(ostream &strm)
Transforms each of the variables of the DataDDS to the NetCDF file.
virtual void transform_dap4()
Transforms each of the variables of the DMR to the NetCDF file.
static void send_dap4_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
The static method registered to transmit OPeNDAP data objects as a netcdf file.
static void send_dap2_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
The static method registered to transmit OPeNDAP data objects as a netcdf file.
FONcTransmitter()
Construct the FONcTransmitter, adding it with name netcdf to be able to transmit a data response.
static RequestServiceTimer * TheTimer()
Return a pointer to a singleton timer instance. If an instance does not exist it will create and init...
void throw_if_timeout_expired(const std::string &message, const std::string &file, const int line)
Checks the RequestServiceTimer to determine if the time spent servicing the request at this point has...
Get a new temporary file.
Definition: TempFile.h:46
std::string get_name() const
Definition: TempFile.h:80
std::string create(const std::string &dir_name="/tmp/hyrax_tmp", const std::string &path_template="opendap")
Create a new temporary file.
Definition: TempFile.cc:170
main RapidJSON namespace