libdap  Updated for version 3.20.5
libdap4 is an implementation of OPeNDAP's DAP protocol.
D4Connect.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of libdap, A C++ implementation of the OPeNDAP Data
4 // Access Protocol.
5 
6 // Copyright (c) 2002,2003 OPeNDAP, Inc.
7 // Author: James Gallagher <jgallagher@opendap.org>
8 // Dan Holloway <dholloway@gso.uri.edu>
9 // Reza Nekovei <reza@intcomm.net>
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26 
27 // (c) COPYRIGHT URI/MIT 1994-2002
28 // Please read the full copyright statement in the file COPYRIGHT_URI.
29 //
30 // Authors:
31 // jhrg,jimg James Gallagher <jgallagher@gso.uri.edu>
32 // dan Dan Holloway <dholloway@gso.uri.edu>
33 // reza Reza Nekovei <reza@intcomm.net>
34 
35 #include "config.h"
36 // #define DODS_DEBUG 1
37 
38 
39 
40 #include <cassert>
41 #include <cstring>
42 #include <sstream>
43 
44 #include "D4Connect.h"
45 #include "HTTPConnect.h"
46 #include "Response.h"
47 #include "DMR.h"
48 #include "D4Group.h"
49 
50 #include "D4ParserSax2.h"
51 #include "chunked_stream.h"
52 #include "chunked_istream.h"
53 #include "D4StreamUnMarshaller.h"
54 
55 #include "escaping.h"
56 #include "mime_util.h"
57 #include "debug.h"
58 
59 
60 
61 using namespace std;
62 
63 namespace libdap {
64 
67 void D4Connect::process_dmr(DMR &dmr, Response &rs)
68 {
69  DBG(cerr << "Entering D4Connect::process_dmr" << endl);
70 
71  dmr.set_dap_version(rs.get_protocol());
72 
73  DBG(cerr << "Entering process_data. Response.getVersion() = " << rs.get_version() << endl);
74  switch (rs.get_type()) {
75  case dap4_error: {
76 #if 0
77  Error e;
78  if (!e.parse(rs.get_stream()))
79  throw InternalErr(__FILE__, __LINE__, "Could not parse the Error object returned by the server!");
80  throw e;
81 #endif
82  throw InternalErr(__FILE__, __LINE__, "DAP4 errors not processed yet: FIXME!");
83  }
84 
85  case web_error:
86  // Web errors (those reported in the return document's MIME header)
87  // are processed by the WWW library.
88  throw InternalErr(__FILE__, __LINE__,
89  "An error was reported by the remote httpd; this should have been processed by HTTPConnect..");
90 
91  case dap4_dmr: {
92  // parse the DMR
93  try {
94  D4ParserSax2 parser;
95  // When parsing a data response, we use the permissive mode of the DMR parser
96  // (which allows Map elements to reference Arrays that are not in the DMR).
97  // Do not use that mode when parsing the DMR response - assume the DMR is
98  // valid. jhrg 4/13/16
99  parser.intern(*rs.get_cpp_stream(), &dmr);
100  }
101  catch (Error &e) {
102  cerr << "Exception: " << e.get_error_message() << endl;
103  return;
104  }
105  catch (std::exception &e) {
106  cerr << "Exception: " << e.what() << endl;
107  return;
108  }
109  catch (...) {
110  cerr << "Exception: unknown error" << endl;
111  return;
112  }
113 
114  return;
115  }
116 
117  default:
118  throw Error("Unknown response type");
119  }
120 }
121 
124 void D4Connect::process_data(DMR &data, Response &rs)
125 {
126  DBG(cerr << "Entering D4Connect::process_data" << endl);
127 
128  assert(rs.get_cpp_stream()); // DAP4 code uses cpp streams
129 
130  data.set_dap_version(rs.get_protocol());
131 
132  DBG(cerr << "Entering process_data. Response.getVersion() = " << rs.get_version() << endl);
133  switch (rs.get_type()) {
134  case dap4_error: {
135 #if 0
136  Error e;
137  if (!e.parse(rs.get_cpp_stream()))
138  throw InternalErr(__FILE__, __LINE__, "Could not parse the Error object returned by the server!");
139  throw e;
140 #endif
141  throw InternalErr(__FILE__, __LINE__, "DAP4 errors not processed yet: FIXME!");
142  }
143 
144  case web_error:
145  // Web errors (those reported in the return document's MIME header)
146  // are processed by the WWW library.
147  throw InternalErr(__FILE__, __LINE__,
148  "An error was reported by the remote httpd; this should have been processed by HTTPConnect..");
149 
150  case dap4_data: {
151 #if BYTE_ORDER_PREFIX
152  // Read the byte-order byte; used later on
153  char byte_order;
154  *rs.get_cpp_stream() >> byte_order;
155  //if (debug) cerr << "Byte order: " << ((byte_order) ? "big endian" : "little endian") << endl;
156 #endif
157  // get a chunked input stream
158 #if BYTE_ORDER_PREFIX
159  chunked_istream cis(*rs.get_cpp_stream(), 1024, byte_order);
160 #else
161  chunked_istream cis(*(rs.get_cpp_stream()), CHUNK_SIZE);
162 #endif
163  // parse the DMR, stopping when the boundary is found.
164  try {
165  // force chunk read
166  // get chunk size
167  int chunk_size = cis.read_next_chunk();
168  if (chunk_size < 0)
169  throw Error("Found an unexpected end of input (EOF) while reading a DAP4 data response. (1)");
170 
171  // get chunk
172  char chunk[chunk_size];
173  cis.read(chunk, chunk_size);
174  // parse char * with given size
175  D4ParserSax2 parser;
176  // permissive mode allows references to Maps that are not in the response.
177  // Use this mode when parsing a data response (but not the DMR). jhrg 4/13/16
178  parser.set_strict(false);
179 
180  // '-2' to discard the CRLF pair
181  parser.intern(chunk, chunk_size - 2, &data);
182  }
183  catch (Error &e) {
184  cerr << "Exception: " << e.get_error_message() << endl;
185  return;
186  }
187  catch (std::exception &e) {
188  cerr << "Exception: " << e.what() << endl;
189  return;
190  }
191  catch (...) {
192  cerr << "Exception: unknown error" << endl;
193  return;
194  }
195 
196 #if BYTE_ORDER_PREFIX
197  D4StreamUnMarshaller um(cis, byte_order);
198 #else
199  D4StreamUnMarshaller um(cis, cis.twiddle_bytes());
200 #endif
201  data.root()->deserialize(um, data);
202 
203  return;
204  }
205 
206  default:
207  throw Error("Unknown response type");
208  }
209 }
210 
219 void D4Connect::parse_mime(Response &rs)
220 {
221  rs.set_version("dods/0.0"); // initial value; for backward compatibility.
222  rs.set_protocol("2.0");
223 
224  istream &data_source = *rs.get_cpp_stream();
225  string mime = get_next_mime_header(data_source);
226  while (!mime.empty()) {
227  string header, value;
228  parse_mime_header(mime, header, value);
229 
230  // Note that this is an ordered list
231  if (header == "content-description") {
232  DBG(cout << header << ": " << value << endl);
233  rs.set_type(get_description_type(value));
234  }
235  // Use the value of xdods-server only if no other value has been read
236  else if (header == "xdods-server" && rs.get_version() == "dods/0.0") {
237  DBG(cout << header << ": " << value << endl);
238  rs.set_version(value);
239  }
240  // This trumps 'xdods-server' and 'server'
241  else if (header == "xopendap-server") {
242  DBG(cout << header << ": " << value << endl);
243  rs.set_version(value);
244  }
245  else if (header == "xdap") {
246  DBG(cout << header << ": " << value << endl);
247  rs.set_protocol(value);
248  }
249  // Only look for 'server' if no other header supplies this info.
250  else if (rs.get_version() == "dods/0.0" && header == "server") {
251  DBG(cout << header << ": " << value << endl);
252  rs.set_version(value);
253  }
254 
255  mime = get_next_mime_header(data_source);
256  }
257 }
258 
259 // public mfuncs
260 
267 D4Connect::D4Connect(const string &url, string uname, string password) :
268  d_http(0), d_local(false), d_URL(""), d_UrlQueryString(""), d_server("unknown"), d_protocol("4.0")
269 {
270  string name = prune_spaces(url);
271 
272  // Figure out if the URL starts with 'http', if so, make sure that we
273  // talk to an instance of HTTPConnect.
274  if (name.find("http") == 0) {
275  DBG(cerr << "Connect: The identifier is an http URL" << endl);
276  d_http = new HTTPConnect(RCReader::instance());
277  d_http->set_use_cpp_streams(true);
278 
279  d_URL = name;
280 
281  // Find and store any CE given with the URL.
282  string::size_type dotpos = name.find('?');
283  if (dotpos != std::string::npos) { // Found a match.
284  d_URL = name.substr(0, dotpos);
285 
286  d_UrlQueryString = name.substr(dotpos + 1);
287 
288  if (d_UrlQueryString.find(DAP4_CE_QUERY_KEY) != std::string::npos) {
289  std::stringstream msg;
290  msg << endl;
291  msg << "WARNING: A DAP4 constraint expression key was found in the query string!" << endl;
292  msg << "The submitted dataset URL: " << name << endl;
293  msg << "Contains the query string: " << d_UrlQueryString << endl;
294  msg << "This will cause issues when making DAP4 requests that specify additional constraints. " << endl;
295  cerr << msg.str() << endl;
296  // throw Error(malformed_expr, msg.str());
297  }
298 
299  }
300  }
301  else {
302  DBG(cerr << "Connect: The identifier is a local data source." << endl);
303  d_local = true; // local in this case means non-DAP
304  }
305 
306  set_credentials(uname, password);
307 }
308 
309 D4Connect::~D4Connect()
310 {
311  if (d_http) delete d_http;
312 }
313 
314 std::string D4Connect::build_dap4_ce(const string requestSuffix, const string dap4ce)
315 {
316  std::stringstream url;
317  bool needsAmpersand = false;
318 
319  url << d_URL << requestSuffix << "?";
320 
321  if (d_UrlQueryString.length() > 0) {
322  url << d_UrlQueryString;
323  needsAmpersand = true;
324  }
325 
326  if (dap4ce.length() > 0) {
327  if (needsAmpersand) url << "&";
328 
329  url << DAP4_CE_QUERY_KEY << "=" << id2www_ce(dap4ce);
330  }
331 
332  DBG(cerr << "D4Connect::build_dap4_ce() - Source URL: " << d_URL << endl);
333  DBG(cerr << "D4Connect::build_dap4_ce() - Source URL Query String: " << d_UrlQueryString << endl);
334  DBG(cerr << "D4Connect::build_dap4_ce() - dap4ce: " << dap4ce << endl);
335  DBG(cerr << "D4Connect::build_dap4_ce() - request URL: " << url.str() << endl);
336 
337  return url.str();
338 }
339 
340 void D4Connect::request_dmr(DMR &dmr, const string expr)
341 {
342  string url = build_dap4_ce(".dmr", expr);
343 
344  Response *rs = 0;
345  try {
346  rs = d_http->fetch_url(url);
347 
348  d_server = rs->get_version();
349  d_protocol = rs->get_protocol();
350 
351  switch (rs->get_type()) {
352  case unknown_type:
353  DBG(cerr << "Response type unknown, assuming it's a DMR response." << endl);
354  /* no break */
355  case dap4_dmr: {
356  D4ParserSax2 parser;
357  parser.intern(*rs->get_cpp_stream(), &dmr);
358  break;
359  }
360 
361  case dap4_error:
362  throw InternalErr(__FILE__, __LINE__, "DAP4 errors are not processed yet.");
363 
364  case web_error:
365  // We should never get here; a web error should be picked up read_url
366  // (called by fetch_url) and result in a thrown Error object.
367  throw InternalErr(__FILE__, __LINE__, "Web error found where it should never be.");
368 
369  default:
370  throw InternalErr(__FILE__, __LINE__,
371  "Response type not handled (got " + long_to_string(rs->get_type()) + ").");
372  }
373  }
374  catch (...) {
375  delete rs;
376  throw;
377  }
378 
379  delete rs;
380 }
381 
382 void D4Connect::request_dap4_data(DMR &dmr, const string expr)
383 {
384  string url = build_dap4_ce(".dap", expr);
385 
386  Response *rs = 0;
387  try {
388  rs = d_http->fetch_url(url);
389 
390  d_server = rs->get_version();
391  d_protocol = rs->get_protocol();
392 
393  switch (rs->get_type()) {
394  case unknown_type:
395  DBG(cerr << "Response type unknown, assuming it's a DAP4 Data response." << endl);
396  /* no break */
397  case dap4_data: {
398 #if BYTE_ORDER_PREFIX
399  istream &in = *rs->get_cpp_stream();
400  // Read the byte-order byte; used later on
401  char byte_order;
402  in >> byte_order;
403 #endif
404 
405  // get a chunked input stream
406 #if BYTE_ORDER_PREFIX
407  chunked_istream cis(*(rs->get_cpp_stream()), 1024, byte_order);
408 #else
409  chunked_istream cis(*(rs->get_cpp_stream()), CHUNK_SIZE);
410 #endif
411 
412  // parse the DMR, stopping when the boundary is found.
413 
414  // force chunk read
415  // get chunk size
416  int chunk_size = cis.read_next_chunk();
417  if (chunk_size < 0)
418  throw Error("Found an unexpected end of input (EOF) while reading a DAP4 data response. (2)");
419 
420  // get chunk
421  char chunk[chunk_size];
422  cis.read(chunk, chunk_size);
423  // parse char * with given size
424  D4ParserSax2 parser;
425  // permissive mode allows references to Maps that are not in the response.
426  parser.set_strict(false);
427  // '-2' to discard the CRLF pair
428  parser.intern(chunk, chunk_size - 2, &dmr, false /*debug*/);
429 
430  // Read data and store in the DMR
431 #if BYTE_ORDER_PREFIX
432  D4StreamUnMarshaller um(cis, byte_order);
433 #else
434  D4StreamUnMarshaller um(cis, cis.twiddle_bytes());
435 #endif
436  dmr.root()->deserialize(um, dmr);
437 
438  break;
439  }
440 
441  case dap4_error:
442  throw InternalErr(__FILE__, __LINE__, "DAP4 errors are not processed yet.");
443 
444  case web_error:
445  // We should never get here; a web error should be picked up read_url
446  // (called by fetch_url) and result in a thrown Error object.
447  throw InternalErr(__FILE__, __LINE__, "Web error found where it should never be.");
448 
449  default:
450  throw InternalErr(__FILE__, __LINE__,
451  "Response type not handled (got " + long_to_string(rs->get_type()) + ").");
452  }
453  }
454  catch (...) {
455  delete rs;
456  throw;
457  }
458 
459  delete rs;
460 }
461 
462 void D4Connect::read_dmr(DMR &dmr, Response &rs)
463 {
464  parse_mime(rs);
465  if (rs.get_type() == unknown_type) throw Error("Unknown response type.");
466 
467  read_dmr_no_mime(dmr, rs);
468 }
469 
470 void D4Connect::read_dmr_no_mime(DMR &dmr, Response &rs)
471 {
472  // Assume callers know what they are doing
473  if (rs.get_type() == unknown_type) rs.set_type(dap4_dmr);
474 
475  switch (rs.get_type()) {
476  case dap4_dmr:
477  process_dmr(dmr, rs);
478  d_server = rs.get_version();
479  d_protocol = dmr.dap_version();
480  break;
481  default:
482  throw Error("Expected a DAP4 DMR response.");
483  }
484 }
485 
486 void D4Connect::read_data(DMR &data, Response &rs)
487 {
488  parse_mime(rs);
489  if (rs.get_type() == unknown_type) throw Error("Unknown response type.");
490 
491  read_data_no_mime(data, rs);
492 }
493 
494 void D4Connect::read_data_no_mime(DMR &data, Response &rs)
495 {
496  // Assume callers know what they are doing
497  if (rs.get_type() == unknown_type) rs.set_type(dap4_data);
498 
499  switch (rs.get_type()) {
500  case dap4_data:
501  process_data(data, rs);
502  d_server = rs.get_version();
503  d_protocol = data.dap_version();
504  break;
505  default:
506  throw Error("Expected a DAP4 Data response.");
507  }
508 }
509 
515 void D4Connect::set_credentials(string u, string p)
516 {
517  if (d_http) d_http->set_credentials(u, p);
518 }
519 
524 {
525  if (d_http) d_http->set_accept_deflate(deflate);
526 }
527 
533 void D4Connect::set_xdap_protocol(int major, int minor)
534 {
535  if (d_http) d_http->set_xdap_protocol(major, minor);
536 }
537 
542 {
543  if (d_http) d_http->set_cache_enabled(cache);
544 }
545 
546 bool D4Connect::is_cache_enabled()
547 {
548  if (d_http)
549  return d_http->is_cache_enabled();
550  else
551  return false;
552 }
553 
554 } // namespace libdap
libdap::id2www_ce
string id2www_ce(string in, const string &allowable)
Definition: escaping.cc:178
libdap::get_next_mime_header
string get_next_mime_header(FILE *in)
Definition: mime_util.cc:838
libdap::D4Connect::set_credentials
void set_credentials(std::string u, std::string p)
Set the credentials for responding to challenges while dereferencing URLs.
Definition: D4Connect.cc:515
libdap::HTTPConnect::set_credentials
void set_credentials(const string &u, const string &p)
Definition: HTTPConnect.cc:1074
libdap::get_description_type
ObjectType get_description_type(const string &value)
Definition: mime_util.cc:339
libdap::prune_spaces
string prune_spaces(const string &name)
Definition: util.cc:459
libdap::HTTPConnect::is_cache_enabled
bool is_cache_enabled()
Definition: HTTPConnect.h:157
libdap::HTTPConnect::fetch_url
HTTPResponse * fetch_url(const string &url)
Definition: HTTPConnect.cc:627
libdap
top level DAP object to house generic methods
Definition: AlarmHandler.h:35
libdap::parse_mime_header
void parse_mime_header(const string &header, string &name, string &value)
Definition: mime_util.cc:912
libdap::HTTPConnect::set_xdap_protocol
void set_xdap_protocol(int major, int minor)
Definition: HTTPConnect.cc:1037
libdap::D4Connect::set_cache_enabled
void set_cache_enabled(bool enabled)
Definition: D4Connect.cc:541
libdap::HTTPConnect::set_cache_enabled
void set_cache_enabled(bool enabled)
Definition: HTTPConnect.h:151
libdap::HTTPConnect::set_accept_deflate
void set_accept_deflate(bool defalte)
Definition: HTTPConnect.cc:1008
libdap::D4Connect::set_accept_deflate
void set_accept_deflate(bool deflate)
Definition: D4Connect.cc:523
libdap::D4Connect::set_xdap_protocol
void set_xdap_protocol(int major, int minor)
Definition: D4Connect.cc:533
libdap::HTTPConnect
Definition: HTTPConnect.h:73