D4Connect.cc

// -*- mode: c++; c-basic-offset:4 -*-

// This file is part of libdap, A C++ implementation of the OPeNDAP Data
// Access Protocol.

// Copyright (c) 2002,2003 OPeNDAP, Inc.
// Author: James Gallagher <jgallagher@opendap.org>
//         Dan Holloway <dholloway@gso.uri.edu>
//         Reza Nekovei <reza@intcomm.net>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//
// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.

// (c) COPYRIGHT URI/MIT 1994-2002
// Please read the full copyright statement in the file COPYRIGHT_URI.
//
// Authors:
//      jhrg,jimg       James Gallagher <jgallagher@gso.uri.edu>
//      dan             Dan Holloway <dholloway@gso.uri.edu>
//      reza            Reza Nekovei <reza@intcomm.net>

#include "config.h"
// #define DODS_DEBUG 1

#include <cassert>

#include <sstream>

#include "D4Connect.h"
#include "HTTPConnect.h"
#include "Response.h"
#include "DMR.h"
#include "D4Group.h"

#include "D4ParserSax2.h"
#include "chunked_stream.h"
#include "chunked_istream.h"
#include "D4StreamUnMarshaller.h"

#include "escaping.h"
#include "mime_util.h"
#include "debug.h"

using namespace std;

namespace libdap {

/** This private method process data from both local and remote sources. It
 exists to eliminate duplication of code. */
void D4Connect::process_dmr(DMR &dmr, Response &rs)
{
    DBG(cerr << "Entering D4Connect::process_dmr" << endl);

    dmr.set_dap_version(rs.get_protocol());

    DBG(cerr << "Entering process_data. Response.getVersion() = " << rs.get_version() << endl);
    switch (rs.get_type()) {
    case dap4_error: {
#if 0
        Error e;
        if (!e.parse(rs.get_stream()))
        throw InternalErr(__FILE__, __LINE__, "Could not parse the Error object returned by the server!");
        throw e;
#endif
        throw InternalErr(__FILE__, __LINE__, "DAP4 errors not processed yet: FIXME!");
    }

    case web_error:
        // Web errors (those reported in the return document's MIME header)
        // are processed by the WWW library.
        throw InternalErr(__FILE__, __LINE__,
            "An error was reported by the remote httpd; this should have been processed by HTTPConnect.");

    case dap4_dmr: {
        // parse the DMR
        try {
            D4ParserSax2 parser;
            // When parsing a data response, we use the permissive mode of the DMR parser
            // (which allows Map elements to reference Arrays that are not in the DMR).
            // Do not use that mode when parsing the DMR response - assume the DMR is
            // valid. jhrg 4/13/16
            parser.intern(*rs.get_cpp_stream(), &dmr);
        }
        catch (Error &e) {
            cerr << "Exception: " << e.get_error_message() << endl;
            return;
        }
        catch (std::exception &e) {
            cerr << "Exception: " << e.what() << endl;
            return;
        }
        catch (...) {
            cerr << "Exception: unknown error" << endl;
            return;
        }

        return;
    }

    default:
        throw Error("Unknown response type");
    }
}

/** This private method process data from both local and remote sources. It
 exists to eliminate duplication of code. */
void D4Connect::process_data(DMR &data, Response &rs)
{
    DBG(cerr << "Entering D4Connect::process_data" << endl);

    assert(rs.get_cpp_stream());	// DAP4 code uses cpp streams

    data.set_dap_version(rs.get_protocol());

    DBG(cerr << "Entering process_data. Response.getVersion() = " << rs.get_version() << endl);
    switch (rs.get_type()) {
    case dap4_error: {
        throw InternalErr(__FILE__, __LINE__, "DAP4 errors not processed yet: FIXME!");
    }

    case web_error:
        // Web errors (those reported in the return document's MIME header)
        // are processed by the WWW library.
        throw InternalErr(__FILE__, __LINE__,
            "An error was reported by the remote httpd; this should have been processed by HTTPConnect..");

    case dap4_data: {
        chunked_istream cis(*(rs.get_cpp_stream()), CHUNK_SIZE);
        // parse the DMR, stopping when the boundary is found.
        try {
            // force chunk read
            // get chunk size
            int chunk_size = cis.read_next_chunk();
            if (chunk_size < 0)
                throw Error("Found an unexpected end of input (EOF) while reading a DAP4 data response. (1)");

            // get chunk
            char chunk[chunk_size];
            cis.read(chunk, chunk_size);
            // parse char * with given size
            D4ParserSax2 parser;
            // permissive mode allows references to Maps that are not in the response.
            // Use this mode when parsing a data response (but not the DMR). jhrg 4/13/16
            parser.set_strict(false);

            // '-2' to discard the CRLF pair
            parser.intern(chunk, chunk_size - 2, &data);
        }
        catch (Error &e) {
            cerr << "Exception: " << e.get_error_message() << endl;
            return;
        }
        catch (std::exception &e) {
            cerr << "Exception: " << e.what() << endl;
            return;
        }
        catch (...) {
            cerr << "Exception: unknown error" << endl;
            return;
        }

        D4StreamUnMarshaller um(cis, cis.twiddle_bytes());
        data.root()->deserialize(um, data);

        return;
    }

    default:
        throw Error("Unknown response type");
    }
}

/** Use when you cannot use libcurl.

 @note This method tests for MIME headers with lines terminated by CRLF
 (\r\n) and Newlines (\n). In either case, the line terminators are removed
 before each header is processed.

 @param rs Value/Result parameter. Dump version and type information here.
 */
void D4Connect::parse_mime(Response &rs)
{
    rs.set_version("dods/0.0"); // initial value; for backward compatibility.
    rs.set_protocol("2.0");

    istream &data_source = *rs.get_cpp_stream();
    string mime = get_next_mime_header(data_source);
    while (!mime.empty()) {
        string header, value;
        parse_mime_header(mime, header, value);

        // Note that this is an ordered list
        if (header == "content-description") {
            DBG(cout << header << ": " << value << endl);
            rs.set_type(get_description_type(value));
        }
        // Use the value of xdods-server only if no other value has been read
        else if (header == "xdods-server" && rs.get_version() == "dods/0.0") {
            DBG(cout << header << ": " << value << endl);
            rs.set_version(value);
        }
        // This trumps 'xdods-server' and 'server'
        else if (header == "xopendap-server") {
            DBG(cout << header << ": " << value << endl);
            rs.set_version(value);
        }
        else if (header == "xdap") {
            DBG(cout << header << ": " << value << endl);
            rs.set_protocol(value);
        }
        // Only look for 'server' if no other header supplies this info.
        else if (rs.get_version() == "dods/0.0" && header == "server") {
            DBG(cout << header << ": " << value << endl);
            rs.set_version(value);
        }

        mime = get_next_mime_header(data_source);
    }
}

// public mfuncs

/** The D4Connect constructor requires a URL or local file.

 @param n The URL for the virtual connection.
 @param uname Use this username for authentication. Null by default.
 @param password Password to use for authentication. Null by default.
 @brief Create an instance of Connect. */
D4Connect::D4Connect(const string &url, string uname, string password) :
    d_http(0), d_local(false), d_URL(""), d_UrlQueryString(""), d_server("unknown"), d_protocol("4.0")
{
    string name = prune_spaces(url);

    // Figure out if the URL starts with 'http', if so, make sure that we
    // talk to an instance of HTTPConnect.
    if (name.find("http") == 0) {
        DBG(cerr << "Connect: The identifier is an http URL" << endl);
        d_http = new HTTPConnect(RCReader::instance());
        d_http->set_use_cpp_streams(true);

        d_URL = name;

        // Find and store any CE given with the URL.
        string::size_type dotpos = name.find('?');
        if (dotpos != std::string::npos) { // Found a match.
            d_URL = name.substr(0, dotpos);

            d_UrlQueryString = name.substr(dotpos + 1);

            if (d_UrlQueryString.find(DAP4_CE_QUERY_KEY) != std::string::npos) {
                std::stringstream msg;
                msg << endl;
                msg << "WARNING: A DAP4 constraint expression key was found in the query string!" << endl;
                msg << "The submitted dataset URL: " << name << endl;
                msg << "Contains the query string: " << d_UrlQueryString << endl;
                msg << "This will cause issues when making DAP4 requests that specify additional constraints. " << endl;
                cerr << msg.str() << endl;
                // throw Error(malformed_expr, msg.str());
            }

        }
    }
    else {
        DBG(cerr << "Connect: The identifier is a local data source." << endl);
        d_local = true; // local in this case means non-DAP
    }

    set_credentials(uname, password);
}

D4Connect::~D4Connect()
{
    if (d_http) delete d_http;
}

std::string D4Connect::build_dap4_ce(const string requestSuffix, const string dap4ce)
{
    std::stringstream url;
    bool needsAmpersand = false;

    url << d_URL << requestSuffix << "?";

    if (d_UrlQueryString.length() > 0) {
        url << d_UrlQueryString;
        needsAmpersand = true;
    }

    if (dap4ce.length() > 0) {
        if (needsAmpersand) url << "&";

        url << DAP4_CE_QUERY_KEY << "=" << id2www_ce(dap4ce);
    }

    DBG(cerr << "D4Connect::build_dap4_ce() - Source URL: " << d_URL << endl);
    DBG(cerr << "D4Connect::build_dap4_ce() - Source URL Query String: " << d_UrlQueryString << endl);
    DBG(cerr << "D4Connect::build_dap4_ce() - dap4ce: " << dap4ce << endl);
    DBG(cerr << "D4Connect::build_dap4_ce() - request URL: " << url.str() << endl);

    return url.str();
}

void D4Connect::request_dmr(DMR &dmr, const string expr)
{
    string url = build_dap4_ce(".dmr", expr);

    Response *rs = 0;
    try {
        rs = d_http->fetch_url(url);

        d_server = rs->get_version();
        d_protocol = rs->get_protocol();

        switch (rs->get_type()) {
        case unknown_type:
            DBG(cerr << "Response type unknown, assuming it's a DMR response." << endl);
            /* no break */
        case dap4_dmr: {
            D4ParserSax2 parser;
            parser.intern(*rs->get_cpp_stream(), &dmr);
            break;
        }

        case dap4_error:
            throw InternalErr(__FILE__, __LINE__, "DAP4 errors are not processed yet.");

        case web_error:
            // We should never get here; a web error should be picked up read_url
            // (called by fetch_url) and result in a thrown Error object.
            throw InternalErr(__FILE__, __LINE__, "Web error found where it should never be.");

        default:
            throw InternalErr(__FILE__, __LINE__,
                "Response type not handled (got " + long_to_string(rs->get_type()) + ").");
        }
    }
    catch (...) {
        delete rs;
        throw;
    }

    delete rs;
}

void D4Connect::request_dap4_data(DMR &dmr, const string expr)
{
    string url = build_dap4_ce(".dap", expr);

    Response *rs = 0;
    try {
        rs = d_http->fetch_url(url);

        d_server = rs->get_version();
        d_protocol = rs->get_protocol();

        switch (rs->get_type()) {
        case unknown_type:
            DBG(cerr << "Response type unknown, assuming it's a DAP4 Data response." << endl);
            /* no break */
        case dap4_data: {
            // get a chunked input stream
            chunked_istream cis(*(rs->get_cpp_stream()), CHUNK_SIZE);

            // parse the DMR, stopping when the boundary is found.

            // force chunk read
            // get chunk size
            int chunk_size = cis.read_next_chunk();
            if (chunk_size < 0)
                throw Error("Found an unexpected end of input (EOF) while reading a DAP4 data response. (2)");

            // get chunk
            char chunk[chunk_size];
            cis.read(chunk, chunk_size);
            // parse char * with given size
            D4ParserSax2 parser;
            // permissive mode allows references to Maps that are not in the response.
            parser.set_strict(false);
            // '-2' to discard the CRLF pair
            parser.intern(chunk, chunk_size - 2, &dmr, false /*debug*/);

            // Read data and store in the DMR
            D4StreamUnMarshaller um(cis, cis.twiddle_bytes());
            dmr.root()->deserialize(um, dmr);

            break;
        }

        case dap4_error:
            throw InternalErr(__FILE__, __LINE__, "DAP4 errors are not processed yet.");

        case web_error:
            // We should never get here; a web error should be picked up read_url
            // (called by fetch_url) and result in a thrown Error object.
            throw InternalErr(__FILE__, __LINE__, "Web error found where it should never be.");

        default:
            throw InternalErr(__FILE__, __LINE__,
                "Response type not handled (got " + long_to_string(rs->get_type()) + ").");
        }
    }
    catch (...) {
        delete rs;
        throw;
    }

    delete rs;
}

void D4Connect::read_dmr(DMR &dmr, Response &rs)
{
    parse_mime(rs);
    if (rs.get_type() == unknown_type) throw Error("Unknown response type.");

    read_dmr_no_mime(dmr, rs);
}

void D4Connect::read_dmr_no_mime(DMR &dmr, Response &rs)
{
    // Assume callers know what they are doing
    if (rs.get_type() == unknown_type) rs.set_type(dap4_dmr);

    switch (rs.get_type()) {
    case dap4_dmr:
        process_dmr(dmr, rs);
        d_server = rs.get_version();
        d_protocol = dmr.dap_version();
        break;
    default:
        throw Error("Expected a DAP4 DMR response.");
    }
}

void D4Connect::read_data(DMR &data, Response &rs)
{
    parse_mime(rs);
    if (rs.get_type() == unknown_type) throw Error("Unknown response type.");

    read_data_no_mime(data, rs);
}

void D4Connect::read_data_no_mime(DMR &data, Response &rs)
{
    // Assume callers know what they are doing
    if (rs.get_type() == unknown_type) rs.set_type(dap4_data);

    switch (rs.get_type()) {
    case dap4_data:
        process_data(data, rs);
        d_server = rs.get_version();
        d_protocol = data.dap_version();
        break;
    default:
        throw Error("Expected a DAP4 Data response.");
    }
}

/** @brief Set the credentials for responding to challenges while dereferencing
 URLs.
 @param u The username.
 @param p The password.
 @see extract_auth_info() */
void D4Connect::set_credentials(string u, string p)
{
    if (d_http) d_http->set_credentials(u, p);
}

/** Set the \e accept deflate property.
 @param deflate True if the client can accept compressed responses, False
 otherwise. */
void D4Connect::set_accept_deflate(bool deflate)
{
    if (d_http) d_http->set_accept_deflate(deflate);
}

/** Set the \e XDAP-Accept property/header. This is used to send to a server
 the (highest) DAP protocol version number that this client understands.

 @param major The client dap protocol major version
 @param minor The client dap protocol minor version */
void D4Connect::set_xdap_protocol(int major, int minor)
{
    if (d_http) d_http->set_xdap_protocol(major, minor);
}

/** Disable any further use of the client-side cache. In a future version
 of this software, this should be handled so that the www library is
 not initialized with the cache running by default. */
void D4Connect::set_cache_enabled(bool cache)
{
    if (d_http) d_http->set_cache_enabled(cache);
}

bool D4Connect::is_cache_enabled()
{
    if (d_http)
        return d_http->is_cache_enabled();
    else
        return false;
}

} // namespace libdap