vdata.cc

Go to the documentation of this file.

// This file is part of the hdf4 data handler for the OPeNDAP data server.

// Copyright (c) 2005 OPeNDAP, Inc.
// Author: James Gallagher <jgallagher@opendap.org>
//
// This is free software; you can redistribute it and/or modify it under the
// terms of the GNU Lesser General Public License as published by the Free
// Software Foundation; either version 2.1 of the License, or (at your
// option) any later version.
//
// This software is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
// License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with this software; if not, write to the Free Software Foundation,
// Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.

// Copyright 1996, by the California Institute of Technology.
// ALL RIGHTS RESERVED. United States Government Sponsorship
// acknowledged. Any commercial use must be negotiated with the
// Office of Technology Transfer at the California Institute of
// Technology. This software may be subject to U.S. export control
// laws and regulations. By accepting this software, the user
// agrees to comply with all applicable U.S. export laws and
// regulations. User has the responsibility to obtain export
// licenses, or other export authority as may be required before
// exporting such information to foreign countries or providing
// access to foreign persons.

// U.S. Government Sponsorship under NASA Contract
// NAS7-1260 is acknowledged.
//
// Author: Todd.K.Karakashian@jpl.nasa.gov
//
// $RCSfile: vdata.cc,v $ - classes for HDF VDATA
//

#include "config_hdf.h"

#include <mfhdf.h>

#ifdef __POWERPC__
#undef isascii
#endif

#include <string>
#include <vector>
#include <set>
#include <algorithm>

using std::set;
using std::less;

// libdap
#include <InternalErr.h>
#include <util.h>
#include <debug.h>

// bes
#include <BESDebug.h>

#include <hcstream.h>
#include <hdfclass.h>

static void LoadField(int32 vid, int index, int32 begin, int32 end,
                hdf_field & f);
static bool IsInternalVdata(int32 fid, int32 ref);

//
// hdfistream_vdata -- protected member functions
//

// initialize hdfistream_vdata
void hdfistream_vdata::_init(void) {
        _vdata_id = _index = _attr_index = _nattrs = 0;
        _meta = false;
        _vdata_refs.clear();
        _recs.set = false;
        return;
}

void hdfistream_vdata::_get_fileinfo(void) {
        // build list ref numbers of all Vdata's in the file
        int32 ref = -1;
        while ((ref = VSgetid(_file_id, ref)) != -1) {
                if (!IsInternalVdata(_file_id, ref))
                        _vdata_refs.push_back(ref);
        }
        return;
}

void hdfistream_vdata::_seek_next(void) {
        _index++;
        if (!eos())
                _seek(_vdata_refs[_index]);
        return;
}

void hdfistream_vdata::_seek(const char *name) {
        int32 ref = VSfind(_file_id, name);
        if (ref < 0)
                THROW(hcerr_vdatafind);
        else
                _seek(ref);

        return;
}

void hdfistream_vdata::_seek(int32 ref) {
        if (_vdata_id != 0)
                VSdetach(_vdata_id);
        vector<int32>::iterator r = find(_vdata_refs.begin(), _vdata_refs.end(),
                        ref);
        if (r == _vdata_refs.end())
                THROW(hcerr_vdatafind);
        _index = r - _vdata_refs.begin();
        if ((_vdata_id = VSattach(_file_id, ref, "r")) < 0) {
                _vdata_id = 0;
                THROW(hcerr_vdataopen);
        }
        _attr_index = 0;
        _nattrs = VSfnattrs(_vdata_id, _HDF_VDATA);
        return;
}

//
// hdfistream_vdata -- public member functions
//

hdfistream_vdata::hdfistream_vdata(const string filename) :
        hdfistream_obj(filename) {
        _init();
        if (_filename.length() != 0) // if ctor specified a null filename
                open(_filename.c_str());
        return;
}

void hdfistream_vdata::open(const string & filename) {
        open(filename.c_str());
        return;
}

void hdfistream_vdata::open(const char *filename) {
        if (_file_id != 0)
                close();
        if ((_file_id = Hopen(filename, DFACC_RDONLY, 0)) < 0)
                THROW(hcerr_openfile);
        if (Vstart(_file_id) < 0) // Vstart is a macro for Vinitialize
                THROW(hcerr_openfile);

        BESDEBUG("h4", "vdata file opened: id=" << _file_id << endl);

        _filename = filename;
        _get_fileinfo();
        rewind();
        return;
}

void hdfistream_vdata::close(void) {
        BESDEBUG("h4",
                        "vdata file closed: id=" << _file_id << ", this: " << this << endl);

        if (_vdata_id != 0)
                VSdetach(_vdata_id);
        if (_file_id != 0) {
                int status = Vend(_file_id); // Vend is a macro for Vfinish
                BESDEBUG("h4",
                                "vdata Vend status: " << status << ", this: " << this << endl);

                status = Hclose(_file_id);
                BESDEBUG("h4",
                                "vdata HClose status: " << status << ", this: " << this << endl);
        }
        _vdata_id = _file_id = _index = _attr_index = _nattrs = 0;
        _vdata_refs.clear(); // clear refs
        _recs.set = false;
        return;
}

void hdfistream_vdata::seek(int index) {
        if (index < 0 || index >= (int) _vdata_refs.size())
                THROW(hcerr_range);
        _seek(_vdata_refs[index]);
        _index = index;
        return;
}

void hdfistream_vdata::seek_ref(int ref) {
        _seek(ref); // _seek() sets _index
        return;
}

void hdfistream_vdata::seek(const string & name) {
        seek(name.c_str());
}

void hdfistream_vdata::seek(const char *name) {
        _seek(name);
        return;
}

bool hdfistream_vdata::setrecs(int32 begin, int32 end) {
        if (_vdata_id != 0) {
                int32 il;
                VSQueryinterlace(_vdata_id, &il);
                if (il != FULL_INTERLACE)
                        return false;
                else {
                        int32 cnt;
                        VSQuerycount(_vdata_id, &cnt);
                        if (begin < 0 || end >= cnt)
                                return false;
                        else {
                                _recs.begin = begin;
                                _recs.end = end;
                                _recs.set = true;
                        }
                }
        }
        return true;
}

// check to see if stream is positioned past the last attribute in the
// currently open Vdata
bool hdfistream_vdata::eo_attr(void) const {
        if (_filename.length() == 0) // no file open
                THROW(hcerr_invstream);
        if (eos() && !bos()) // if eos(), then always eo_attr()
                return true;
        else {
                return (_attr_index >= _nattrs); // or positioned after last Vdata attr?
        }
}

// Read all attributes in the stream
hdfistream_vdata & hdfistream_vdata::operator>>(vector<hdf_attr> &hav) {
        //    hav = vector<hdf_attr>0;  // reset vector
        for (hdf_attr att; !eo_attr();) {
                *this >> att;
                hav.push_back(att);
        }
        return *this;
}

// read all Vdata's in the stream
hdfistream_vdata & hdfistream_vdata::operator>>(vector<hdf_vdata> &hvv) {
        for (hdf_vdata hv; !eos();) {
                *this >> hv;
                hvv.push_back(hv);
        }
        return *this;
}

// read an attribute from the stream
hdfistream_vdata & hdfistream_vdata::operator>>(hdf_attr & ha) {
        // delete any previous data in ha
        ha.name = string();
        ha.values = hdf_genvec();

        if (_filename.length() == 0) // no file open
                THROW(hcerr_invstream);
        if (eo_attr()) // if positioned past last attr, do nothing
                return *this;

        char name[hdfclass::MAXSTR];
        int32 number_type, count, size;
        if (VSattrinfo(_vdata_id, _HDF_VDATA, _attr_index, name, &number_type,
                        &count, &size) < 0)
                THROW(hcerr_vdatainfo);

        // allocate a temporary C array to hold data from VSgetattr()
        char *data;
        data = new char[count * DFKNTsize(number_type)];
        if (data == 0)
                THROW(hcerr_nomemory);

        // read attribute values and store them in an hdf_genvec
        if (VSgetattr(_vdata_id, _HDF_VDATA, _attr_index, data) < 0) {
                delete[] data; // problem: clean up and throw an exception
                THROW(hcerr_vdatainfo);
        }
        // try { // try to allocate an hdf_genvec
        if (count > 0) {
                ha.values = hdf_genvec(number_type, data, count);
                // }
                // catch(...) { // problem allocating hdf_genvec: clean up and rethrow
                //    delete []data;
                //    throw;
                // }
        }
        delete[] data; // deallocate temporary C array

        // increment attribute index to next attribute
        ++_attr_index;
        ha.name = name; // assign attribute name
        return *this;
}

// read a Vdata from the stream
hdfistream_vdata & hdfistream_vdata::operator>>(hdf_vdata & hv) {

        // delete any previous data in hv
        hv.fields.clear();
        hv.vclass = hv.name = string();

        if (_vdata_id == 0)
                THROW(hcerr_invstream); // no vdata open!
        if (eos())
                return *this;

        // assign Vdata ref
        hv.ref = _vdata_refs[_index];
        // retrieve Vdata attributes
        *this >> hv.attrs;
        // retrieve Vdata name, class, number of records
        char name[hdfclass::MAXSTR];
        char vclass[hdfclass::MAXSTR];
        int32 nrecs;
        if (VSinquire(_vdata_id, &nrecs, (int32 *) 0, (char *) 0, (int32 *) 0, name)
                        < 0)
                THROW(hcerr_vdatainfo);
        hv.name = string(name);
        if (VSgetclass(_vdata_id, vclass) < 0)
                THROW(hcerr_vdatainfo);
        hv.vclass = string(vclass);

        // retrieve number of fields
        int nfields = VFnfields(_vdata_id);
        if (nfields < 0)
                THROW(hcerr_vdatainfo);

        // retrieve field information
        hv.fields = vector<hdf_field> ();
        for (int i = 0; i < nfields; ++i) {
                hv.fields.push_back(hdf_field());
                if (_meta)
                        LoadField(_vdata_id, i, 0, 0, hv.fields[i]);
                else if (_recs.set)
                        LoadField(_vdata_id, i, _recs.begin, _recs.end, hv.fields[i]);
                else
                        LoadField(_vdata_id, i, 0, nrecs - 1, hv.fields[i]);
        }
        _seek_next();
        return *this;
}

bool hdfistream_vdata::isInternalVdata(int ref) const {
        set<string, less<string> > reserved_names;
        reserved_names.insert("RIATTR0.0N");

        set<string, less<string> > reserved_classes;
        reserved_classes.insert("Attr0.0");
        reserved_classes.insert("RIATTR0.0C");
        reserved_classes.insert("DimVal0.0");
        reserved_classes.insert("DimVal0.1");
        reserved_classes.insert("_HDF_CHK_TBL_0");

        // get name, class of vdata
        int vid;
        if ((vid = VSattach(_file_id, ref, "r")) < 0) {
                THROW(hcerr_vdataopen);
        }
        char name[hdfclass::MAXSTR];
        char vclass[hdfclass::MAXSTR];
        if (VSgetname(vid, name) < 0) {
                VSdetach(vid);
                THROW(hcerr_vdatainfo);
        }
        if (reserved_names.find(string(name)) != reserved_names.end()) {
                VSdetach(vid);
                return true;
        }

        if (VSgetclass(vid, vclass) < 0) {
                VSdetach(vid);
                THROW(hcerr_vdatainfo);
        }

        VSdetach(vid);

        if (reserved_classes.find(string(vclass)) != reserved_classes.end())
                return true;

        return false;
}

static void LoadField(int32 vid, int index, int32 begin, int32 end,
                hdf_field & f) {
        DBG(cerr << "LoadField - vid: " << vid << endl);

        // position to first record too read
        if (VSseek(vid, begin) < 0)
                THROW(hcerr_vdataseek);
        int32 nrecs = end - begin + 1;

        // retrieve name of field
        DBG(cerr << "vid: " << vid << ", index: " << index << endl);
        char *fieldname = VFfieldname(vid, index);
        if (fieldname == 0)
                THROW(hcerr_vdatainfo);
        f.name = string(fieldname);

        // retrieve order of field
        int32 fieldorder = VFfieldorder(vid, index);
        if (fieldorder < 0)
                THROW(hcerr_vdatainfo);

        // retrieve size of the field in memory
        int32 fieldsize = VFfieldisize(vid, index);
        if (fieldsize < 0)
                THROW(hcerr_vdatainfo);

        // retrieve HDF data type of field
        int32 fieldtype = VFfieldtype(vid, index);
        if (fieldtype < 0)
                THROW(hcerr_vdatainfo);

        // for each component, set type and optionally load data
        hdf_genvec gv;
        vector<char> data;
        // char *data = 0;
        if (nrecs > 0) { // if nrecs > 0 then load data for field
                // data = new char[fieldsize * nrecs];
                data.resize(fieldsize * nrecs);
                DBG(cerr << "LoadField: vid=" << vid << ", fieldname=" << fieldname << endl);
                // TODO: Is this correct?
                // This code originally treated a negative return as a failure and
                // threw an exception. I'm still waiting on a verdict, but it seems
                // that the negative return may have been indicating an empty vdata
                // (or SDS dataset?) instead. For now, simply calling return here
                // addresses a problem where some NASA data files cannot otherwise be
                // read. See Trac ticket http://scm.opendap.org/trac/ticket/1793.
                // jhrg 8/17/11
                if (VSsetfields(vid, fieldname) < 0) {
                        return;
                        // throw InternalErr(__FILE__, __LINE__, "VSsetfields");
                }

                if (VSread(vid, (uchar8 *)&data[0], nrecs, FULL_INTERLACE) < 0) {
                        throw InternalErr(__FILE__, __LINE__, "VSread error with the field: " + f.name + " (" + long_to_string(vid) + ").");
                }
#if 0
                if ((VSsetfields(vid, fieldname) < 0) || (VSread(vid, (uchar8 *) data,
                                nrecs, FULL_INTERLACE) < 0)) {
                        delete[] data;
                        throw hcerr_vdataread(__FILE__, __LINE__);
                }
#endif
        }
        int stride = fieldorder;
        for (int i = 0; i < fieldorder; ++i) {
                if (nrecs == 0)
                        gv = hdf_genvec(fieldtype, 0, 0, 0, 0);
                else
                        gv = hdf_genvec(fieldtype, &data[0], i, (nrecs * fieldorder) - 1, stride);
                f.vals.push_back(gv);
        }
        // delete[] data;
}

//
// hdf_vdata related member functions
//

// verify that the hdf_field class is in an OK state.
bool hdf_field::_ok(void) const {

        // make sure that there is some data stored in the hdf_field
        if (vals.size() == 0)
                return false;

        // if the field has order > 1, check to make sure that the number types of all
        // the columns in the field are the same and are not 0
        if (vals.size() > 1) {
                int32 nt = vals[0].number_type();
                if (nt == 0)
                        return false;
                for (int i = 1; i < (int) vals.size(); ++i)
                        if (vals[i].number_type() != nt || vals[i].number_type() == 0)
                                return false;
        }

        return true; // passed all the tests
}

bool hdf_vdata::_ok(void) const {

        // make sure there are fields stored in this vdata
        if (fields.size() == 0)
                return false;

        // make sure the fields are themselves OK
        for (int i = 0; i < (int) fields.size(); ++i)
                if (!fields[i])
                        return false;

        return true; // passed all the tests
}

bool IsInternalVdata(int32 fid, int32 ref) {
        set<string, less<string> > reserved_names;
        reserved_names.insert("RIATTR0.0N");

        set<string, less<string> > reserved_classes;
        reserved_classes.insert("Attr0.0");
        reserved_classes.insert("RIATTR0.0C");
        reserved_classes.insert("DimVal0.0");
        reserved_classes.insert("DimVal0.1");
        reserved_classes.insert("_HDF_CHK_TBL_0");

        // get name, class of vdata
        int vid;
        if ((vid = VSattach(fid, ref, "r")) < 0) {
                THROW(hcerr_vdataopen);
        }
        char name[hdfclass::MAXSTR];
        char vclass[hdfclass::MAXSTR];
        if (VSgetname(vid, name) < 0) {
                VSdetach(vid);
                THROW(hcerr_vdatainfo);
        }
        if (reserved_names.find(string(name)) != reserved_names.end()) {
                VSdetach(vid);
                return true;
        }

        if (VSgetclass(vid, vclass) < 0) {
                VSdetach(vid);
                THROW(hcerr_vdatainfo);
        }

        VSdetach(vid);

        if (reserved_classes.find(string(vclass)) != reserved_classes.end())
                return true;

        return false;
}