ImportPdsTable.cpp

 
/* SPDX-License-Identifier: CC0-1.0 */
#include "ImportPdsTable.h"
 
#include <cctype>
#include <iomanip>
#include <iostream>
#include <sstream>
 
#include <QDebug>
#include <QRegularExpression>
#include <QScopedPointer>
#include <QString>
 
#include "EndianSwapper.h"
#include "FileName.h"
#include "IString.h"
#include "Pvl.h"
#include "PvlObject.h"
#include "Table.h"
#include "TableField.h"
#include "TableRecord.h"
#include "TextFile.h"
 
 
using namespace std;
 
namespace Isis {
 

  ImportPdsTable::ImportPdsTable() {
    // just inititalize the member variables
    init();
    m_tableName = "TABLE";
  }
 

  ImportPdsTable::ImportPdsTable(const QString &pdsLabFile,
                                 const QString &pdsTableFile,
                                 const QString &pdsTableName) {
    m_tableName = pdsTableName;
    load(pdsLabFile, pdsTableFile, pdsTableName);
  }
 

  ImportPdsTable::~ImportPdsTable() {
  }
 

  QString ImportPdsTable::name() const {
    return (m_tableName);
  }

  void ImportPdsTable::setName(const QString &name) {
    m_tableName = name;
    return;
  }
 

  void ImportPdsTable::load(const QString &pdsLabFile,
                            const QString &pdsTableFile,
                            const QString &pdsTableName) {
 
    init();
    QString tempTblFile;
    loadLabel(pdsLabFile, tempTblFile, pdsTableName);
    if (!pdsTableFile.isEmpty()) tempTblFile = pdsTableFile;
    //  Vet the table filename.  Many PDS files record the filename in
    //  uppercase and in practice, the filename is in lowercase.  Do this
    //  check here.
    FileName tableFile(tempTblFile);
    try {
      int tableStartRecord = toInt(tableFile.baseName());
      tempTblFile = pdsLabFile;
      m_pdsTableStart = tableStartRecord;
    }
    catch (IException &e) {
      // if we are unable to cast the table file value to an integer, it must be a
      // file name, not a location in the label file.
      if (!tableFile.fileExists()) {
        // if the table file name doesn't exist, try lowercased version...
        FileName tableFileLowercase(tableFile.path() + "/"
                                    + tableFile.name().toLower());
        if (!tableFileLowercase.fileExists()) {
          IString msg = "Unable to import PDS table.  Neither of the following "
                        "possible table files were found: ["
                        + tableFile.expanded() + "]  or ["
                        + tableFileLowercase.expanded() + "]";
          throw IException(e, IException::Unknown, msg, _FILEINFO_);
        }
        tableFile = tableFileLowercase.expanded();
        tempTblFile = tableFile.expanded();
      }
      m_pdsTableStart = 1;
    }
    if (m_pdsTableType == "ASCII") {
      loadTable(tempTblFile);
    }
    m_pdsTableFile = tempTblFile;
    return;
  }
 

  bool ImportPdsTable::hasColumn(const QString &colName) const {
     return (findColumn(colName) != 0);
  }
 

  QString ImportPdsTable::getColumnName(const unsigned int &index,
                                        const bool &formatted) const {
    if ((int) index >= columns() - 1) {
      QString msg = "Unable to import the binary PDS table [" + m_tableName
                    + "] into Isis. The requested column index ["
                    + toString((int) index) + "] exceeds the last column index ["
                    + toString(columns() - 1) + "]";
      throw IException(IException::Programmer, msg.toStdString(), _FILEINFO_);
    }
    QString name = m_coldesc[index].m_name;
    if (formatted) name = getFormattedName(name);
    return (name);
  }
 

  QStringList ImportPdsTable::getColumnNames(const bool &formatted) const {
    QStringList colnames;
    for (int i = 0 ; i < columns() ; i++) {
      QString name = m_coldesc[i].m_name;
      if (formatted) name = getFormattedName(name);
      colnames.push_back(name);
    }
    return (colnames);
  }
 

  QString ImportPdsTable::getType(const QString &colName) const {
    const ColumnDescr *column = findColumn(colName);
    QString dtype("");
    if (column != 0) {
      dtype = column->m_dataType;
    }
    return (dtype);
  }
 

  bool ImportPdsTable::setType(const QString &colName,
                               const QString &dataType) {
    ColumnDescr *column = findColumn(colName);
    if (column != 0) {
      column->m_dataType = dataType.toUpper();
    }
    return (column != 0);
  }
 

  Table ImportPdsTable::importTable(const QString &isisTableName) {
    try {
      TableRecord record = makeRecord(m_coldesc);
      Table table(isisTableName, record);
      fillTable(table, m_coldesc, record);
      return (table);
    }
    catch (IException &e) {
      QString msg = "Unable to import the PDS table [" + m_tableName
                    + "] from the PDS file [" + m_pdsTableFile + "] into Isis.";
      throw IException(e, IException::Unknown, msg.toStdString(), _FILEINFO_);
 
    }
  }
 

  Table ImportPdsTable::importTable(const QString &colnames,
                                    const QString &isisTableName) {
    return (importTable(colnames.split(","), isisTableName));
  }
 

  Table ImportPdsTable::importTable(const QStringList &colnames,
                                    const QString &isisTableName) {
    ColumnTypes ctypes;
    for (int i = 0 ; i < colnames.size() ; i++) {
      const ColumnDescr *descr = findColumn(colnames[i]);
      if (!descr) {
        QString msg = "Unable to import the PDS table [" + m_tableName
                      + "] into Isis. The requested column name ["
                      + colnames[i] + "] does not "
                      "exist in table.";
        throw IException(IException::Programmer, msg.toStdString(), _FILEINFO_);
      }
      ctypes.push_back(*descr);
    }
 
    // Create and populate the table
    TableRecord record = makeRecord(ctypes);
    Table table(isisTableName, record);
    fillTable(table, ctypes, record);
    return (table);
  }
 

  void ImportPdsTable::init() {
 
    m_byteOrder = "";
    m_trows = 0;
    m_pdsTableStart = 0;
    m_coldesc.clear();
    m_rows.clear();
    m_pdsTableType = "";
    m_pdsTableFile = "";
    return;
  }
 

  void ImportPdsTable::loadLabel(const QString &pdsLabFile,
                                 QString &pdsTableFile,
                                 const QString &tblname) {
 
    Isis::Pvl label(pdsLabFile);
 
    QString tableName = ( tblname.isEmpty() ) ? m_tableName : tblname;
    if (!label.hasObject(tableName)) {
      QString msg = "The PDS file " + pdsLabFile +
                    " does not have the required TABLE object, ["
                    + tableName +"]. The PDS label file is probably invalid";
      throw IException(IException::Unknown, msg.toStdString(), _FILEINFO_);
    }
    //  Get some pertinent information from the label
    PvlObject &tabObj = label.findObject(tableName);
    // The table description contains the actual "RECORD_BYTES"
    if (tabObj.hasKeyword("RECORD_BYTES")) {
      m_recordBytes = (int) tabObj.findKeyword("RECORD_BYTES");
    }
    // The table description has "ROW_BYTES" and "ROW_SUFFIX_BYTES". These summed is the
    // record length. Can be for detached and attached labels
    else if (tabObj.hasKeyword("ROW_BYTES") && tabObj.hasKeyword("ROW_SUFFIX_BYTES")) {
      m_recordBytes = (int) tabObj.findKeyword("ROW_BYTES") +
                      (int) tabObj.findKeyword("ROW_SUFFIX_BYTES");
    }
    // The table record length is defined by the file record
    // length (i.e., table is in with the image)
    else {
      m_recordBytes = (int) label.findKeyword("RECORD_BYTES");
    }
 
    QString trueTableName;
    PvlObject *tableDetails = &tabObj;
    if (label.hasKeyword("^" + tableName)) {
      trueTableName = tableName;
      pdsTableFile = FileName(pdsLabFile).path() + "/"
                     + label["^" + tableName][0];
    }
    else if (tabObj.objects() == 1) {
      trueTableName = tabObj.object(0).name();
      tableDetails = &tabObj.object(0);
      pdsTableFile = FileName(pdsLabFile).path() + "/"
                     + tabObj["^" + trueTableName][0];
    }
    m_trows = (int) tableDetails->findKeyword("ROWS");
    int ncols =  (int) tableDetails->findKeyword("COLUMNS");
    m_pdsTableType = QString(tableDetails->findKeyword("INTERCHANGE_FORMAT"));
    if (m_pdsTableType != "ASCII" && m_pdsTableType.toUpper() != "BINARY") {
      QString msg = "Unable to import the PDS table [" + tableName
                    + "] from the PDS file ["
                    + pdsTableFile + "] into Isis. "
                    "The PDS INTERCHANGE_FORMAT [" + m_pdsTableType
                    + "] is not supported. Valid values are ASCII or BINARY.";
      throw IException(IException::User, msg.toStdString(), _FILEINFO_);
    }
    m_rowBytes = tableDetails->findKeyword("ROW_BYTES");
 
    m_coldesc.clear();
    PvlObject::PvlObjectIterator colobj = tableDetails->beginObject();
    int icol(0);
    while (colobj != tableDetails->endObject()) {
      if (colobj->isNamed("COLUMN")) {
        m_coldesc.push_back(getColumnDescription(*colobj, icol));
        icol++;
      }
      ++colobj;
    }
 
    //  Test to ensure columns match the number listed in the label
    if (ncols != columns()) {
      ostringstream msg;
      msg << "Number of columns in the COLUMNS label keyword (" << ncols
           << ") does not match number of COLUMN objects found ("
           << columns() << ")";
      cout << msg.str() << endl;
    }
    return;
  }
 

  void ImportPdsTable::loadTable(const QString &pdsTableFile) {
 
    //  Vet the filename.  Many PDS files record the filename in uppercase and
    //  in practice, the filename is in lowercase.  Do this check here.
    QString tempTblFile(pdsTableFile);
    TextFile tfile(tempTblFile);
    QString tline;
    m_rows.clear();
    int irow(0);
    while (tfile.GetLine(tline, false)) {
      if (irow >= m_trows) break;
 
      (void) processRow(irow, tline);
 
      irow++;
    }
    return;
  }
 

  ImportPdsTable::ColumnDescr
      ImportPdsTable::getColumnDescription(PvlObject &colobj, int nth) const {
    ColumnDescr cd;
    cd.m_name = colobj["NAME"][0];
    cd.m_colnum = nth;    // 0-based indexing, will be COLUMN_NUM - 1
 
    if (m_pdsTableType == "ASCII") {
      cd.m_dataType = getGenericType(colobj["DATA_TYPE"][0]).toUpper();
    }
    else {
      cd.m_dataType = colobj["DATA_TYPE"][0].toUpper();
    }
 
    cd.m_startByte = ((int) colobj["START_BYTE"]) - 1;   // 0-based indexing
    cd.m_numBytes = (int) colobj["BYTES"];
 
 
    cd.m_itemBytes = cd.m_numBytes;
    if ( colobj.hasKeyword("ITEM_BYTES") ) {
      cd.m_itemBytes = (int) colobj["ITEM_BYTES"];
    }
 
    cd.m_items = 1;
    if ( colobj.hasKeyword("ITEMS") ) {
      cd.m_items = (int) colobj["ITEMS"];
    }
 
    return (cd);
  }
 

  const ImportPdsTable::ColumnDescr &ImportPdsTable::getColumnDescriptor(const int &nth) const {
 
    // Ensure the nrequested column is valid
    if ( (nth >= columns()) || ( nth < 0) ) {
      QString mess = "Index (" + QString::number(nth) +
      ") into Columns invalid (max: " + QString::number(columns()) + ")";
      throw IException(IException::Programmer, mess, _FILEINFO_);
    }
 
    // All good, return the expected
    return (m_coldesc[nth]);
  }
 

  ImportPdsTable::ColumnDescr *ImportPdsTable::findColumn(const QString &colName) {
    QString cName = getFormattedName(colName);
    ColumnTypes::iterator col = m_coldesc.begin();
    while (col != m_coldesc.end()) {
      QString c = getFormattedName(col->m_name);
      if (c.toUpper() == cName.toUpper()) { return (&(*col));  }
      col++;
    }
    return (0);
  }
 

  const ImportPdsTable::ColumnDescr *ImportPdsTable::findColumn(const QString &colName) const {
    QString cName = getFormattedName(colName);
    ColumnTypes::const_iterator col = m_coldesc.begin();
    while (col != m_coldesc.end()) {
      QString c = getFormattedName(col->m_name);
      if (c.toUpper() == cName.toUpper()) { return (&(*col));  }
      col++;
    }
    return (0);
  }
 

  QString ImportPdsTable::getColumnValue(const QString &tline,
                                         const ColumnDescr &cdesc,
                                         const QString &delimiter) const {
    return (tline.mid(cdesc.m_startByte, cdesc.m_numBytes));
  }
 

  QStringList ImportPdsTable::getColumnFields(const QString &tline,
                                              const ColumnDescr &cdesc,
                                              const QString &delimiter) const {
 
    // If item count is 1, simply return the whole column
    QString value = getColumnValue(tline, cdesc, delimiter);
    if ( 1 == cdesc.m_items) return ( QStringList(value) );
 
    // If there is no item size specified, see if we should seperate with
    // delimiter
    if ( 0 == cdesc.m_itemBytes ) {
      if ( delimiter.isEmpty() ) return ( QStringList(value));
 
      return ( value.split(delimiter, Qt::SkipEmptyParts) );
    }
 
    // Have an item size specified.  Assume it has single character separator
    QStringList fields;
    int pos = 0;
    for (int i = 0 ; i < cdesc.m_items ; i++) {
      fields.push_back(value.mid(pos, cdesc.m_itemBytes));
      pos += cdesc.m_itemBytes + 1;
    }
 
    return (fields);
  }
 

  QString  ImportPdsTable::getFormattedName(const QString &colname) const {
    QString cname = QString(colname).replace(QRegularExpression("[(),]"), " ").simplified();
 
    bool uppercase = true;
    QString oString;
    for (int i = 0 ; i < cname.size() ; i++) {
      if (uppercase) {
        oString.push_back(cname[i].toUpper());
        uppercase = false;
      }
      else if ( (cname[i] == ' ') || (cname[i] == '_') ) {
        uppercase = true;
      }
      else {
        oString.push_back(cname[i].toLower());
      }
    }
 
    return (oString);
  }
 

  QString ImportPdsTable::getGenericType(const QString &ttype) const {
    return ttype.split("_").last();
  }
 

  TableField ImportPdsTable::makeField(const ColumnDescr &cdesc) {
    QString dtype = cdesc.m_dataType;
    QString name = getFormattedName(cdesc.m_name);
    if (m_pdsTableType == "ASCII") {
      if ( dtype == "INTEGER" ) {
        return (TableField(name, TableField::Integer));
      }
      else if ( ((dtype == "DOUBLE")
              || (dtype == "REAL")
              || (dtype == "FLOAT")) ) {
        return (TableField(name, TableField::Double));
      }
      else {
        return (TableField(name, TableField::Text, cdesc.m_numBytes));
      }
    }
    else {
      return makeFieldFromBinaryTable(cdesc);
    }
  }
 

  TableRecord ImportPdsTable::makeRecord(const ColumnTypes &ctypes) {
    TableRecord rec;
    for (int i = 0 ; i < ctypes.size() ; i++) {
      TableField field = makeField(ctypes[i]);
      rec += field;
    }
    return (rec);
  }
 

  TableField &ImportPdsTable::extract(const Columns &cols,
                                      const ColumnDescr &cdesc,
                                      TableField &tfield) const {
    int ith = cdesc.m_colnum;
    try {
      IString data(cols[ith]);
      if (tfield.isInteger()) {
        data.Trim(" \t\r\n");
        tfield = data.ToInteger();
      }
      else if (tfield.isDouble()) {
        data.Trim(" \t\r\n");
        tfield = data.ToDouble();
      }
      else {  // Its a text field
        QString str(tfield.size(), ' ');
        str.insert(0, data.Trim(" \t\r\n").ToQt());
        tfield = str;
      }
    }
    catch (IException &e) {
      QString msg = "Conversion failure of column " + cdesc.m_name;
      throw IException(e, IException::Programmer, msg, _FILEINFO_);
    }
 
    return (tfield);
  }
 

  TableRecord &ImportPdsTable::extract(const Columns &cols,
                                       const ColumnTypes &ctypes,
                                       TableRecord &record) const {
    for (int i = 0 ; i < ctypes.size() ; i++) {
      extract(cols, ctypes[i], record[i]);
    }
    return (record);
  }
 

  void ImportPdsTable::fillTable(Table &table,
                                 const ColumnTypes &cols,
                                 TableRecord &record) const {
    if (m_pdsTableType == "ASCII") {
      for (int i = 0 ; i < m_rows.size() ; i++) {
        try {
          table += extract(m_rows[i], cols, record);
        }
        catch (IException &e) {
          QString msg = "Failed to convert data in row [" + toString((int) i) + "]";
          throw IException(e, IException::Programmer, msg, _FILEINFO_);
        }
      }
    }
 
    else {
      QString tempTblFile = m_pdsTableFile;
      ifstream pdsFileStream(tempTblFile.toLatin1().data(), ifstream::binary);
 
      if (!pdsFileStream) {
        IString msg = "Unable to open file containing PDS table ["
                      + tempTblFile + "].";
        throw IException(IException::Unknown, msg, _FILEINFO_);
      }
 
      // read and discard the rows above the table data
      QScopedPointer<char, QScopedPointerArrayDeleter<char> > rowBuffer(new char[m_recordBytes]);
      for (int i = 1; i < m_pdsTableStart; i++) {
        pdsFileStream.read(rowBuffer.data(), m_recordBytes);
      }
      // now, import and add the records to the table
      for (int rowIndex = 0; rowIndex < m_trows; rowIndex++) {
        pdsFileStream.read(rowBuffer.data(), m_recordBytes);
        TableRecord rec = extractBinary(rowBuffer.data(), record);
        table += rec;
      }
    }
 
    return;
  }
 

  int ImportPdsTable::columns() const {
    return (m_coldesc.size());
  }
 

  int ImportPdsTable::rows() const {
    return (m_rows.size());
  }
 

  TableRecord ImportPdsTable::extractBinary(char *rowBuffer, TableRecord &record) const {
    // for each record loop through the columns to get field values
    for (int colIndex = 0; colIndex < columns(); colIndex++) {
      QString columnName = m_coldesc[colIndex].m_name;
      for (int fieldIndex = 0 ; fieldIndex < record.Fields() ; fieldIndex++) {
        QString fieldName = record[fieldIndex].name();
        if (fieldName == columnName) {
          int startByte = m_coldesc[colIndex].m_startByte;
          int numBytes = m_coldesc[colIndex].m_numBytes;
 
          if (record[fieldIndex].isInteger()) {
            int columnValue;
            memmove(&columnValue, &rowBuffer[startByte], numBytes);
            EndianSwapper endianSwap(m_byteOrder);
            int fieldValue = endianSwap.Int(&columnValue);
            record[fieldIndex] = fieldValue;
          }
 
          else if (record[fieldIndex].isDouble()) {
            EndianSwapper endianSwap(m_byteOrder);
            double columnValue;
            memmove(&columnValue, &rowBuffer[startByte], numBytes);
            double fieldValue = endianSwap.Double(&columnValue);
            record[fieldIndex] = fieldValue;
          }
 
          else if (record[fieldIndex].isReal()) {
            EndianSwapper endianSwap(m_byteOrder);
            float columnValue;
            memmove(&columnValue, &rowBuffer[startByte], numBytes);
            float fieldValue = endianSwap.Float(&columnValue);
            record[fieldIndex] = fieldValue;
          }
 
          else if (record[fieldIndex].isText()) {
            QString fieldValue(numBytes, '\0');
            for (int byte = 0; byte < numBytes; byte++) {
              fieldValue[byte] = rowBuffer[startByte + byte];
            }
            record[fieldIndex] = fieldValue;
          }
 
        }
      }
    }
    return record;
  }
 

  TableField ImportPdsTable::makeFieldFromBinaryTable(const ColumnDescr &cdesc){
    QString dataType = cdesc.m_dataType;
    // For binary tables, we will not reformat the name of the column
    QString name = cdesc.m_name;
    if (dataType == "MSB_INTEGER" || dataType == "INTEGER"
        || dataType == "SUN_INTEGER" || dataType == "MAC_INTEGER") {
      if (cdesc.m_numBytes != 4) {
        QString msg = "Only 4 byte integer values are supported in Isis. "
                      "PDS Column [" + cdesc.m_name
                      + "] has an integer DATA_TYPE with [BYTES = "
                      + toString(cdesc.m_numBytes) + "].";
        throw IException(IException::Unknown, msg, _FILEINFO_);
      }
      setPdsByteOrder("MSB");
      return TableField(name, TableField::Integer);
    }
    else if (dataType == "LSB_INTEGER" || dataType == "VAX_INTEGER"
             || dataType == "PC_INTEGER" ) {
      if (cdesc.m_numBytes != 4) {
        QString msg = "Only 4 byte integer values are supported in Isis. "
                      "PDS Column [" + cdesc.m_name
                      + "] has an integer DATA_TYPE with [BYTES = "
                      + toString(cdesc.m_numBytes) + "].";
        throw IException(IException::Unknown, msg, _FILEINFO_);
      }
      setPdsByteOrder("LSB");
      return TableField(name, TableField::Integer);
    }
    else if (dataType == "FLOAT"    //IEEE_REAL alias (MSB)
             || dataType == "REAL"     //IEEE_REAL alias (MSB)
             || dataType == "SUN_REAL" //IEEE_REAL alias (MSB)
             || dataType == "MAC_REAL" //IEEE_REAL alias (MSB)
             || dataType == "IEEE_REAL" ) {
      setPdsByteOrder("MSB");
      if (cdesc.m_numBytes == 8) {
        return TableField(name, TableField::Double);
      }
      else if (cdesc.m_numBytes == 4) {
        return TableField(name, TableField::Real);
      }
      else {
        IString msg = "Only 4 byte or 8 byte real values are supported in Isis. "
                      "PDS Column [" + cdesc.m_name
                      + "] has a real DATA_TYPE with [BYTES = "
                      + toString(cdesc.m_numBytes) + "].";
        throw IException(IException::Unknown, msg, _FILEINFO_);
      }
    }
    else if (dataType == "PC_REAL") {
      setPdsByteOrder("LSB");
      if (cdesc.m_numBytes == 8) {
        return TableField(name, TableField::Double);
      }
      else if (cdesc.m_numBytes == 4) {
        return TableField(name, TableField::Real);
      }
      else {
        QString msg = "Only 4 byte or 8 byte real values are supported in Isis. "
                      "PDS Column [" + cdesc.m_name
                      + "] has a real DATA_TYPE with [BYTES = "
                      + toString(cdesc.m_numBytes) + "].";
        throw IException(IException::Unknown, msg, _FILEINFO_);
      }
    }
    else if (dataType.contains("CHARACTER")
             || dataType.contains("ASCII")
             || dataType == "DATE" || dataType == "TIME" ) {
      return TableField(name, TableField::Text, cdesc.m_numBytes);
    }
    // Isis tables currently don't support any of the following PDS DATA_TYPE:
    // BIT_STRING, COMPLEX, N/A, BOOLEAN, UNSIGNED_INTEGER, IBM types, some VAX types
    IString msg = "PDS Column [" + cdesc.m_name
                  + "] has an unsupported DATA_TYPE ["
                  + dataType + "].";
    throw IException(IException::Unknown, msg, _FILEINFO_);
  }
 

  void ImportPdsTable::setPdsByteOrder(QString byteOrder) {
    if (!m_byteOrder.isEmpty() && m_byteOrder != byteOrder) {
      QString msg = "Unable to import the binary PDS table [" + m_tableName
                    + "]. The column DATA_TYPE values indicate differing byte "
                      "orders. ";
      throw IException(IException::Unknown, msg.toStdString(), _FILEINFO_);
    }
    m_byteOrder = byteOrder;
  }
 

  bool ImportPdsTable::processRow(const int &row, const QString &rowdata) {
    Columns cols;
    for (int i = 0 ; i < columns() ; i++) {
      cols.push_back(getColumnValue(rowdata, m_coldesc[i]));
    }
    m_rows.append(cols);
 
    return (true);
  }
 
}  //  namespace Isis