LoadLog.cpp

/*WIKI* 

'''Parameters Note:''' Note that it is possible to use both of the optional 'spectrum' properties (i.e. a range and a list) together if so desired.

===Load ISIS log file(s)===
Assumes that a log file originates from a PC (not VMS) environment, i.e. the log files to be loaded are assumed to have the extension .txt. Its filename is assumed to starts with the raw data file identifier followed by the character '_', and a log file is assumed to have a format of two columns, where the first column consists of data-time strings of the ISO 8601 form and the second column consists of either numbers or strings that may contain spaces.

===Parent algorithm===
LoadLog is also a child algorithm of [[LoadRaw]], i.e. it gets called whenever LoadRaw is executed. 

===Load SNS text log file===
If the file is determined to be a SNS text log file it should be of the form

 655747325.450625	0.000000	24.000000	26.000000	0.000000
 655747325.716250	0.296875	24.000000	26.000000	0.000000
 655747325.997500	0.593750	24.000000	26.000000	0.000000
 655747326.263125	0.906250	24.000000	26.000000	0.000000
 655747326.544375	1.093750	24.000000	26.000000	0.000000
 655747326.825625	1.406250	24.000000	26.000000	0.000000
 655747327.091250	1.703125	24.000000	26.000000	0.000000
 655747327.372500	2.000000	24.000000	26.000000	0.000000
 655747327.638125	2.203125	24.000000	26.000000	0.000000
 655747327.919375	2.500000	24.000000	26.000000	0.000000
 655747328.200625	2.796875	24.000000	26.000000	0.000000
 655747328.466250	3.093750	24.000000	26.000000	0.000000

The first column is the number of seconds since January 1, 1990, then the other columns (space delimited) are the log values. For this mode the ''name'' and ''units'' parameters must be specified.


*WIKI*/
//----------------------------------------------------------------------
// Includes
//----------------------------------------------------------------------
#include "LoadRaw/isisraw2.h"
#include "MantidAPI/FileProperty.h"
#include "MantidDataHandling/LoadLog.h"
#include "MantidDataObjects/Workspace2D.h"
#include "MantidKernel/ArrayProperty.h"
#include "MantidKernel/Glob.h"
#include "MantidKernel/LogParser.h"
#include "MantidKernel/Strings.h"
#include "MantidKernel/PropertyWithValue.h"
#include "MantidKernel/TimeSeriesProperty.h"

#include <boost/regex.hpp>
#include <boost/algorithm/string.hpp>
#include <Poco/File.h>
#include <Poco/Path.h>
#include <Poco/DirectoryIterator.h>
#include <Poco/DateTimeParser.h>
#include <Poco/DateTimeFormat.h>
#include <fstream>  // used to get ifstream
#include <sstream>

namespace Mantid
{
  namespace DataHandling
  {
    // Register the algorithm into the algorithm factory
    DECLARE_ALGORITHM(LoadLog)

    /// Sets documentation strings for this algorithm
    void LoadLog::initDocs()
    {
      this->setWikiSummary("Load ISIS log file(s) or a SNS text log file into a [[workspace]].");
      this->setOptionalMessage("Load ISIS log file(s) into a workspace.");
    }

    using namespace Kernel;
    using API::WorkspaceProperty;
    using API::MatrixWorkspace;
    using API::MatrixWorkspace_sptr;
    using API::FileProperty;
    using DataObjects::Workspace2D;
    using DataObjects::Workspace2D_sptr;

    /// Empty default constructor
    LoadLog::LoadLog()
    {}

    /// Initialisation method.
    void LoadLog::init()
    {
      // When used as a Child Algorithm the workspace name is not used - hence the "Anonymous" to satisfy the validator
      declareProperty(
        new WorkspaceProperty<MatrixWorkspace>("Workspace","Anonymous",Direction::InOut),
                          "The name of the workspace to which the log data will be added.");

      std::vector<std::string> exts(4, "");
      exts[0] = ".txt";
      exts[1] = ".raw";
      exts[2] = ".s*";
      exts[3] = ".add";
      declareProperty(new FileProperty("Filename", "", FileProperty::Load, exts),
        "The filename (including its full or relative path) of a SNS text log file (not cvinfo), "
        "an ISIS log file, or an ISIS raw file. "
        "If a raw file is specified all log files associated with "
        "that raw file are loaded into the specified workspace. The file extension must "
        "either be .raw or .s when specifying a raw file");

      declareProperty(new ArrayProperty<std::string>("Names"),
        "For SNS-style log files only: the names of each column's log, separated by commas. "
        "This must be one fewer than the number of columns in the file.");

      declareProperty(new ArrayProperty<std::string>("Units"),
        "For SNS-style log files only: the units of each column's log, separated by commas. "
        "This must be one fewer than the number of columns in the file. "
        "Optional: leave blank for no units in any log.");

    }

    /**
     * Executes the algorithm. Reading in ISIS log file(s)
     * @throw Mantid::Kernel::Exception::FileError  Thrown if file is not recognised to be a raw datafile or log file
     * @throw std::runtime_error Thrown with Workspace problems
     */
    void LoadLog::exec()
    {
      // Retrieve the filename from the properties and perform some initial checks on the filename
      m_filename = getPropertyValue("Filename");

      // File property checks whether the given path exists, just check that is actually a file
      Poco::File l_path( m_filename );
      if ( l_path.isDirectory() )
      {
        throw Exception::FileError("Filename is a directory:" , m_filename);
      }

      // Get the input workspace and retrieve run from workspace.
      // the log file(s) will be loaded into the run object of the workspace
      const MatrixWorkspace_sptr localWorkspace = getProperty("Workspace");

      if ( isAscii(m_filename) )
      {
        // Is it a SNS style file? If so, we load it and abort.
        if ( LoadSNSText() )
        {
          return;
        } // Otherwise we continue.
      }

      //open file, have method to extract number of columns...
      //if 2, use 2 column method, if 3 use 3...
      //findNumberColumns();
      //REMEMBER - reset stream. (pass it around, in correct position).
      //seekg
      //pass fstreamed object to the method, as to re-use it
      // make sure to pass the stream of the file to all methods using m_filename
      // make & take stream rather than filename.

      // Check if a three column log file exists. (Calculated using m_filename)
      std::string threecolumnLogfile = getThreeColumnName();
      if ( !threecolumnLogfile.empty() )
      {
        //restructure this method to take ifstream, and better name (see below)
        // pass the rub object to load two column log file. Both should have same signature.
        createthreecolumnFileLogProperty( threecolumnLogfile,localWorkspace->mutableRun() );
        return;
      }

      // Get the log file names if provided.
      std::vector<std::string> names = getProperty("Names");
      //If there's more than one log name provided, then it's an invalid ISIS file.
      if (names.size() > 1)
      {
        throw std::invalid_argument("More than one log name provided. Invalid ISIS log file.");
        return;
      }
      // Otherwise, we load the ISIS log file.
      loadTwoColumnLogFile(extractLogName(names));
    }

    /**
     * Load an ISIS log file into the local workspace.
     * @param logFileName :: The name of the log file to load.
     */
    void LoadLog::loadTwoColumnLogFile(std::string logFileName)
    {
      // Get the input workspace and retrieve run from workspace.
      // the log file(s) will be loaded into the run object of the workspace
      const MatrixWorkspace_sptr localWorkspace = getProperty("Workspace");

      std::ifstream inLogFile(m_filename.c_str());

      if (!inLogFile)
      {
        g_log.warning("Unable to open file " + m_filename);
      }

      // figure out if second column is a number or a string
      std::string aLine;
      if( Mantid::Kernel::extractToEOL(inLogFile,aLine) )
      {
        if ( !isDateTimeString(aLine) )
        {
          g_log.warning("File" + m_filename + " is not a standard ISIS log file. Expected to be a two column file.");
          inLogFile.close();
        }

        std::string DateAndTime;
        std::stringstream ins(aLine);
        ins >> DateAndTime;

        // read in what follows the date-time string in the log file and figure out what type it is
        std::string whatType;
        ins >> whatType;
        kind l_kind = classify(whatType);

        if ( LoadLog::string != l_kind && LoadLog::number != l_kind )
        {
          g_log.warning("ISIS log file contains unrecognised second column entries: " + m_filename);
          inLogFile.close();
        }

        try
        {
          Property* log = LogParser::createLogProperty(m_filename,stringToLower(logFileName));
          if (log)
          {
            localWorkspace->mutableRun().addLogData(log);
          }
        }
        catch(std::exception&)
        {
          inLogFile.close();
        }
      }
      inLogFile.close();
    }

    /**
     * Check if log file property name has been set. If not set, return the
     * workspace + log file name (e.g. HRP37129_ICPevent). Otherwise return first log name.
     * @param logName :: The vector containing log file names.
     * @return The name of the log file.
     */
    std::string LoadLog::extractLogName(std::vector<std::string> logName)
    {
      if(logName.empty())
      {
        return (Poco::Path(Poco::Path(m_filename).getFileName()).getBaseName());
      }
      else
      {
        return (logName.front());
      }
    }

    /**
     * Check if the file is SNS text; load it if it is, return false otherwise.
     * @return true if the file was a SNS style; false otherwise.
     */
    bool LoadLog::LoadSNSText()
    {

      // Get the SNS-specific parameter
      std::vector<std::string> names = getProperty("Names");
      std::vector<std::string> units = getProperty("Units");

      // Get the input workspace and retrieve run from workspace.
      // the log file(s) will be loaded into the run object of the workspace
      const MatrixWorkspace_sptr localWorkspace = getProperty("Workspace");

      // open log file
      std::ifstream inLogFile(m_filename.c_str());

      // Get the first line
      std::string aLine;
      if (!Mantid::Kernel::extractToEOL(inLogFile,aLine))
        return false;

      std::vector<double> cols;
      bool ret = SNSTextFormatColumns(aLine, cols);
      // Any error?
      if (!ret || cols.size() < 2)
        return false;

      size_t numCols = static_cast<size_t>(cols.size()-1);
      if (names.size() != numCols)
        throw std::invalid_argument("The Names parameter should have one fewer entry as the number of columns in a SNS-style text log file.");
      if ((!units.empty()) && (units.size() != numCols))
        throw std::invalid_argument("The Units parameter should have either 0 entries or one fewer entry as the number of columns in a SNS-style text log file.");

      // Ok, create all the logs
      std::vector<TimeSeriesProperty<double>*> props;
      for(size_t i=0; i < numCols; i++)
      {
        TimeSeriesProperty<double>* p = new TimeSeriesProperty<double>(names[i]);
        if (units.size() == numCols)
          p->setUnits(units[i]);
        props.push_back(p);
      }
      // Go back to start
      inLogFile.seekg(0);
      while(Mantid::Kernel::extractToEOL(inLogFile,aLine))
      {
        if (aLine.size() == 0)
          break;

        if (SNSTextFormatColumns(aLine, cols))
        {
          if (cols.size() == numCols+1)
          {
            DateAndTime time(cols[0], 0.0);
            for(size_t i=0; i<numCols; i++)
              props[i]->addValue(time, cols[i+1]);
          }
          else
            throw std::runtime_error("Inconsistent number of columns while reading SNS-style text file.");
        }
        else
          throw std::runtime_error("Error while reading columns in SNS-style text file.");
      }
      // Now add all the full logs to the workspace
      for(size_t i=0; i < numCols; i++)
      {
        std::string name = props[i]->name();
        if (localWorkspace->mutableRun().hasProperty(name))
        {
          localWorkspace->mutableRun().removeLogData(name);
          g_log.information() << "Log data named " << name << " already existed and was overwritten.\n";
        }
        localWorkspace->mutableRun().addLogData(props[i]);
      }

      return true;
    }

    /**
     * Return the name of the three column log file if we have one.
     * @returns A string containing the full log file path to a three column log file if one exists. An empty string otherwise.
     */
    std::string LoadLog::getThreeColumnName() const
    {
      std::string rawID;
      std::string::size_type dot = m_filename.rfind(".");
      if( dot != std::string::npos)
      {
        rawID = m_filename.substr(0, dot);
      }
      // append .log to get the .log file name
      std::string logfileName=rawID+".log";
      if (Poco::File(logfileName).exists())
      {
        //validate the file
        std::ifstream inLogFile(logfileName.c_str());
        if (!inLogFile)
        {
          throw Exception::FileError("Unable to open file:" ,logfileName );
        }

        //check if first 19 characters of a string is data-time string according to yyyy-mm-ddThh:mm:ss
        std::string aLine;
        int count=0;
        kind l_kind(LoadLog::empty);
        while(Mantid::Kernel::extractToEOL(inLogFile,aLine))
        {
          if ( !isDateTimeString(aLine) )
          { g_log.warning("File" + logfileName + " is not a standard ISIS log file. Expected to be a file starting with DateTime String format.");
          inLogFile.close();
          return "";
          }

          std::stringstream ins(aLine);
          std::string firstcolumn;
          ins >> firstcolumn;
          // read in what follows the date-time string in the log file and figure out
          // what type it is
          std::string secondcolumn;
          ins >> secondcolumn;
          l_kind = classify(secondcolumn);
          if ( LoadLog::string != l_kind )
          {
            g_log.warning("ISIS log file contains unrecognised second column entries: " + logfileName);
            inLogFile.close();
            return "";
          }

          std::string thirdcolumn;
          ins>>thirdcolumn;
          l_kind = classify(thirdcolumn);
          if ( LoadLog::string != l_kind && LoadLog::number!=l_kind)
          {
            g_log.warning("ISIS log file contains unrecognised third column entries: " + logfileName);
            inLogFile.close();
            return "";
          }
          ++count;
          if(count==2) ///reading first two lines from file for validation purpose.
            break;
        }
        return logfileName;
      }
      else return "";
    }

    /**
     * This method reads the.log file and creates timeseries property and sets that to the run object
     * @param logfile :: three column log(.log) file name.
     * @param run :: The run information object
     * @returns list of logfiles which exists as blockname in the .log file
     */
    std::set<std::string> LoadLog::createthreecolumnFileLogProperty(const std::string& logfile,API::Run& run)
    {
      std::set<std::string> blockFileNameList;
      std::string str;
      std::string propname;
      Mantid::Kernel::TimeSeriesProperty<double>* logd=0;
      Mantid::Kernel::TimeSeriesProperty<std::string>* logs=0;
      std::map<std::string,Kernel::TimeSeriesProperty<double>*> dMap;
      std::map<std::string,Kernel::TimeSeriesProperty<std::string>*> sMap;
      typedef std::pair<std::string,Kernel::TimeSeriesProperty<double>* > dpair;
      typedef std::pair<std::string,Kernel::TimeSeriesProperty<std::string>* > spair;

      std::string path = m_filename;
      std::string::size_type pos=m_filename.rfind(".");
      if( pos != std::string::npos )
      {
        path = path.substr(0, pos);
      }
      bool isNumeric(false);

      std::ifstream file(logfile.c_str());
      if (!file)
      {
        g_log.warning()<<"Cannot open log file "<<logfile<<"\n";
        return std::set<std::string>();
      }
      while(Mantid::Kernel::extractToEOL(file,str))
      {
        if (!Kernel::TimeSeriesProperty<double>::isTimeString(str) || (str[0]=='#'))
        {    //if the line doesn't start with a time read the next line
          continue;
        }
        std::stringstream line(str);
        std::string timecolumn;
        line>>timecolumn;
        std::string blockcolumn;
        line>>blockcolumn;
        std::string valuecolumn;
        line>>valuecolumn;

        /////column two in .log file is called block column
        /////if any .txt file with rawfilename_blockcolumn.txt exists
        ///// donot load that txt  files
        ///// blockFileNameList conatins the file names to be removed from potentiallogfiles list.
        propname=stringToLower(blockcolumn);
        //check if the data is numeric
        std::istringstream istr(valuecolumn);
        double dvalue;
        istr >> dvalue;
        isNumeric = !istr.fail();
        if (isNumeric)
        {
          std::map<std::string,Kernel::TimeSeriesProperty<double>*>::iterator ditr=dMap.find(propname);
          if(ditr!=dMap.end())
          {
            Kernel::TimeSeriesProperty<double>* p=ditr->second;
            if (p) p->addValue(timecolumn,dvalue);
          }
          else
          {

            logd = new Kernel::TimeSeriesProperty<double>(propname);
            logd->addValue(timecolumn,dvalue);
            dMap.insert(dpair(propname,logd));
            std::string blockcolumnFileName=path+"_"+blockcolumn+".txt";
            if(blockcolumnFileExists(blockcolumnFileName))
            {
              blockFileNameList.insert(blockcolumnFileName);
            }
          }
        }
        else
        {
          std::map<std::string,Kernel::TimeSeriesProperty<std::string>*>::iterator sitr=sMap.find(propname);
          if(sitr!=sMap.end())
          {
            Kernel::TimeSeriesProperty<std::string>* prop=sitr->second;
            if (prop) prop->addValue(timecolumn,valuecolumn);
          }
          else
          {
            logs = new Kernel::TimeSeriesProperty<std::string>(propname);
            logs->addValue(timecolumn,valuecolumn);
            sMap.insert(spair(propname,logs));
          }
          std::string blockcolumnFileName=path+"_"+blockcolumn+".txt";
          if(blockcolumnFileExists(blockcolumnFileName))
          {
            blockFileNameList.insert(blockcolumnFileName);
          }
        }
      }
      try
      {
        std::map<std::string,Kernel::TimeSeriesProperty<double>*>::const_iterator itr=dMap.begin();
        for(;itr!=dMap.end();++itr)
        {
          run.addLogData(itr->second);
        }
        std::map<std::string,Kernel::TimeSeriesProperty<std::string>*>::const_iterator sitr=sMap.begin();
        for(;sitr!=sMap.end();++sitr)
        {
          run.addLogData(sitr->second);
        }
      }
      catch(std::invalid_argument &e)
      {
        g_log.warning()<<e.what();
      }
      catch(Exception::ExistsError&e)
      {
        g_log.warning()<<e.what();
      }

      return blockFileNameList;

    }

    /**
     * This method looks for file with second column(block column) name exists in the raw file directory
     * @param fileName :: -name of the file
     * @return True if the file exists
     */
    bool LoadLog::blockcolumnFileExists(const std::string& fileName)
    {
      if (Poco::File(fileName).exists()) return true;
      else return false;
    }

    /**
     * Takes as input a string and try to determine what type it is.
     * @param s :: The input string
     * @param s ::  string to be classified
     * @return A enum kind which tells what type the string is
     */
    LoadLog::kind LoadLog::classify(const std::string& s) const
    {
      if( s.empty() )
      {
        return LoadLog::empty;
      }

      using std::string;
      const string lower("abcdefghijklmnopqrstuvwxyz");
      const string upper("ABCDEFGHIJKLMNOPQRSTUVWXYZ");
      const string letters = lower + upper + '_';

      if (letters.find_first_of(s) != string::npos)
      {
        return LoadLog::string;
      }
      else
      {
        return LoadLog::number;
      }
    }

    /**
     * Change each element of the string to lower case
     * @param strToConvert :: The input string
     * @returns The string but with all characters in lower case
     */
    std::string LoadLog::stringToLower(std::string strToConvert)
    {
      std::transform(strToConvert.begin(), strToConvert.end(), strToConvert.begin(), tolower);
      return strToConvert;
    }

    /**
     * Checks whether filename is a simple text file
     * @param filename :: The filename to inspect
     * @returns true if the filename has the .txt extension
     */
    bool LoadLog::isAscii(const std::string& filename)
    {
      FILE* file = fopen(filename.c_str(), "rb");
      char data[256];
      size_t n = fread(data, 1, sizeof(data), file);
      fclose(file);
      char *pend = &data[n];
      /*
       * Call it a binary file if we find a non-ascii character in the
       * first 256 bytes of the file.
       */
      for( char *p = data;  p < pend; ++p )
      {
        unsigned long ch = (unsigned long)*p;
        if( !(ch <= 0x7F) )
        {
          return false;
        }

      }
      return true;
    }

    /**
     * Check if first 19 characters of a string is date-time string according to yyyy-mm-ddThh:mm:ss
     * @param str :: The string to test
     * @returns true if the strings format matched the expected date format
     */
    bool LoadLog::isDateTimeString(const std::string& str) const
    {
      return DateAndTime::stringIsISO8601(str.substr(0,19));
    }


    /**
     * Read a line of a SNS-style text file.
     * @param str :: The string to test
     * @param out :: a vector that will be filled with the double values.
     * @return false if the format is NOT SNS style or a conversion failed.
     */
    bool LoadLog::SNSTextFormatColumns(const std::string& str, std::vector<double> & out) const
    {
      std::vector<std::string> strs;
      out.clear();
      boost::split(strs, str, boost::is_any_of("\t "));
      double val;
      // Every column must evaluate to a double
      for (size_t i=0; i<strs.size(); i++)
      {
        if (!Strings::convert<double>(strs[i],val))
          return false;
        else
          out.push_back(val);
      }
      // Nothing failed = it is that format.
      return true;
    }

  } // namespace DataHandling
} // namespace Mantid