Merge pull request #482 from mantidproject/feature/10231_FITS_loader_change_and_add_header_keys

LoadFITS: change and add header keys, and other improvements

Code/Mantid/Framework/DataHandling/inc/MantidDataHandling/LoadFITS.h

@@ -15,7 +15,7 @@ using namespace std;
 
 struct FITSInfo {
   vector<string> headerItems;
-  map<string, string> headerKeys;
+  std::map<string, string> headerKeys;
   int bitsPerPixel;
   int numberOfAxis;
   int offset;
@@ -24,54 +24,44 @@ struct FITSInfo {
   double tof;
   double timeBin;
   double scale;
-  int imageKey;
+  std::string imageKey;
   long int countsInImage;
   long int numberOfTriggers;
-  string extension;
-  string filePath;
+  std::string extension;
+  std::string filePath;
   bool isFloat;
 };
 
 namespace Mantid {
 namespace DataHandling {
 /**
-  LoadFITS : Load a number of FITS files into a histogram Workspace
+LoadFITS: Load one or more of FITS files into a Workspace2D. The FITS
+format, normally used for images, is described for example here:
+http://www.fileformat.info/format/fits/egff.htm
 
-  File format is described here: http://www.fileformat.info/format/fits/egff.htm
-  This loader doesn't support the full specification, caveats are:
-    Support for unsigned 8, 16, 32 bit values only
-    Support only for 2 data axis
-    No support for format extensions
+At the moment this algorithm only supports 2 data axis and the
+following data types: unsigned 8, 16, 32 bits per pixel.
 
-  Loader is designed to work with multiple files, loading into a single
-  workspace.
-  At points there are assumptions that all files in a batch use the same number
-  of bits per pixel,
-  and that the number of spectra in each file are the same.
+Copyright &copy; 2014,2015 ISIS Rutherford Appleton Laboratory, NScD
+Oak Ridge National Laboratory & European Spallation Source
 
-  @author John R Hill, RAL
-  @date 29/08/2014
+This file is part of Mantid.
 
-  Copyright &copy; 2014 ISIS Rutherford Appleton Laboratory, NScD Oak Ridge
-  National Laboratory & European Spallation Source
+Mantid is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+(at your option) any later version.
 
-  This file is part of Mantid.
+Mantid is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
 
-  Mantid is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 3 of the License, or
-  (at your option) any later version.
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-  Mantid is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-  File change history is stored at: <https://github.com/mantidproject/mantid>
-  Code Documentation is available at: <http://doxygen.mantidproject.org>
+File change history is stored at: <https://github.com/mantidproject/mantid>
+Code Documentation is available at: <http://doxygen.mantidproject.org>
 */
 
 class DLLExport LoadFITS : public API::IFileLoader<Kernel::FileDescriptor> {
@@ -84,7 +74,7 @@ public:
 
   /// Summary of algorithms purpose
   virtual const std::string summary() const {
-    return "Load data from FITS files.";
+    return "Load FITS files into workspaces of type Workspace2D.";
   }
 
   /// Algorithm's version for identification overriding a virtual method
@@ -105,51 +95,80 @@ private:
   void init();
   /// Execution code
   void exec();
-  /// Parses the header values for the FITS file
-  bool parseHeader(FITSInfo &headerInfo);
 
-  /// Creates a vector of all rotations from a file
-  std::vector<double> readRotations(std::string rotFilePath, size_t fileCount);
+  /// Loads files into workspace(s)
+  void doLoadFiles(const std::vector<std::string> &paths);
+
+  /// Loads the FITS header(s) into a struct
+  void doLoadHeaders(const std::vector<std::string> &paths,
+                     std::vector<FITSInfo> &headers);
+
+  /// Parses the header values for the FITS file
+  void parseHeader(FITSInfo &headerInfo);
 
   /// Initialises a workspace with IDF and fills it with data
   DataObjects::Workspace2D_sptr
-  addWorkspace(const FITSInfo &fileInfo, size_t &newFileNumber,
-               void *&bufferAny, API::MantidImage &imageY,
-               API::MantidImage &imageE, double rotation,
-               const DataObjects::Workspace2D_sptr parent);
+  makeWorkspace(const FITSInfo &fileInfo, size_t &newFileNumber,
+                std::vector<char> &buffer, API::MantidImage &imageY,
+                API::MantidImage &imageE,
+                const DataObjects::Workspace2D_sptr parent);
+
+  // Reads the data from a single FITS file into a workspace
+  void readDataToWorkspace2D(DataObjects::Workspace2D_sptr ws,
+                             const FITSInfo &fileInfo, API::MantidImage &imageY,
+                             API::MantidImage &imageE,
+                             std::vector<char> &buffer);
+
+  /// Once loaded, check against standard and limitations of this algorithm
+  void headerSanityCheck(const FITSInfo &hdr, const FITSInfo &hdrFirst);
+
+  void setupDefaultKeywordNames();
 
   /// Returns the trailing number from a string minus leading 0's (so 25 from
   /// workspace_00025)
-  size_t fetchNumber(std::string name);
+  size_t fetchNumber(const std::string &name);
 
   // Adds a number of leading 0's to another number up to the totalDigitCount.
-  std::string padZeros(size_t number, size_t totalDigitCount);
-
-  // Reads the data from a single FITS file into a workspace
-  void readFileToWorkspace(DataObjects::Workspace2D_sptr ws,
-                           const FITSInfo &fileInfo, API::MantidImage &imageY,
-                           API::MantidImage &imageE, void *&bufferAny);
+  std::string padZeros(const size_t number, const size_t totalDigitCount);
 
   // Maps the header keys to specified values
   void mapHeaderKeys();
 
   // Strings used to map header keys
-  string m_headerScaleKey;
-  string m_headerOffsetKey;
-  string m_headerBitDepthKey;
-  string m_headerRotationKey;
-  string m_headerImageKeyKey;
-  string m_mapFile;
+  std::string m_headerScaleKey;
+  std::string m_headerOffsetKey;
+  std::string m_headerBitDepthKey;
+  std::string m_headerRotationKey;
+  std::string m_headerImageKeyKey;
+  std::string m_headerNAxisNameKey;
   std::vector<std::string> m_headerAxisNameKeys;
+  std::string m_mapFile;
 
-  string m_baseName;
+  static const std::string m_defaultImgType;
+
+  // names of extension headers
+  std::string m_sampleRotation;
+  std::string m_imageType;
+
+  std::string m_baseName;
   size_t m_spectraCount;
   API::Progress *m_progress;
 
-  // Number of digits which will be appended to a workspace name, i.e. 4 =
-  // workspace_0001
-  static const size_t DIGIT_SIZE_APPEND = 4;
+  // Number of digits for the fixed width appendix number added to
+  // workspace names, i.e. 3=> workspace_001; 5 => workspace_00001
+  static const size_t DIGIT_SIZE_APPEND = 5;
+  /// size of a FITS header block (room for 36 entries, of 80
+  /// characters each), in bytes. A FITS header always comes in
+  /// multiples of this block.
   static const int BASE_HEADER_SIZE = 2880;
+
+  // names for several options that can be given in a "FITS" header
+  // setup file
+  static const std::string m_BIT_DEPTH_NAME;
+  static const std::string m_AXIS_NAMES_NAME;
+  static const std::string m_ROTATION_NAME;
+  static const std::string m_IMAGE_KEY_NAME;
+  static const std::string m_HEADER_MAP_NAME;
 };
 
 } // namespace DataHandling

Code/Mantid/Framework/DataHandling/test/LoadFITSTest.h

-#ifndef LOADFITSTEST_H_
-#define LOADFITSTEST_H_
+#ifndef MANTID_DATAHANDLING_LOADFITSTEST_H_
+#define MANTID_DATAHANDLING_LOADFITSTEST_H_
 
 #include <cxxtest/TestSuite.h>
+
+#include "MantidAPI/AlgorithmManager.h"
+#include "MantidAPI/AnalysisDataService.h"
 #include "MantidDataHandling/LoadFITS.h"
 
 using namespace Mantid::API;
 using namespace Mantid::DataHandling;
-using namespace Mantid::Kernel;
 
-class LoadFITSTest : public CxxTest::TestSuite
-{
-public: 
-  void testInit()
-  {
+class LoadFITSTest : public CxxTest::TestSuite {
+public:
+  // This pair of boilerplate methods prevent the suite being created statically
+  // This means the constructor isn't called when running other tests
+  static LoadFITSTest *createSuite() { return new LoadFITSTest(); }
+  static void destroySuite(LoadFITSTest *suite) { delete suite; }
+
+  void test_algorithm() {
+    std::string name = "LoadFITS";
+    int version = 1;
+    testAlg =
+        Mantid::API::AlgorithmManager::Instance().create(name /*, version*/);
+    TS_ASSERT(testAlg);
+    TS_ASSERT_EQUALS(testAlg->name(), name);
+    TS_ASSERT_EQUALS(testAlg->version(), version);
+  }
+
+  void test_castAlgorithm() {
+    // can create
+    boost::shared_ptr<LoadFITS> a;
+    TS_ASSERT(a = boost::make_shared<LoadFITS>());
+    // can cast to inherited interfaces and base classes
+
+    TS_ASSERT(dynamic_cast<Mantid::DataHandling::LoadFITS *>(a.get()));
+    TS_ASSERT(dynamic_cast<Mantid::API::Algorithm *>(a.get()));
+    TS_ASSERT(dynamic_cast<Mantid::Kernel::PropertyManagerOwner *>(a.get()));
+    TS_ASSERT(dynamic_cast<Mantid::API::IAlgorithm *>(a.get()));
+    TS_ASSERT(dynamic_cast<Mantid::Kernel::IPropertyManager *>(a.get()));
+  }
+
+  void test_initAlgorithm() {
+    LoadFITS lf;
+    TS_ASSERT_THROWS_NOTHING(lf.initialize());
+  }
+
+  void test_propertiesMissing() {
+    LoadFITS lf;
+    TS_ASSERT_THROWS_NOTHING(lf.initialize());
+    TS_ASSERT_THROWS_NOTHING(lf.setPropertyValue("Filename", smallFname1));
+    TS_ASSERT_THROWS(lf.execute(), std::runtime_error);
+    TS_ASSERT(!lf.isExecuted());
+
+    LoadFITS lf2;
+    TS_ASSERT_THROWS_NOTHING(lf2.initialize());
+    TS_ASSERT_THROWS_NOTHING(
+        lf2.setPropertyValue("OutputWorkspace", "out_ws_name"));
+    TS_ASSERT_THROWS(lf2.execute(), std::runtime_error);
+    TS_ASSERT(!lf2.isExecuted());
+  }
+
+  void test_wrongProp() {
+    LoadFITS lf;
+    TS_ASSERT_THROWS_NOTHING(lf.initialize());
+    TS_ASSERT_THROWS(lf.setPropertyValue("file", "anything"),
+                     std::runtime_error);
+    TS_ASSERT_THROWS(lf.setPropertyValue("output", "anything"),
+                     std::runtime_error);
+    TS_ASSERT_THROWS(lf.setPropertyValue("FITS", "anything"),
+                     std::runtime_error);
+
+    TS_ASSERT_THROWS(lf.setPropertyValue("ImageKey", "anything"),
+                     Mantid::Kernel::Exception::NotFoundError);
+    TS_ASSERT_THROWS(lf.setPropertyValue("BITPIX", "anything"),
+                     std::runtime_error);
+    TS_ASSERT_THROWS(lf.setPropertyValue("NAXIS", "anything"),
+                     std::runtime_error);
+    TS_ASSERT_THROWS(lf.setPropertyValue("NAXIS1", "anything"),
+                     std::runtime_error);
+  }
+
+  void test_init() {
     TS_ASSERT_THROWS_NOTHING(algToBeTested.initialize());
-    TS_ASSERT( algToBeTested.isInitialized() );
-    
-    if ( !algToBeTested.isInitialized() ) algToBeTested.initialize();
-  
-    outputSpace="LoadFITSTest";
-    algToBeTested.setPropertyValue("OutputWorkspace", outputSpace);     
-    
+    TS_ASSERT(algToBeTested.isInitialized());
+
+    if (!algToBeTested.isInitialized())
+      algToBeTested.initialize();
+
+    outputSpace = "LoadFITSTest";
+    algToBeTested.setPropertyValue("OutputWorkspace", outputSpace);
+
     // Should fail because mandatory parameter has not been set
-    TS_ASSERT_THROWS(algToBeTested.execute(),std::runtime_error);
-    
-    inputFile = "FITS_small_01.fits,FITS_small_02.fits";
-    algToBeTested.setPropertyValue("Filename", inputFile);  
-
-    // Set the ImageKey to be 0 (as it is missing from the test file and is required);
-    algToBeTested.setProperty<int>("ImageKey", 0);
-  }    
- 
-  void testPerformAssertions()
-  {
-    TS_ASSERT_THROWS_NOTHING(algToBeTested.execute());    
-    TS_ASSERT( algToBeTested.isExecuted() );
+    TS_ASSERT_THROWS(algToBeTested.execute(), std::runtime_error);
+
+    inputFile = smallFname1 + ", " + smallFname2;
+    algToBeTested.setPropertyValue("Filename", inputFile);
+
+    // Set the ImageKey to be 0 (this used to be required, but the key
+    // should not be there any longer);
+    TS_ASSERT_THROWS( algToBeTested.setProperty<int>("ImageKey", 0),
+                      Mantid::Kernel::Exception::NotFoundError);
+  }
+
+  void test_performAssertions() {
+    TS_ASSERT_THROWS_NOTHING(algToBeTested.execute());
+    TS_ASSERT(algToBeTested.isExecuted());
     //  get workspace generated
-    WorkspaceGroup_sptr output = AnalysisDataService::Instance().retrieveWS<WorkspaceGroup>(outputSpace);   
-    TS_ASSERT_EQUALS( output->getNumberOfEntries(), 2);  // Number of time bins should equal number of files
-    MatrixWorkspace_sptr ws1 = boost::dynamic_pointer_cast<MatrixWorkspace>(output->getItem(0));
-    MatrixWorkspace_sptr ws2 = boost::dynamic_pointer_cast<MatrixWorkspace>(output->getItem(1));
-    
-    TS_ASSERT_EQUALS(ws1->getNumberHistograms(), SPECTRA_COUNT);  // Number of spectra
+    WorkspaceGroup_sptr out;
+    TS_ASSERT(AnalysisDataService::Instance().doesExist(outputSpace));
+    TS_ASSERT_THROWS_NOTHING(
+        out = AnalysisDataService::Instance().retrieveWS<WorkspaceGroup>(
+            outputSpace));
+    TS_ASSERT_EQUALS(out->getNumberOfEntries(),
+                     2); // Number of time bins should equal number of files
+    MatrixWorkspace_sptr ws1;
+    TS_ASSERT_THROWS_NOTHING(
+        ws1 = boost::dynamic_pointer_cast<MatrixWorkspace>(out->getItem(0)));
+    MatrixWorkspace_sptr ws2;
+    TS_ASSERT_THROWS_NOTHING(
+        ws2 = boost::dynamic_pointer_cast<MatrixWorkspace>(out->getItem(1)));
+
+    // basic FITS headers
+    const auto run = ws1->run();
+    TS_ASSERT_EQUALS(run.getLogData("SIMPLE")->value(), hdrSIMPLE);
+    TS_ASSERT_EQUALS(run.getLogData("BITPIX")->value(), hdrBITPIX);
+    TS_ASSERT_EQUALS(run.getLogData("NAXIS")->value(), hdrNAXIS);
+    TS_ASSERT_EQUALS(run.getLogData("NAXIS1")->value(), hdrNAXIS1);
+    TS_ASSERT_EQUALS(run.getLogData("NAXIS2")->value(), hdrNAXIS2);
+
+    // Number of spectra
+    TS_ASSERT_EQUALS(ws1->getNumberHistograms(), SPECTRA_COUNT);
+    TS_ASSERT_EQUALS(ws2->getNumberHistograms(), SPECTRA_COUNT);
+
     // Sum the two bins from the last spectra - should be 70400
-    double sumY = ws1->readY(SPECTRA_COUNT-1)[0] + ws2->readY(SPECTRA_COUNT-1)[0];   
-    TS_ASSERT_EQUALS(sumY, 275);   
-    // Check the sum of the error values for the last spectra in each file - should be 375.183
-    double sumE = ws1->readE(SPECTRA_COUNT-1)[0] + ws2->readE(SPECTRA_COUNT-1)[0];
-    TS_ASSERT_LESS_THAN(abs(sumE-23.4489), 0.0001);  // Include a small tolerance check with the assert - not exactly 375.183
+    double sumY =
+        ws1->readY(SPECTRA_COUNT - 1)[0] + ws2->readY(SPECTRA_COUNT - 1)[0];
+    TS_ASSERT_EQUALS(sumY, 275);
+    // Check the sum of the error values for the last spectra in each file -
+    // should be 375.183
+    double sumE =
+        ws1->readE(SPECTRA_COUNT - 1)[0] + ws2->readE(SPECTRA_COUNT - 1)[0];
+    TS_ASSERT_LESS_THAN(abs(sumE - 23.4489), 0.0001); // Include a small
+                                                      // tolerance check with
+                                                      // the assert - not
+                                                      // exactly 375.183
   }
 
 private:
+  Mantid::API::IAlgorithm_sptr testAlg;
   LoadFITS algToBeTested;
+
   std::string inputFile;
   std::string outputSpace;
-  const static size_t SPECTRA_COUNT = 262144; // Based on the 512*512 test image
+  static const std::string smallFname1;
+  static const std::string smallFname2;
+
+  const static size_t xdim = 512;
+  const static size_t ydim = 512;
+  const static size_t SPECTRA_COUNT = xdim * ydim;
+  // FITS headers
+  const static std::string hdrSIMPLE;
+  const static std::string hdrBITPIX;
+  const static std::string hdrNAXIS;
+  const static std::string hdrNAXIS1;
+  const static std::string hdrNAXIS2;
 };
 
+const std::string LoadFITSTest::smallFname1 = "FITS_small_01.fits";
+const std::string LoadFITSTest::smallFname2 = "FITS_small_02.fits";
+
+const std::string LoadFITSTest::hdrSIMPLE = "T";
+const std::string LoadFITSTest::hdrBITPIX = "16";
+const std::string LoadFITSTest::hdrNAXIS = "2";
+const std::string LoadFITSTest::hdrNAXIS1 = "512";
+const std::string LoadFITSTest::hdrNAXIS2 = "512";
 
-#endif
\ No newline at end of file
+#endif // MANTID_DATAHANDLING_LOADFITSTEST_H_

Code/Mantid/docs/source/algorithms/LoadFITS-v1.rst

@@ -62,15 +62,15 @@ Usage
     ws = wsg.getItem(0)
 
     # A couple of standard FITS header entries
-    bpp_log = '_BITPIX'
+    bpp_log = 'BITPIX'
     try:
         log = ws.getRun().getLogData(bpp_log).value
         print "Bits per pixel: %s" % int(log)
     except RuntimeError:
         print "Could not find the keyword '%s' in this FITS file" % bpp_log
 
-    axis1_log = '_NAXIS1'
-    axis2_log = '_NAXIS2'
+    axis1_log = 'NAXIS1'
+    axis2_log = 'NAXIS2'
     try:
         log1 = ws.getRun().getLogData(axis1_log).value
         log2 = ws.getRun().getLogData(axis2_log).value