ScriptRepositoryImpl.cpp

// from mantid
#include "MantidScriptRepository/ScriptRepositoryImpl.h"
#include "MantidAPI/ScriptRepositoryFactory.h"
#include "MantidKernel/ConfigService.h"
#include "MantidKernel/Exception.h"
#include "MantidKernel/InternetHelper.h"
#include "MantidKernel/Logger.h"
#include "MantidKernel/NetworkProxy.h"
#include "MantidKernel/ProxyInfo.h"
#include <utility>
using Mantid::Kernel::DateAndTime;
using Mantid::Kernel::Logger;
using Mantid::Kernel::ConfigService;
using Mantid::Kernel::ConfigServiceImpl;
using Mantid::Kernel::ProxyInfo;
using Mantid::Kernel::NetworkProxy;

// from poco
#include <Poco/Path.h>
#include <Poco/File.h>
#include <Poco/TemporaryFile.h>
#include <Poco/URI.h>
#include <Poco/Exception.h>
#include <Poco/Net/NetException.h>
/*#include <Poco/Net/HTTPClientSession.h>
#include <Poco/Net/HTTPRequest.h>
#include <Poco/Net/HTTPResponse.h>
*/
#include <Poco/Net/HTMLForm.h>
#include "Poco/Net/FilePartSource.h"

// Visual Studio complains with the inclusion of Poco/FileStream
// disabling this warning.
#if defined(_WIN32) || defined(_WIN64)
#pragma warning(push)
#pragma warning(disable : 4250)
#include <Poco/FileStream.h>
#include <Poco/NullStream.h>
#include <Winhttp.h>
#pragma warning(pop)
#else
#include <Poco/FileStream.h>
#include <Poco/NullStream.h>
#include <stdlib.h>
#endif
#include <Poco/StreamCopier.h>
#include <Poco/DirectoryIterator.h>
#include <Poco/DateTimeParser.h>
#include <Poco/DateTimeFormatter.h>

// from boost
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/foreach.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/regex.hpp>

#include <json/json.h>

using boost::property_tree::ptree;  // Todo remove once using jsoncpp, plus boost ptree includes

namespace Mantid {
namespace API {
namespace {
/// static logger
Kernel::Logger g_log("ScriptRepositoryImpl");
}

const char *timeformat = "%Y-%b-%d %H:%M:%S";

const char *emptyURL =
    "The initialization failed because no URL was given that points "
    "to the central repository.\nThis entry should be defined at the "
    "properties file, "
    "at ScriptRepository";

/**
Write json object to file
*/
void writeJsonFile(const std::string& filename, Json::Value json, const std::string& error)
{
  Poco::FileOutputStream filestream(filename);
  if (!filestream.good()) {
    g_log.error() << error << std::endl;
  }
  Json::StyledWriter writer;
  filestream << writer.write(json);
  filestream.close();
}

/**
Read json object from file
*/
Json::Value readJsonFile(const std::string& filename, const std::string& error)
{
  Poco::FileInputStream filestream(filename);
  if (!filestream.good()) {
    g_log.error() << error << std::endl;
  }
  Json::Reader json_reader;
  Json::Value read;
  json_reader.parse(filestream, read);
  return read;
}

/**
Write string to file
*/
void writeStringFile(const std::string& filename, const std::string& stringToWrite, const std::string& error)
{
  Poco::FileStream filestream(filename);
  if (!filestream.good()) {
    g_log.error() << error << std::endl;
  }
  filestream << stringToWrite;
  filestream.close();
}

/**
Test if a file with this filename already exists
*/
bool fileExists(const std::string& filename)
{
  Poco::File test_file(filename);
  if (test_file.exists()) {
    return true;
  }
  return false;
}

DECLARE_SCRIPTREPOSITORY(ScriptRepositoryImpl)
/**
 The main information that ScriptrepositoryImpl needs to be able
 to operate are where the local repository is (or will be), and
 the url for the mantid web server.

 Usually these values are available at the Mantid properties files,
 so, it is possible to construct the ScriptrepositoryImpl without
 parameters.

 But, for flexibility reasons, (for example, testing with other
 repositories), a more general constructor is provided.

 In case a string is passed to the constructor different from the
 default one, it will have precedence, but it will not override what
 is defined by the Mantid properties files. These values will be valid
 only for that instance.

 Currently, two properties are defined: ScriptLocalRepository, and
 ScriptRepository.

 @code
 // get ScriptRepository and ScriptLocalRepository values from Mantid Config
 Service
 ScriptrepositoryImpl sharing();
 // apply given values
 ScriptrepositoryImpl sharing("/tmp/gitrep",
 "http://repository.mantidproject.com");
 @endcode
 */
ScriptRepositoryImpl::ScriptRepositoryImpl(const std::string &local_rep,
                                           const std::string &remote)
    : valid(false) {
  // get the local path and the remote path
  std::string loc, rem;
  ConfigServiceImpl &config = ConfigService::Instance();
  remote_upload = config.getString("UploaderWebServer");
  if (local_rep.empty() || remote.empty()) {
    loc = config.getString("ScriptLocalRepository");
    rem = config.getString("ScriptRepository");
  } else {
    local_repository = local_rep;
    remote_url = remote;
  }
  // the parameters given from the constructor have precedence
  if (local_rep.empty())
    local_repository = loc;
  else
    local_repository = local_rep;

  if (remote.empty())
    remote_url = rem;
  else
    remote_url = remote;

  // empty remote url is not allowed
  if (remote_url.empty()) {
    g_log.error() << emptyURL << std::endl;
    throw ScriptRepoException(emptyURL, "Constructor Failed: remote_url.empty");
  }

  if (remote_url[remote_url.size() - 1] != '/')
    remote_url.append("/");

  // if no folder is given, the repository is invalid.
  if (local_repository.empty())
    return;

  if (local_repository[local_repository.size() - 1] != '/')
    local_repository.append("/");

  g_log.debug() << "ScriptRepository creation pointing to " << local_repository
                << " and " << remote_url << "\n";

  // check if the repository is valid.

  // parsing the ignore pattern
  std::string ignore = ignorePatterns();
  boost::replace_all(ignore, "/", "\\/");
  boost::replace_all(ignore, ";", "|");
  boost::replace_all(ignore, ".", "\\.");
  boost::replace_all(ignore, "*", ".*");
  ignoreregex = std::string("(").append(ignore).append(")");

  // A valid repository must pass 3 tests:
  //  - An existing folder
  //  - This folder must have the .repository.json file
  //  - This folder must have the .local.json file
  // These tests will be done with Poco library

  Poco::Path local(local_repository);

  std::string aux_local_rep;
  if (local.isRelative()) {
    aux_local_rep = std::string(Poco::Path::current()).append(local_repository);
    local_repository = aux_local_rep;
  }

  try { // tests 1 and 2
    {
      Poco::File local_rep_dir(local);
      std::string repository_json =
          std::string(local_repository).append(".repository.json");
      Poco::File rep_json(repository_json);
      if (!local_rep_dir.exists() || !rep_json.exists()) {
        g_log.information() << "ScriptRepository was not installed at "
                            << local_repository << std::endl;
        return; // this is an invalid repository, because it was not created
                // (installed)
      }
    }
    // third test
    {
      std::string repository_json =
          std::string(local_repository).append(".local.json");
      Poco::File rep_json(repository_json);
      if (!rep_json.exists()) {
        g_log.error() << "Corrupted ScriptRepository at " << local_repository
                      << ". Please, remove this folder, and install "
                         "ScriptRepository again" << std::endl;
      }
    }
  } catch (Poco::FileNotFoundException & /*ex*/) {
    g_log.error()
        << "Testing the existence of repository.json and local.json failed"
        << std::endl;
    return;
  }

  // this is necessary because in windows, the absolute path is given
  // with \ slash.
  boost::replace_all(local_repository, "\\", "/");
  if (local_repository[local_repository.size() - 1] != '/')
    local_repository.append("/");

  repo.clear();
  valid = true;
}

ScriptRepositoryImpl::~ScriptRepositoryImpl() throw() {}

/**
 Check the connection with the server through the ::doDownloadFile method.
 @path server : The url that will be used to connect.
 */
void ScriptRepositoryImpl::connect(const std::string &server) {
  doDownloadFile(server);
}

/** Implements the ScriptRepository::install method.

 The instalation consists of:

 - creation of the folder for the ScriptRepository (if it does not exists).
 - download of the repository.json file (Make it hidden)
 - creation of the local.json file. (Make if hidden)

 The installation will also upate the ScriptLocalRepository setting, if
 necessary,
 to match the given path.

 If it success, it will change the status of the ScriptRepository as valid.

 @note Any directory may be given, from existing directories a new directory.
 If an existing directory is given, the installation will install the two
 necessary
 files to deal with this folder as a ScriptRepository.


 @param path : Path for a folder inside the local machine.


 */
void ScriptRepositoryImpl::install(const std::string &path) {
  using Poco::DirectoryIterator;
  if (remote_url.empty()) {
    std::stringstream ss;
    ss << "ScriptRepository is configured to download from a invalid URL "
          "(empty URL)."
       << "\nThis URL comes from the property file and it is called "
          "ScriptRepository.";
    throw ScriptRepoException(ss.str());
  }
  std::string folder = std::string(path);
  Poco::File repository_folder(folder);
  std::string rep_json_file = std::string(path).append("/.repository.json");
  std::string local_json_file = std::string(path).append("/.local.json");
  if (!repository_folder.exists()) {
    repository_folder.createDirectories();
  }

  // install the two files inside the given folder
  g_log.debug() << "ScriptRepository attempt to doDownload file " << path
                << std::endl;
  // download the repository json
  doDownloadFile(std::string(remote_url).append("repository.json"),
                 rep_json_file);
  g_log.debug() << "ScriptRepository downloaded repository information"
                << std::endl;
  // creation of the instance of local_json file
  if (!fileExists(local_json_file))
  {
    writeStringFile(local_json_file, "{\n}",
                    "ScriptRepository failed to create local repository");
    g_log.debug() << "ScriptRepository created the local repository information"
    << std::endl;
  }

#if defined(_WIN32) || defined(_WIN64)
  // set the .repository.json and .local.json hidden
  SetFileAttributes(local_json_file.c_str(), FILE_ATTRIBUTE_HIDDEN);
  SetFileAttributes(rep_json_file.c_str(), FILE_ATTRIBUTE_HIDDEN);
#endif

  // save the path to the config service
  //
  ConfigServiceImpl &config = ConfigService::Instance();
  std::string loc = config.getString("ScriptLocalRepository");
  if (loc != path) {
    config.setString("ScriptLocalRepository", path);
    config.saveConfig(config.getUserFilename());
  }

  local_repository = path;
  // this is necessary because in windows, the absolute path is given
  // with \ slash.
  boost::replace_all(local_repository, "\\", "/");
  if (local_repository[local_repository.size() - 1] != '/')
    local_repository.append("/");

  valid = true;
}

void ScriptRepositoryImpl::ensureValidRepository() {
  if (!isValid()) {
    std::stringstream ss;
    ss << "ScriptRepository is not installed correctly. The current path for "
          "ScriptRepository is " << local_repository
       << " but some important files that are required are corrupted or not "
          "present."
       << "\nPlease, re-install the ScriptRepository!\n"
       << "Hint: if you have a proper installation in other path, check the "
          "property ScriptLocalRepository "
       << "at the Mantid.user.properties and correct it if necessary.";
    throw ScriptRepoException(ss.str(), "CORRUPTED");
  }
}

/** Implements ScriptRepository::info

 For each entry inside the repository, there are some usefull information, that
 are stored in
 a ::ScriptInfo struct.

 Use this method, to get information about the description, last modified date,
 the auto update
 flag and the author.

 @param input_path: The path (relative or absolute) to the file/folder entry,

 @return ScriptInfo with the information of this file/folder.

 @note: This method requires that ::listFiles was executed at least once.

 */
ScriptInfo ScriptRepositoryImpl::info(const std::string &input_path) {
  ensureValidRepository();
  std::string path = convertPath(input_path);
  ScriptInfo info;
  try {
    RepositoryEntry &entry = repo.at(path);
    info.author = entry.author;
    info.pub_date = entry.pub_date;
    info.auto_update = entry.auto_update;
    info.directory = entry.directory;
  } catch (const std::out_of_range &ex) {
    std::stringstream ss;
    ss << "The file \"" << input_path
       << "\" was not found inside the repository!";
    throw ScriptRepoException(ss.str(), ex.what());
  }
  return info;
}

const std::string &
ScriptRepositoryImpl::description(const std::string &input_path) {
  ensureValidRepository();
  std::string path = convertPath(input_path);
  try {
    RepositoryEntry &entry = repo.at(path);
    return entry.description;
  } catch (const std::out_of_range &ex) {
    std::stringstream ss;
    ss << "The file \"" << input_path
       << "\" was not found inside the repository!";
    throw ScriptRepoException(ss.str(), ex.what());
  }
}

/**
 Implement the ScriptRepository::listFiles.

 It will fill up the ScriptRepositoryImpl::Repository variable in order to
 provide
 information about the status of the file as well.

 In order to list all the values from the repository, it uses three methods:

 - ::parseCentralRepository
 - ::parseDonwloadedEntries
 - ::parseLocalRepository

 After this, it will perform a reverse iteration on all the entries of the
 repository
 in order to evaluate the status (::findStatus) of every file, and will also get
 the
 status of every directory, by accumulating the influency of every directory.

 The listFiles will list:
 - all files in the central repository
 - all files in the local repository

 @return It returns a list of all the files and directories (the relative path
 inside the repository).
 */
std::vector<std::string> ScriptRepositoryImpl::listFiles() {
  ensureValidRepository();

  repo.clear();
  assert(repo.size() == 0);
  try {
    parseCentralRepository(repo);
    parseLocalRepository(repo);
    parseDownloadedEntries(repo);
    // it will not catch ScriptRepositoryExc, because, this means, that it was
    // already processed.
    // it will proceed in this situation.
  } catch (Poco::Exception &ex) {
    g_log.error() << "ScriptRepository failed to list all entries inside the "
                     "repository. Details: " << ex.className() << ":> "
                  << ex.displayText() << std::endl;
  } catch (std::exception &ex) {
    g_log.error() << "ScriptRepository failed to list all entries inside the "
                     "repository. Details: " << ex.what() << std::endl;
  }
  std::vector<std::string> out(repo.size());
  size_t i = repo.size();

  // evaluate the status for all entries
  // and also fill up the output vector (in reverse order)
  Mantid::API::SCRIPTSTATUS acc_status = Mantid::API::BOTH_UNCHANGED;
  std::string last_directory = "";
  for (Repository::reverse_iterator it = repo.rbegin(); it != repo.rend();
       ++it) {
    // for every entry, it takes the path and RepositoryEntry
    std::string entry_path = it->first;
    RepositoryEntry &entry = it->second;
    // g_log.debug() << "Evaluating the status of " << entry_path << std::endl;
    // fill up the output vector
    out[--i] = it->first;
    // g_log.debug() << "inserting file: " << it->first << std::endl;

    // for the directories, update the status of this directory
    if (entry.directory) {
      entry.status = acc_status;
      if (!entry.remote)
        entry.status = Mantid::API::LOCAL_ONLY;
      last_directory = entry_path;
    } else {
      // for the files, it evaluates the status of this file

      if (entry.local && !entry.remote) {
        // entry local only
        entry.status = LOCAL_ONLY;
      } else if (!entry.local && entry.remote) {
        // entry remote only
        entry.status = REMOTE_ONLY;
      } else {
        // there is no way of not being remote nor local!

        // entry is local and is remote
        // the following status are available:
        // BOTH_CHANGED, BOTH_UNCHANGED, REMOTE_CHANGED, LOCAL_CHANGED.
        enum CHANGES { UNCH = 0, REMO = 0X1, LOC = 0X2, BOTH = 0X3 };
        int st = UNCH;
        // the file is local_changed, if the date of the current file is
        // diferent
        // from the downloaded one.
        if (entry.current_date != entry.downloaded_date)
          st |= LOC;
        // the file is remote_changed if the date of the pub_date file is
        // diferent from the local downloaded pubdate.
        if (entry.pub_date > entry.downloaded_pubdate)
          st |= REMO;

        switch (st) {
        case UNCH:
          entry.status = BOTH_UNCHANGED;
          break;
        case REMO:
          entry.status = REMOTE_CHANGED;
          break;
        case LOC:
          entry.status = LOCAL_CHANGED;
          break;
        case BOTH:
        default:
          entry.status = BOTH_CHANGED;
          break;
        } // end switch

      } // end evaluating the file status

    } // end dealing with files

    // is this entry a child of the last directory?
    if (!last_directory.empty()) {
      if (entry_path.find(last_directory) == std::string::npos) {
        // no, this entry is not a child of the last directory
        // restart the status
        acc_status = Mantid::API::BOTH_UNCHANGED;
      }
    }

    // update the status of the parent directory:
    // the strategy here is to compare binary the current status with the
    // acc_state
    switch (acc_status | entry.status) {
    // pure matching, meaning that the matching is done with the same state
    // or with BOTH_UNCHANGED (neutral)
    case BOTH_UNCHANGED: // BOTH_UNCHANGED IS 0, so only 0|0 match this option
    case REMOTE_ONLY: // REMOTE_ONLY IS 0x01, so only 0|0x01 and 0x01|0x01 match
                      // this option
    case LOCAL_ONLY:
    case LOCAL_CHANGED:
    case REMOTE_CHANGED:
      acc_status = (SCRIPTSTATUS)(acc_status | entry.status);
      break;
    case LOCAL_ONLY | LOCAL_CHANGED:
      acc_status = LOCAL_CHANGED;
      break;
    case REMOTE_ONLY | REMOTE_CHANGED:
      acc_status = REMOTE_CHANGED;
      break;
    default:
      acc_status = BOTH_CHANGED;
      break;
    }
  }

  return out;
}

/**
 Implements the ScriptRepository::download.

 @note Require that ::listFiles been called at least once.

 The download is able to download files or directories. Internally,
 it will assign the job to the ::download_diretory or ::download_file.

 This method, just ensure that the entry is valid (wich means,
 it is inside the repository).

 @param input_path: The path for the file/folder to be downloaded.

 @note As a result of the download a new file, the local repository
 information .local.repository will be changed.

 */

void ScriptRepositoryImpl::download(const std::string &input_path) {
  ensureValidRepository();
  std::string file_path = convertPath(input_path);
  try {
    RepositoryEntry &entry = repo.at(file_path);
    if (entry.directory)
      download_directory(file_path);
    else
      download_file(file_path, entry);
  } catch (const std::out_of_range &ex) {
    // fixme: readable exception
    throw ScriptRepoException(ex.what());
  }
}

/**
 Go recursively to download all the children of an input directory.

 @param directory_path : the path for the directory.
 */
void ScriptRepositoryImpl::download_directory(
    const std::string &directory_path) {
  std::string directory_path_with_slash =
      std::string(directory_path).append("/");
  bool found = false;
  for (Repository::iterator it = repo.begin(); it != repo.end(); ++it) {
    // skip all entries that are not children of directory_path
    // the map will list the entries in alphabetical order, so,
    // when it first find the directory, it will list all the
    // childrens of this directory, and them,
    // it will list other things, so we can, break the loop
    if (it->first.find(directory_path) != 0) {
      if (found)
        break; // for the sake of performance
      else
        continue;
    }
    found = true;
    if (it->first != directory_path &&
        it->first.find(directory_path_with_slash) != 0) {
      // it is not a children of this entry, just similar. Example:
      // TofConverter/README
      // TofConverter.py
      // these two pass the first test, but will not pass this one.
      found = false;
      continue;
    }
    // now, we are dealing with the children of directory path
    if (!it->second.directory)
      download_file(it->first, it->second);
    else {
      // download the directory.

      // we will not download the directory, but create one with the
      // same name, and update the local json

      Poco::File dir(std::string(local_repository).append(it->first));
      dir.createDirectories();

      it->second.status = BOTH_UNCHANGED;
      it->second.downloaded_date = DateAndTime(
          Poco::DateTimeFormatter::format(dir.getLastModified(), timeformat));
      it->second.downloaded_pubdate = it->second.pub_date;
      updateLocalJson(it->first, it->second);

    }                                   // end downloading directory
                                        // update the status
    it->second.status = BOTH_UNCHANGED; // update this entry
  }                                     // end interaction with all entries
}

/**
 Download the real file from the remote_url.

 @todo describe better this method.
 */
void ScriptRepositoryImpl::download_file(const std::string &file_path,
                                         RepositoryEntry &entry) {
  SCRIPTSTATUS state = entry.status;
  // if we have the state, this means that the entry is available
  if (state == LOCAL_ONLY || state == LOCAL_CHANGED) {
    std::stringstream ss;
    ss << "The file " << file_path
       << " can not be download because it has only local changes."
       << " If you want, please, publish this file uploading it";
    throw ScriptRepoException(ss.str());
  }

  if (state == BOTH_UNCHANGED)
    // instead of throwing exception, silently assumes that the download was
    // done.
    return;

  // download the file
  std::string url_path = std::string(remote_url).append(file_path);
  Poco::TemporaryFile tmpFile;
  doDownloadFile(url_path, tmpFile.path());

  std::string local_path = std::string(local_repository).append(file_path);
  g_log.debug() << "ScriptRepository download url_path: " << url_path << " to "
                << local_path << std::endl;

  std::string dir_path;

  try {

    if (state == BOTH_CHANGED) {
      // make a back up of the local version
      Poco::File f(std::string(local_repository).append(file_path));
      std::string bck = std::string(f.path()).append("_bck");
      g_log.notice() << "The current file " << f.path()
                     << " has some local changes"
                     << " so, a back up copy will be created at " << bck
                     << std::endl;
      f.copyTo(bck);
    }

    // ensure that the path to the local_path exists
    size_t slash_pos = local_path.rfind('/');
    Poco::File file_out(local_path);
    if (slash_pos != std::string::npos) {
      dir_path =
          std::string(local_path.begin(), local_path.begin() + slash_pos);
      if (!dir_path.empty()) {
        Poco::File dir_parent(dir_path);
        if (!dir_parent.exists()) {
          dir_parent.createDirectories();
        }
      } // dir path is empty
    }

    if (!file_out.exists())
      file_out.createFile();

    tmpFile.copyTo(local_path);

  } catch (Poco::FileAccessDeniedException &) {
    std::stringstream ss;
    ss << "You cannot create file at " << local_path << ". Not downloading ...";
    throw ScriptRepoException(ss.str());
  }

  {
    Poco::File local(local_path);
    entry.downloaded_date = DateAndTime(
        Poco::DateTimeFormatter::format(local.getLastModified(), timeformat));
    entry.downloaded_pubdate = entry.pub_date;
    entry.status = BOTH_UNCHANGED;
  }

  // Update pythonscripts.directories if necessary
  // (TEST_DOWNLOAD_ADD_FOLDER_TO_PYTHON_SCRIPTS)
  if (!dir_path.empty()) {
    const char *python_sc_option = "pythonscripts.directories";
    ConfigServiceImpl &config = ConfigService::Instance();
    std::string python_dir = config.getString(python_sc_option);
    if (python_dir.find(dir_path) == std::string::npos) {
      // this means that the directory is not inside the
      // pythonscripts.directories
      // add to the repository
      python_dir.append(";").append(dir_path);
      config.setString(python_sc_option, python_dir);
      config.saveConfig(config.getUserFilename());

      // the previous code make the path available for the following
      // instances of Mantid, but, for the current one, it is necessary
      // do add to the python path...
    }
  }

  updateLocalJson(file_path, entry); /// FIXME: performance!
  g_log.debug() << "ScriptRepository download " << local_path << " success!"
                << std::endl;
}

/**
 @todo Describe
 */
SCRIPTSTATUS ScriptRepositoryImpl::fileStatus(const std::string &input_path) {
  /// @todo: implement the trigger method to know it we need to revised the
  ///        directories trees.
  ensureValidRepository();
  std::string file_path = convertPath(input_path);
  // g_log.debug() << "Attempt to ask for the status of "<< file_path <<
  // std::endl;
  try {
    RepositoryEntry &entry = repo.at(file_path);
    return entry.status;
  } catch (const std::out_of_range &ex) {
    std::stringstream ss;
    ss << "The file \"" << input_path
       << "\" was not found inside the repository!";
    throw ScriptRepoException(ss.str(), ex.what());
  }
  // this line will never be executed, just for avoid compiler warnings.
  return BOTH_UNCHANGED;
}

/**
 * Uploads one file to the ScriptRepository web server, pushing, indirectly, to
 *the
 * git repository. It will send in a POST method, the file and the following
 *fields:
 *  - author : Will identify the author of the change
 *  - email:  Will identify the email of the author
 *  - comment: Description of the nature of the file or of the update
 *
 * It will them upload to the URL pointed to UploaderWebServer. It will them
 *receive a json file
 * with some usefull information about the success or failure of the attempt to
 *upload.
 * In failure, it will be converted to an appropriated ScriptRepoException.
 */
void ScriptRepositoryImpl::upload(const std::string &file_path,
                                  const std::string &comment,
                                  const std::string &author,
                                  const std::string &email)

{
  using namespace Poco::Net;
  try {
    g_log.notice() << "ScriptRepository uploading " << file_path << " ..."
                   << std::endl;

    Kernel::InternetHelper inetHelper;

    HTMLForm form(HTMLForm::ENCODING_MULTIPART);

    // add the fields author, email and comment
    form.add("author", author);
    form.add("mail", email);
    form.add("comment", comment);

    // deal with the folder
    std::string relative_path = convertPath(file_path);
    std::string absolute_path = local_repository + relative_path;
    std::string folder = "./";
    size_t pos = relative_path.rfind('/');
    if (pos != std::string::npos)
      folder += std::string(relative_path.begin(), relative_path.begin() + pos);
    if (folder[folder.size() - 1] != '/')
      folder += "/";
    g_log.information() << "Uploading to folder: " << folder << std::endl;
    form.add("path", folder);

    // inserting the file
    FilePartSource *m_file = new FilePartSource(absolute_path);
    form.addPart("file", m_file);

    inetHelper.setBody(form);
    std::stringstream server_reply;

    int status;
    try {
      status = inetHelper.sendRequest(remote_upload, server_reply);
    } catch (Kernel::Exception::InternetError &ie) {
      status = ie.errorCode();
    }

    g_log.information() << "ScriptRepository upload status: " << status
                        << std::endl;
    std::stringstream answer;
    { // remove the status message from the end of the reply, in order not to
      // get exception from the read_json parser
      std::string server_reply_str;
      server_reply_str = server_reply.str();
      size_t pos = server_reply_str.rfind("}");
      if (pos != std::string::npos)
        answer << std::string(server_reply_str.begin(),
                              server_reply_str.begin() + pos + 1);
      else
        answer << server_reply_str;
    }
    g_log.debug() << "Form Output: " << answer.str() << std::endl;

    std::string info;
    std::string detail;
    std::string published_date;

    Json::Value pt;
    Json::Reader json_reader;
    if (!json_reader.parse(answer, pt)) {
      throw ScriptRepoException("Bad answer from the Server");
    }
    info = pt.get("message", "").asString();
    detail = pt.get("detail", "").asString();
    published_date = pt.get("pub_date", "").asString();
    std::string cmd = pt.get("shell", "").asString();
    if (!cmd.empty())
      detail.append("\nFrom Command: ").append(cmd);

    if (info == "success") {
      g_log.notice() << "ScriptRepository:" << file_path << " uploaded!"
                     << std::endl;

      // update the file
      RepositoryEntry &entry = repo.at(file_path);
      {
        Poco::File local(absolute_path);
        entry.downloaded_date = DateAndTime(Poco::DateTimeFormatter::format(
            local.getLastModified(), timeformat));
        // update the pub_date and downloaded_pubdate with the pub_date given by
        // the upload.
        // this ensures that the status will be correctly defined.
        if (!published_date.empty())
          entry.pub_date = DateAndTime(published_date);
        entry.downloaded_pubdate = entry.pub_date;
        entry.status = BOTH_UNCHANGED;
      }
      g_log.information() << "ScriptRepository update local json " << std::endl;
      updateLocalJson(file_path, entry); /// FIXME: performance!

      // add the entry to the repository.json. The
      // repository.json should change at the
      // remote repository, and we could just download the new one, but
      // we can not rely on the server updating it fast enough.
      // So add to the file locally to avoid race condition.
      RepositoryEntry &remote_entry = repo.at(file_path);
      if (!published_date.empty())
        remote_entry.pub_date = DateAndTime(published_date);
      remote_entry.status = BOTH_UNCHANGED;
      g_log.debug() << "ScriptRepository updating repository json "
                    << std::endl;
      updateRepositoryJson(file_path, remote_entry);

    } else
      throw ScriptRepoException(info, detail);

  } catch (Poco::Exception &ex) {
    throw ScriptRepoException(ex.displayText(), ex.className());
  }
}

/*
* Adds an entry to .repository.json
* This is necessary when uploading a file to keep .repository.json and
* .local.json in sync, and thus display correct file status in the GUI.
* Requesting an updated .repository.json from the server is not viable
* at such a time as it would create a race condition.
* @param path: relative path of uploaded file
* @param entry: the entry to add to the json file
*/
void ScriptRepositoryImpl::updateRepositoryJson(const std::string &path,
                                                const RepositoryEntry &entry) {

  Json::Value repository_json;
  std::string filename =
  std::string(local_repository).append(".repository.json");
  repository_json = readJsonFile(filename, "Error reading .repository.json file");

  if (!repository_json.isMember(path)) {
    // Create Json value for entry
    Json::Value entry_json;
    entry_json["author"] = entry.author;
    entry_json["description"] = entry.description;
    entry_json["directory"] = (entry.directory ? "true" : "false");
    entry_json["pub_date"] = entry.pub_date.toFormattedString();

    // Add Json value for entry to repository Json value
    repository_json[path] = entry_json;
  }

  g_log.debug() << "Update LOCAL JSON FILE" << std::endl;
#if defined(_WIN32) || defined(_WIN64)
  // set the .repository.json and .local.json not hidden to be able to edit it
  SetFileAttributes(filename.c_str(), FILE_ATTRIBUTE_NORMAL);
#endif
  writeJsonFile(filename, repository_json, "Error writing .repository.json file");
#if defined(_WIN32) || defined(_WIN64)
  // set the .repository.json and .local.json hidden
  SetFileAttributes(filename.c_str(), FILE_ATTRIBUTE_HIDDEN);
#endif
}

/**
 * Delete one file from the local and the central ScriptRepository
 * It will send in a POST method, with the file path to find the path :
 *  - author : Will identify the author of the change
 *  - email:  Will identify the email of the author
 *  - comment: Description of the nature of the file or of the update
 *
 * It will them send the request to the URL pointed to UploaderWebServer,
 *changing the word
 * publish to remove. For example:
 *
 * http://upload.mantidproject.org/scriptrepository/payload/remove
 *
 * The server will them create a git commit deleting the file. And will reply
 *with a json string
 * with some usefull information about the success or failure of the attempt to
 *delete.
 * In failure, it will be converted to an appropriated ScriptRepoException.
 *
 * Requirements: in order to be allowed to delete files from the central
 *repository,
 * it is required that the state of the file must be BOTH_UNCHANGED or
 *LOCAL_CHANGED.
 *
 * @param file_path: The path (relative to the repository) or absolute to