Commit 5bed70c3 authored by Russell Taylor's avatar Russell Taylor
Browse files

Merge remote-tracking branch 'origin/feature/6175_script_repo'

parents 5394ad4a 31e969a0
......@@ -24,8 +24,8 @@ namespace API{
std::string description;
/// Time of the last update of this file (remotelly)
Kernel::DateAndTime pub_date;
/// Integer that distinguishes the version of the file entry
int versionId;
/// Marked for auto update
bool auto_update;
};
/** Represent the possible states for a given file:
......@@ -320,11 +320,13 @@ They will work as was expected for folders @ref folders-sec.
@code
ScriptSharing spt;
ScriptInfo info = spt.fileInfo("README.md");
ScriptInfo info = spt.info("README.md");
// info.description : returns the file description.
@endcode
*/
virtual ScriptInfo fileInfo(const std::string path) = 0;
virtual ScriptInfo info(const std::string path) = 0;
/// @deprecated Previous version, to be removed.
ScriptInfo fileInfo(const std::string path){return info(path);}
/**
......@@ -410,11 +412,11 @@ They will work as was expected for folders @ref folders-sec.
locally.
It is allowed to create hidden files that would be necessary for the operation of this class.
At the end, a new folder is created, with the given local_path given.
At the end, a new folder is created (if it does not exists already), with the given local_path given.
@param local_path: path where the folder (having the same name given) will be created.
@exception ScriptRepoException: If the local_path may not be created (because is an existing folder not empty).
@exception ScriptRepoException: If the local_path may not be created (Permission issues).
*/
virtual void install(std::string local_path) = 0;
......@@ -445,8 +447,8 @@ They will work as was expected for folders @ref folders-sec.
may eventually, notify that the local repository may not be created.
*/
virtual void check4Update(void) = 0;
/// @deprecated
virtual void update(void) = 0;
/// @deprecated Used in the previous version. Use check4Update instead.
void update(void) {check4Update();};
......@@ -493,6 +495,44 @@ They will work as was expected for folders @ref folders-sec.
const std::string author,
const std::string description = std::string()) = 0;
/** Define the file patterns that will not be listed in listFiles.
This is important to force the ScriptRepository to not list hidden files,
automatic generated files and so on. This helps to present to the user a
clean presentation of the Repository
For example, there are the pyc files that are automatically generated, and
should be discarded. We could set also to ignore files that end with ~,
temporary files in linux. The patterns will be evaluated as a csv regex patterns.
To discard all pyc files, set: "*pyc".
To discard all pyc files and hidden files and folders: "*pyc;\b\.*"
@param patterns : csv regex patterns to be ignored when listing files.
This settings must be preserved, and be available after trough the configure system.
*/
virtual void setIgnorePatterns(std::string patterns) = 0;
/** Return the ignore patters that was defined through ScriptRepository::setIgnorePatterns*/
virtual std::string ignorePatterns(void) = 0;
/** Define the AutoUpdate option, which define if a file will be updated as soon as
new versions are available at the central repository.
This information will be kept in a property system, in order to be available afterwards.
@param path : file or folder inside the local repository
@param option: flag to set for auto-update, or not. If true, new versions of the path will replace the local file as soon as they are available at the central repository.
@exception ScriptRepoException : Invalid entry.
*/
virtual void setAutoUpdate(std::string path, bool option = true) = 0;
protected:
/// @deprecated get all the files from a repository
std::vector<struct file_entry> repository_list;
......
......@@ -129,7 +129,7 @@ endif ()
add_subdirectory (MDAlgorithms)
add_subdirectory (MDEvents)
add_subdirectory (Doxygen)
#add_subdirectory (ScriptRepository)
add_subdirectory (ScriptRepository)
###########################################################################
# Add a custom target to build all of the Framework
......
......@@ -198,9 +198,11 @@ algorithms.categories.hidden=Workflow\\Inelastic\\UsesPropertyManager;Workflow\\
# ScriptRepository Properties:
# Url for the WebServer that support the upload of the files that the users want to share
UploaderWebServer = http://127.0.0.1:8000/upload/
# Url for the WebServer that support the upload of the files that the users want to share (not in usage)
UploaderWebServer =
# Local system path for the script repository.
ScriptLocalRepository =
# Url for the remote script repository. Write a list of urls separating them with ";", the system will try them in the order passed.
ScriptRepository = git://github.com/mantidproject/scripts.git;https://github.com/mantidproject/scripts.git
# Url for the remote script repository.
ScriptRepository = http://download.mantidproject.org/master_builds/
# Pattern given to ScriptRepository that is used to hide entries from repository to the users. It is a csv string separated with ';'
ScriptRepositoryIgnore = *pyc;
......@@ -41,15 +41,9 @@ namespace
return registered;
}
PyObject * getName(ScriptRepository & self){
UNUSED_ARG(self);
PyObject * value = PyString_FromString("GitMyScriptRepository");
return value;
}
tuple getInfo(ScriptRepository & self, const std::string path){
ScriptInfo info = self.fileInfo(path);
return boost::python::make_tuple<std::string>(info.author, info.description);
ScriptInfo info = self.info(path);
return boost::python::make_tuple<std::string>(info.author, info.description, info.pub_date.toSimpleString() );
}
PyObject * getStatus(ScriptRepository & self, const std::string path){
......@@ -96,27 +90,97 @@ void export_ScriptRepository()
const char * repo_desc =
"Manage the interaction between the users and the Script folder (mantid subproject). \n\
\n\
Inside the mantid repository (https://github.com/mantidproject) there is also a subproject called \n\
scripts (https://github.com/mantidproject/scripts), created to allow users to share their scripts, \n\
as well as to allow Mantid Team to distribute to the Mantid community scripts for analysis and \n\
also to enhance the quality of the scripts used for the sake of data analysis. \n\
Inside the mantid repository (https://github.com/mantidproject) there is also a subproject \n\
called scripts (https://github.com/mantidproject/scripts), created to allow users to share \n\
their scripts, as well as to allow Mantid Team to distribute to the Mantid community \n\
scripts for analysis and also to enhance the quality of the scripts used for the sake of \n\
data analysis. \n\
\n\
The ScriptSharing class aims to provide a simple way to interact with that repository in order to \n\
promote its usage. In order to enhance the usage, it is necessary:\n\
The ScriptRepository aims to provide a simple way to interact with that repository in \n\
order to promote its usage. In order to enhance the usage, it is necessary:\n\
\n\
- List all scripts available at the repository\n\
- Download selected scripts. \n\
- Check for updates\n\
- Allow to publish users scripts/folders. \n";
- Allow to publish users scripts/folders. \n\
\n\
Basically, you will need to install the repository through ::install(local_path). \n\
After, you will be able to ::listFiles inside your repository to check the files that are \n\
available, eventually, you may want to check the information of a single entry \n\
::fileInfo(path). You may also be interested to know if there is a new version available, \n\
or if the file has been modified since last downloaded (::fileStatus(path)). You may want \n\
to download scripts through ::download(path), or check for updates through ::update(). \n\
\n\
'''NOTE:''' Upload is not implemented yet.\n";
const char * list_files_desc =
"Return an array with all the entries inside the repository. \n\
\n\
Folder os files, locally or remotely, all will be listed together through the listFiles. \n\
The listFiles has another function, which is related to update the internal cache about \n\
the status and information of the files. So, local changes or remote changes will only be \n\
available to fileStatus of fileInfo after listFiles.\n\
\n\
:return : list of entries inside the repository.\n";
const char * file_info_desc =
"Return general information from the entries inside ScriptRepository. \n\
\n\
The author, description and publication date are available through this method. \n\
\n\
:param path: Path to the entry.\n\
:return : Tuple with (author, description, last publication date)\n";
const char * file_status_desc =
"Return the status of a given entry.\n\
\n\
The following status are applied to the entries:\n\
- REMOTE_ONLY: The file is only at the central repository\n\
- LOCAL_ONLY: The file is only in your file system\n\
- BOTH_UNCHAGED: The file is in your file system and remotely, and are iqual.\n\
- REMOTE_CHANGED: A new version is available for this file.\n\
- LOCAL_CHANGED: You have edited the file\n\
- BOTH_CHANGED: There is a new version and you have changed as well\n\
\n\
'''NOTE:''' ScriptRepository recognizes changes locally and remotely only through \n\
listFiles method.\n\
:param path: The path for the entry.\n\
:return : String with the status of the entry.\n";
const char * download_desc =
"Download from repository into your local file system.\n\
\n\
You may give a file or folder. If the later is given, ScriptRepository will \n\
download all the files inside that folder from the remote repository to you.\n\
:param path: Path for the entry do download";
const char * update_desc =
"Check for updates at the remote repository.\n\
\n\
New versions of the files may be available, and the update method will check the \n\
remote repository to see if there is anything new. It will not download new versions \n\
of the available files unless you ask to do so. You should do this often to check if \n\
there is a new script to solve your problem ;)";
const char * install_desc =
"Install the ScriptRepository in your local file system\n\
\n\
The installation of the ScriptRepository is very simple. You must only provide a path, \n\
existing or new folder, where the ScriptRepository will put the database it requires to \n\
run itself. The installation requires network connection, to connect to the central \n\
repository but usually takes very few moments to be installed. After installing, all the \n\
others methods will be available.\n\
: param path: An existing or path to a new folder to be created, where the \n\
ScriptRepository will install itself.";
///@todo beter description
class_<ScriptRepository,boost::noncopyable>("ScriptRepository", repo_desc, no_init)
.def("name", &getName, "Return the name of the ScriptRepository")
.def("listFiles",&getListFiles,"Return the list of the files inside the Repository")
.def("fileInfo",&getInfo,"Return description of the file inside the repository")
.def("download",&ScriptRepository::download,"Download file or folder ")
.def("fileStatus",&getStatus,"Return the status")
.def("upload",&ScriptRepository::upload,"Publish your script")
.def("update",&ScriptRepository::update,"Check if there is update remotely");
.def("install",&ScriptRepository::install, install_desc)
.def("listFiles",&getListFiles, list_files_desc)
.def("fileInfo",&getInfo,file_info_desc)
.def("fileStatus",&getStatus,file_status_desc)
.def("download",&ScriptRepository::download,download_desc)
//.def("upload",&ScriptRepository::upload, "")
.def("update",&ScriptRepository::check4Update,update_desc);
}
####
## SET THE LIB GIT 2 FILES
####
# Find required dependencies
#maybe INCLUDE_DIRECTORIES (libgit2/src as well
INCLUDE_DIRECTORIES(SYSTEM libgit2/include libgit2/deps/http-parser libgit2/src/ libgit2/deps/regex)
IF (NOT WIN32)
FIND_PACKAGE(ZLIB)
ENDIF()
IF (WIN32 AND NOT MINGW)
ADD_DEFINITIONS(-DGIT_WINHTTP)
ELSE ()
FIND_PACKAGE(OpenSSL)
SET (SRC_HTTP libgit2/deps/http-parser/http_parser.c)
SET (INC_HTTP libgit2/deps/http-parser/http_parser.h)
ENDIF()
IF (ZLIB_FOUND)
INCLUDE_DIRECTORIES (SYSTEM ${ZLIB_INCLUDE_DIRS})
LINK_LIBRARIES(${ZLIB_LIBRARIES})
SET (SRC_ZLIB "")
ELSE()
INCLUDE_DIRECTORIES(SYSTEM libgit2/deps/zlib)
ADD_DEFINITIONS(-DNO_VIZ -DSTDC -DNO_GZIP)
SET (SRC_ZLIB
libgit2/deps/zlib/adler32.c
libgit2/deps/zlib/crc32.c
libgit2/deps/zlib/deflate.c
libgit2/deps/zlib/inffast.c
libgit2/deps/zlib/inflate.c
libgit2/deps/zlib/inftrees.c
libgit2/deps/zlib/trees.c
libgit2/deps/zlib/zutil.c
)
ENDIF()
# Specify sha1 implementation
IF (SHA1_TYPE STREQUAL "ppc")
ADD_DEFINITIONS(-DPPC_SHA1)
FILE(GLOB SRC_SHA1 libgit2/src/hash/hash_ppc.c libgit2/src/hash/hash_ppc_core.S)
ELSEIF (WIN32 AND NOT MINGW AND NOT SHA1_TYPE STREQUAL "builtin")
ADD_DEFINITIONS(-DWIN32_SHA1)
FILE(GLOB SRC_SHA1 libgit2/src/hash/hash_win32.c)
ELSEIF (OPENSSL_FOUND AND NOT SHA1_TYPE STREQUAL "builtin")
ADD_DEFINITIONS(-DOPENSSL_SHA1)
ELSE()
FILE(GLOB SRC_SHA1 libgit2/src/hash/hash_generic.c)
ENDIF()
# On Windows use specific platform sources
IF (WIN32 AND NOT CYGWIN)
##ADD_DEFINITIONS(-DWIN32 -D_WIN32_WINNT=0x0501)
FILE(GLOB GIT2_SRC libgit2/src/*.c libgit2/src/transports/*.c libgit2/src/xdiff/*.c libgit2/src/win32/*.c libgit2/src/compat/*.c)
ELSEIF (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
FILE(GLOB GIT2_SRC libgit2/src/*.c libgit2/src/transports/*.c libgit2/src/xdiff/*.c libgit2/src/unix/*.c libgit2/src/compat/*.c)
ELSEIF (AMIGA)
ADD_DEFINITIONS(-DNO_ADDRINFO -DNO_READDIR_R)
FILE(GLOB GIT2_SRC libgit2/src/*.c libgit2/src/transports/*.c libgit2/src/xdiff/*.c libgit2/src/amiga/*.c libgit2/src/compat/*.c)
ELSE()
FILE(GLOB GIT2_SRC libgit2/src/*.c libgit2/src/transports/*.c libgit2/src/xdiff/*.c libgit2/src/unix/*.c)
ENDIF ()
IF (MSVC)
# Default to stdcall, as that's what the CLR expects and how the Windows API is built
OPTION (STDCALL "Buildl libgit2 with the __stdcall convention" ON)
ENDIF()
SET (SRC_REGEX libgit2/deps/regex/regex.c)
SET (INC_REGEX libgit2/deps/regex/regex.h)
# Libgit2 source
set ( LIBGIT2_SRC
${SRC_ZLIB} ${SRC_REGEX} ${SRC_HTTP}
${SRC_SHA1} ${GIT2_SRC}
)
# Mantid source
set ( SRC_FILES
src/GitScriptRepository.cpp
src/ScriptRepositoryImpl.cpp
)
set ( INC_FILES inc/MantidScriptRepository/GitScriptRepository.h
${INC_ZLIB}
${INC_HTTP}
set ( INC_FILES inc/MantidScriptRepository/ScriptRepositoryImpl.h
)
#
#set ( TEST_FILES
# GitScriptRepositoryTest.h
set ( TEST_FILES
ScriptRepositoryTestImpl.h)
# ScriptRepositoryFactoryTest.h)
#
# uncomment these lines if you want to be able to test the
# features that depend on internet connection
#
......@@ -99,23 +19,8 @@ set ( INC_FILES inc/MantidScriptRepository/GitScriptRepository.h
# GitNewFeatureTest.h
#)
## Ignore all warnings from libgit2 source
if (MSVC)
set_source_files_properties ( ${LIBGIT2_SRC}
PROPERTIES COMPILE_FLAGS "/w" )
else()
set_source_files_properties ( ${LIBGIT2_SRC}
PROPERTIES COMPILE_FLAGS "-w" )
endif()
# add support for ssl for the transport https
IF (OPENSSL_FOUND)
ADD_DEFINITIONS(-DGIT_SSL)
INCLUDE_DIRECTORIES(${OPENSSL_INCLUDE_DIR})
SET(SSL_LIBRARIES ${OPENSSL_LIBRARIES})
ENDIF()
# Add the target for this directory
add_library ( ScriptRepository ${SRC_FILES} ${LIBGIT2_SRC} ${INC_FILES} )
add_library ( ScriptRepository ${SRC_FILES} ${INC_FILES} )
# Add the unit tests directory
add_subdirectory ( test )
# Set the name of the generated library
......@@ -125,12 +30,7 @@ set_target_properties ( ScriptRepository PROPERTIES OUTPUT_NAME MantidScriptRepo
set_property (TARGET ScriptRepository PROPERTY FOLDER "MantidFramework")
include_directories(inc)
if ( APPLE )
set ( LIBS ${MANTIDLIBS} ${SSL_LIBRARIES} ${OPENSSL_CRYPTO_LIBRARY} )
else ()
set ( LIBS ${MANTIDLIBS} )
endif()
set ( LIBS ${MANTIDLIBS} )
target_link_libraries(ScriptRepository ${LIBS})
install (TARGETS ScriptRepository ${SYSTEM_PACKAGE_TARGET} DESTINATION ${PLUGINS_DIR} )
#ifndef _MANTIDSCRIPTREPOSITORY_SCRIPTREPOSITORYIMPL_H_
#define _MANTIDSCRIPTREPOSITORY_SCRIPTREPOSITORYIMPL_H_
#include "MantidAPI/ScriptRepository.h"
#include "MantidKernel/DateAndTime.h"
using Mantid::API::ScriptRepository;
using Mantid::Kernel::DateAndTime;
#include <map>
#ifdef _WIN32
#if (IN_MANTID_SCRIPTREPO)
#define SCRIPT_DLL_EXPORT DLLExport
#else
#define SCRIPT_DLL_EXPORT DLLImport
#endif
#else
#define SCRIPT_DLL_EXPORT
#endif
namespace Mantid
{
namespace Kernel{
class Logger;
}
namespace API{
/** Implementation of Mantid::API::ScriptRepository
This implementation relies on the definition of the Script Repository WebServer.
@todo Describe better the implementation
*/
class SCRIPT_DLL_EXPORT ScriptRepositoryImpl: public ScriptRepository
{
/**
Keep the usefull information for each entry of the repository.
*/
class RepositoryEntry{
public:
/// Indicate if the file is presented at the central repository.
bool remote;
/// Indicate if the file is presented locally
bool local;
/// This entry is a directory?
bool directory;
/// For the local files, get the DateAndTime reported by the operative system
/// or defaultTime if not available.
DateAndTime current_date;
/// For the files that were downloaded, get the DateAndTime reported when they
/// were created.
DateAndTime downloaded_date;
/// For the remote files, get the DateAndTime of the last revision.
DateAndTime pub_date;
/// Description of the files.
std::string description;
/// The version downloaded of this file
DateAndTime downloaded_pubdate;
/// Indicate if this file should be updated automatically.
bool auto_update;
/// Identify the author of this file.
std::string author;
///status of the current entry
SCRIPTSTATUS status;
/// provide a constructor, to set the default values.
RepositoryEntry():remote(false), local(false),
directory(false), current_date(DateAndTime::defaultTime()),
downloaded_date(DateAndTime::defaultTime()),
pub_date(DateAndTime::defaultTime()),
description(""),
downloaded_pubdate(DateAndTime::defaultTime()),
auto_update(false),author(""){};
};
typedef std::map<std::string, RepositoryEntry > Repository;
Repository repo;
public:
ScriptRepositoryImpl(const std::string local_repository = std::string(),
const std::string remote_url = std::string()) ;
virtual ~ScriptRepositoryImpl() throw();
void connect(std::string server);
void install(std::string local_path);
ScriptInfo info(const std::string path);
std::vector<std::string> listFiles();
void download(const std::string file_path);
SCRIPTSTATUS fileStatus(const std::string file_path);
void upload(const std::string file_path, const std::string comment,
const std::string author,
const std::string description = std::string());
/* Return true if there is a local repository installed*/
bool isValid(void);
void check4Update(void);
void setIgnorePatterns(std::string patterns);
std::string ignorePatterns(void);
void setAutoUpdate(std::string path, bool option = true);
/// @deprecated Should avoid this, it is not in the design file.
std::string localRepository() const {return local_repository; }
virtual void doDownloadFile(const std::string url_file, const std::string local_file_path = "");
protected:
void parseCentralRepository(Repository & repo);
void parseLocalRepository(Repository & repo);
void parseDownloadedEntries(Repository & repo);
void ensureValidRepository();
bool isEntryValid(std::string path);
/// Path of the local repository.
std::string local_repository;
/// URL for the remote repository, usually:
std::string remote_url;
private:
void recursiveParsingDirectories(const std::string & path, Repository & repo);
std::string convertPath(const std::string path);
/* /// Used to throw when a local repository is mal-formed.
ScriptRepoException invalidRepository();
ScriptRepoException systemException(const std::string info = std::string(),
const std::string file = std::string(),
int line = -1);
enum FILEINFOSUPPORT{READMEFILE, PYTHONFILE};
std::string processInfo(const std::string path, FILEINFOSUPPORT filetype); */
private:
static std::string printStatus(SCRIPTSTATUS st);
void download_directory(const std::string);
void download_file(const std::string, RepositoryEntry & );
void updateLocalJson(const std::string & , const RepositoryEntry & );
/// reference to the logger class
Mantid::Kernel::Logger& g_log;
/// flag that indicate a valid repository
bool valid;
std::string ignoreregex;
};
}; // namespace API
}; // namespace Mantid
#endif // _MANTIDSCRIPTREPOSITORY_SCRIPTREPOSITORYIMPL_H_
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment