Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
#!/usr/bin/env python
""" A few tools for validating the wiki at mantidproject.org """
import warnings
import argparse
import os
import mwclient
import ConfigParser
import string
import time
import datetime
import subprocess
import commands
import sys
import codecs
import fnmatch
mantid_initialized = False
header_files = []
header_files_bare = []
python_files = []
python_files_bare = []
#======================================================================
def intialize_files():
""" Get path to every header file """
global file_matches
parent_dir = os.path.abspath(os.path.join(os.path.split(__file__)[0], os.path.pardir))
file_matches = []
for root, dirnames, filenames in os.walk(parent_dir):
for filename in fnmatch.filter(filenames, '*.h'):
fullfile = os.path.join(root, filename)
header_files.append(fullfile)
header_files_bare.append( os.path.split(fullfile)[1] )
for filename in fnmatch.filter(filenames, '*.py'):
fullfile = os.path.join(root, filename)
python_files.append(fullfile)
python_files_bare.append( os.path.split(fullfile)[1] )
#======================================================================
def find_algo_file(algo):
"""Find the files for a given algorithm"""
source = ''
header = algo + ".h"
pyfile = algo + ".py"
if header in header_files_bare:
n = header_files_bare.index(header, )
source = header_files[n]
elif pyfile in python_files_bare:
n = python_files_bare.index(pyfile, )
source = python_files[n]
return source
#======================================================================
def initialize_wiki(args):
global site
# Init site object
print "Connecting to site mantidproject.org"
site = mwclient.Site('www.mantidproject.org', path='/')
# Login, if specified
if hasattr(args, 'username'):
print "Logging in as %s" % args.username
res = site.login(args.username, args.password) # Optional
if res is None:
warnings.warn("Login with username '%s' failed! Check your username and password." % args.username)
#======================================================================
def initialize_Mantid(mantidpath):
""" Start mantid framework """
global mtd
global mantid_initialized
if mantid_initialized: return
sys.path.append(mantidpath)
sys.path.append(os.getcwd())
sys.path.append( os.path.join( os.getcwd(), 'bin') )
try:
import MantidFramework
from MantidFramework import mtd
except:
raise Exception("Error importing MantidFramework. Did you specify the --mantidpath option?")
mtd.initialise()
mantid_initialized = True
#======================================================================
def get_all_algorithms():
"""REturns a list of all algorithm names"""
temp = mtd._getRegisteredAlgorithms(True)
algos = [x for (x, version) in temp]
return algos
#======================================================================
def find_misnamed_algos():
"""Prints out algos that have mismatched class names and algorithm names"""
algos = get_all_algorithms()
print "\n--- The following algorithms have misnamed class files."
print "--- No file matching their algorithm name could be found.\n"
for algo in algos:
source = find_algo_file(algo)
if source=='':
print "%s was NOT FOUND" % (algo, )
#======================================================================
def find_section_text(lines, section, go_to_end=False, section2=""):
""" Search WIKI text to find a section text there """
if len(lines) == 0:
return ""
n = 0
for line in lines:
line_mod = line.replace(" ", "")
if line_mod.startswith("==%s" % section) \
or (section2 != "" and line_mod.startswith("==%s" % section2)):
# Section started
n += 1
doc = ""
# collect the documents till next section or the end
newline = lines[n]
while (go_to_end or not newline.strip().startswith('==')) \
and not newline.strip().startswith('[[Category'):
doc += newline + '\n'
n += 1
if n < len(lines):
newline = lines[n]
else:
break
return doc
n += 1
return ""
#======================================================================
def find_property_doc(lines, propname):
""" Search WIKI text to find a properties' documentation there """
if len(lines) == 0:
return ""
n = 0
for line in lines:
if line.strip() == "|" + propname:
doc = lines[n+4].strip()
if len(doc)>1:
doc = doc[1:]
return doc
n += 1
return ""
#======================================================================
def validate_wiki(args, algos):
""" Look for any missing wiki pages / inconsistencies """
missing = []
for algo in algos:
print "\n================= %s ===================" % algo
# Check wiki page and download contents
page = site.Pages[algo]
wiki = page.edit()
lines = []
if len(wiki) == 0:
print "- Algorithm wiki page for %s is MISSING!" % algo
missing.append(algo)
else:
lines = wiki.split('\n')
source = find_algo_file(algo)
if source=='':
print "- Source file not found."
# Validate the algorithm itself
alg = mtd.createAlgorithm(algo)
summary = alg._ProxyObject__obj.getWikiSummary()
if len(summary) == 0:
print "- Summary is missing (in the code)."
wikidoc = find_section_text(lines, "Summary", go_to_end=False, section2="")
if args.show_missing: print wikidoc
# desc = alg._ProxyObject__obj.getWikiDescription()
# if len(desc) == 0:
# print "- Wiki Description is missing (in the code)."
# desc = find_section_text(lines, "Description", True, "Introduction")
# if args.show_missing: print desc
#add_wiki_description(algo, desc)
props = alg._ProxyObject__obj.getProperties()
for prop in props:
if len(prop.documentation) == 0:
print "- Property %s has no documentation/tooltip (in the code)." % prop.name,
wikidoc = find_property_doc(lines, prop.name)
if len(wikidoc) > 0:
print "Found in wiki"
if args.show_missing: print " \"%s\"" % wikidoc
else:
print "Not found in wiki."
print "\n\n"
print "Algorithms with missing wiki page:\n", " ".join(missing)
#======================================================================
def find_orphan_wiki():
""" Look for pages on the wiki that do NOT correspond to an algorithm """
category = site.Pages['Category:Algorithms']
algos = get_all_algorithms()
for page in category:
algo_name = page.name
if not (algo_name.startswith("Category:")):
if not (algo_name in algos) :
print "Algorithm %s was not found." % algo_name
#======================================================================
if __name__ == "__main__":
# First, get the config for the last settings
config = ConfigParser.ConfigParser()
config_filename = os.path.split(__file__)[0] + "/wiki_tools.ini"
config.read(config_filename)
defaultmantidpath = ""
try:
defaultmantidpath = config.get("mantid", "path")
except:
pass
parser = argparse.ArgumentParser(description='Generate the Wiki documentation for one '
'or more algorithms, and updates the mantidproject.org website')
parser.add_argument('algos', metavar='ALGORITHM', type=str, nargs='*',
help='Name of the algorithm(s) to generate wiki docs.')
parser.add_argument('--mantidpath', dest='mantidpath', default=defaultmantidpath,
help="Full path to the Mantid compiled binary folder. Default: '%s'. This will be saved to an .ini file" % defaultmantidpath)
parser.add_argument('--validate', dest='validate_wiki', action='store_const',
const=True, default=False,
help="Validate algorithms' documentation. Validates them all if no algo is specified. Look for missing wiki pages, missing properties documentation, etc. using the list of registered algorithms.")
parser.add_argument('--show-missing', dest='show_missing', action='store_const',
const=True, default=False,
help="When validating, pull missing in-code documentation from the wiki and show it.")
parser.add_argument('--find-orphans', dest='find_orphans', action='store_const',
const=True, default=False,
help="Look for 'orphan' wiki pages that are set as Category:Algorithms but do not have a matching Algorithm in Mantid.")
parser.add_argument('--find-misnamed', dest='find_misnamed', action='store_const',
const=True, default=False,
help="Look for algorithms where the name of the algorithm does not match any filename.")
args = parser.parse_args()
# Write out config for next time
config = ConfigParser.ConfigParser()
config.add_section("mantid")
config.set("mantid", "path", args.mantidpath)
f = open(config_filename, 'w')
config.write(f)
f.close()
if not args.validate_wiki and not args.find_misnamed and not args.find_orphans and len(args.algos)==0:
parser.error("You must specify at least one algorithm if not using --validate")
initialize_Mantid(args.mantidpath)
intialize_files()
initialize_wiki(args)
if args.find_misnamed:
find_misnamed_algos()
sys.exit(1)
if args.find_orphans:
find_orphan_wiki()
sys.exit(1)
if args.validate_wiki:
# Validate a few, or ALL algos
algos = args.algos
if len(algos) == 0:
algos = get_all_algorithms()
validate_wiki(args, algos)