convert to py3.4 syntax, a few cleanups (93eb566a) · Commits · NDIP / Nova Packages / Nova Carpentry Tutorial

bin/lesson_check.py

+43 −48

Original line number	Diff line number	Diff line
		@@ -4,15 +4,14 @@
		Check lesson files and their contents.
		"""

		from __future__ import print_function
		import sys

		import os
		import glob
		import json
		import re
		from optparse import OptionParser

		from util import Reporter, read_markdown, load_yaml, check_unwanted_files, require, IMAGE_FILE_SUFFIX
		from util import (Reporter, read_markdown, load_yaml, check_unwanted_files,
		require)

		__version__ = '0.3'

		@@ -23,8 +22,9 @@ SOURCE_DIRS = ['', '_episodes', '_extras']
		# FIXME: We do not yet validate whether any files have the required
		# YAML headers, but should in the future.
		# The '%' is replaced with the source directory path for checking.
		# Episodes are handled specially, and extra files in '_extras' are also handled specially.
		# This list must include all the Markdown files listed in the 'bin/initialize' script.
		# Episodes are handled specially, and extra files in '_extras' are also handled
		# specially. This list must include all the Markdown files listed in the
		# 'bin/initialize' script.
		REQUIRED_FILES = {
		'%/CONDUCT.md': True,
		'%/CONTRIBUTING.md': False,
		@@ -101,6 +101,7 @@ BREAK_METADATA_FIELDS = {
		# How long are lines allowed to be?
		MAX_LINE_LEN = 100


		def main():
		"""Main driver."""

		@@ -110,9 +111,9 @@ def main():
		args.references = read_references(args.reporter, args.reference_path)

		docs = read_all_markdown(args.source_dir, args.parser)
		check_fileset(args.source_dir, args.reporter, docs.keys())
		check_fileset(args.source_dir, args.reporter, list(docs.keys()))
		check_unwanted_files(args.source_dir, args.reporter)
		for filename in docs.keys():
		for filename in list(docs.keys()):
		checker = create_checker(args, filename, docs[filename])
		checker.check()

		@@ -160,8 +161,10 @@ def check_config(reporter, source_dir):

		config_file = os.path.join(source_dir, '_config.yml')
		config = load_yaml(config_file)
		reporter.check_field(config_file, 'configuration', config, 'kind', 'lesson')
		reporter.check_field(config_file, 'configuration', config, 'carpentry', ('swc', 'dc', 'lc'))
		reporter.check_field(config_file, 'configuration',
		config, 'kind', 'lesson')
		reporter.check_field(config_file, 'configuration',
		config, 'carpentry', ('swc', 'dc', 'lc'))
		reporter.check_field(config_file, 'configuration', config, 'title')
		reporter.check_field(config_file, 'configuration', config, 'email')

		@@ -235,7 +238,8 @@ def check_fileset(source_dir, reporter, filenames_present):
		if m and m.group(1):
		seen.append(m.group(1))
		else:
		reporter.add(None, 'Episode {0} has badly-formatted filename', filename)
		reporter.add(
		None, 'Episode {0} has badly-formatted filename', filename)

		# Check for duplicate episode numbers.
		reporter.check(len(seen) == len(set(seen)),
		@@ -244,8 +248,7 @@ def check_fileset(source_dir, reporter, filenames_present):
		sorted(seen), sorted(set(seen)))

		# Check that numbers are consecutive.
		seen = [int(s) for s in seen]
		seen.sort()
		seen = sorted([int(s) for s in seen])
		clean = True
		for i in range(len(seen) - 1):
		clean = clean and ((seen[i+1] - seen[i]) == 1)
		@@ -281,7 +284,6 @@ class CheckBase(object):

		self.layout = None


		def check(self):
		"""Run tests."""

		@@ -292,7 +294,6 @@ class CheckBase(object):
		self.check_codeblock_classes()
		self.check_defined_link_references()


		def check_metadata(self):
		"""Check the YAML metadata."""

		@@ -301,31 +302,31 @@ class CheckBase(object):
		'Missing metadata entirely')

		if self.metadata and (self.layout is not None):
		self.reporter.check_field(self.filename, 'metadata', self.metadata, 'layout', self.layout)

		self.reporter.check_field(
		self.filename, 'metadata', self.metadata, 'layout', self.layout)

		def check_line_lengths(self):
		"""Check the raw text of the lesson body."""

		if self.args.line_lengths:
		over = [i for (i, l, n) in self.lines if (n > MAX_LINE_LEN) and (not l.startswith('!'))]
		over = [i for (i, l, n) in self.lines if (
		n > MAX_LINE_LEN) and (not l.startswith('!'))]
		self.reporter.check(not over,
		self.filename,
		'Line(s) are too long: {0}',
		', '.join([str(i) for i in over]))


		def check_trailing_whitespace(self):
		"""Check for whitespace at the ends of lines."""

		if self.args.trailing_whitespace:
		trailing = [i for (i, l, n) in self.lines if P_TRAILING_WHITESPACE.match(l)]
		trailing = [
		i for (i, l, n) in self.lines if P_TRAILING_WHITESPACE.match(l)]
		self.reporter.check(not trailing,
		self.filename,
		'Line(s) end with whitespace: {0}',
		', '.join([str(i) for i in trailing]))


		def check_blockquote_classes(self):
		"""Check that all blockquotes have known classes."""

		@@ -336,7 +337,6 @@ class CheckBase(object):
		'Unknown or missing blockquote type {0}',
		cls)


		def check_codeblock_classes(self):
		"""Check that all code blocks have known classes."""

		@@ -347,7 +347,6 @@ class CheckBase(object):
		'Unknown or missing code block type {0}',
		cls)


		def check_defined_link_references(self):
		"""Check that defined links resolve in the file.

		@@ -366,11 +365,10 @@ class CheckBase(object):
		'Internally-defined links may be missing definitions: {0}',
		', '.join(sorted(result)))


		def find_all(self, node, pattern, accum=None):
		"""Find all matches for a pattern."""

		assert type(pattern) == dict, 'Patterns must be dictionaries'
		assert isinstance(pattern, dict), 'Patterns must be dictionaries'
		if accum is None:
		accum = []
		if self.match(node, pattern):
		@@ -379,7 +377,6 @@ class CheckBase(object):
		self.find_all(child, pattern, accum)
		return accum


		def match(self, node, pattern):
		"""Does this node match the given pattern?"""

		@@ -387,15 +384,14 @@ class CheckBase(object):
		if key not in node:
		return False
		val = pattern[key]
		if type(val) == str:
		if isinstance(val, str):
		if node[key] != val:
		return False
		elif type(val) == dict:
		elif isinstance(val, dict):
		if not self.match(node[key], val):
		return False
		return True


		def get_val(self, node, *chain):
		"""Get value one or more levels down."""

		@@ -406,7 +402,6 @@ class CheckBase(object):
		break
		return curr


		def get_loc(self, node):
		"""Convenience method to get node's line number."""

		@@ -420,8 +415,8 @@ class CheckNonJekyll(CheckBase):
		"""Check a file that isn't translated by Jekyll."""

		def __init__(self, args, filename, metadata, metadata_len, text, lines, doc):
		super(CheckNonJekyll, self).__init__(args, filename, metadata, metadata_len, text, lines, doc)

		super(CheckNonJekyll, self).__init__(
		args, filename, metadata, metadata_len, text, lines, doc)

		def check_metadata(self):
		self.reporter.check(self.metadata is None,
		@@ -433,7 +428,8 @@ class CheckIndex(CheckBase):
		"""Check the main index page."""

		def __init__(self, args, filename, metadata, metadata_len, text, lines, doc):
		super(CheckIndex, self).__init__(args, filename, metadata, metadata_len, text, lines, doc)
		super(CheckIndex, self).__init__(args, filename,
		metadata, metadata_len, text, lines, doc)
		self.layout = 'lesson'

		def check_metadata(self):
		@@ -447,8 +443,8 @@ class CheckEpisode(CheckBase):
		"""Check an episode page."""

		def __init__(self, args, filename, metadata, metadata_len, text, lines, doc):
		super(CheckEpisode, self).__init__(args, filename, metadata, metadata_len, text, lines, doc)

		super(CheckEpisode, self).__init__(args, filename,
		metadata, metadata_len, text, lines, doc)

		def check(self):
		"""Run extra tests."""
		@@ -456,7 +452,6 @@ class CheckEpisode(CheckBase):
		super(CheckEpisode, self).check()
		self.check_reference_inclusion()


		def check_metadata(self):
		super(CheckEpisode, self).check_metadata()
		if self.metadata:
		@@ -470,19 +465,17 @@ class CheckEpisode(CheckBase):
		else:
		self.check_metadata_fields(TEACHING_METADATA_FIELDS)


		def check_metadata_fields(self, expected):
		for (name, type_) in expected:
		if name not in self.metadata:
		self.reporter.add(self.filename,
		'Missing metadata field {0}',
		name)
		elif type(self.metadata[name]) != type_:
		elif not isinstance(self.metadata[name], type_):
		self.reporter.add(self.filename,
		'"{0}" has wrong type in metadata ({1} instead of {2})',
		name, type(self.metadata[name]), type_)


		def check_reference_inclusion(self):
		"""Check that links file has been included."""

		@@ -507,7 +500,8 @@ class CheckReference(CheckBase):
		"""Check the reference page."""

		def __init__(self, args, filename, metadata, metadata_len, text, lines, doc):
		super(CheckReference, self).__init__(args, filename, metadata, metadata_len, text, lines, doc)
		super(CheckReference, self).__init__(
		args, filename, metadata, metadata_len, text, lines, doc)
		self.layout = 'reference'


		@@ -515,7 +509,8 @@ class CheckGeneric(CheckBase):
		"""Check a generic page."""

		def __init__(self, args, filename, metadata, metadata_len, text, lines, doc):
		super(CheckGeneric, self).__init__(args, filename, metadata, metadata_len, text, lines, doc)
		super(CheckGeneric, self).__init__(args, filename,
		metadata, metadata_len, text, lines, doc)
		self.layout = 'page'

bin/lesson_initialize.py

+2 −3

Original line number	Diff line number	Diff line
		@@ -3,7 +3,6 @@
		"""Initialize a newly-created repository."""


		from __future__ import print_function
		import sys
		import os

bin/repo_check.py

+32 −27

Original line number	Diff line number	Diff line
		@@ -4,14 +4,14 @@
		Check repository settings.
		"""

		from __future__ import print_function

		import sys
		import os
		from subprocess import Popen, PIPE
		import re
		from optparse import OptionParser

		from util import Reporter, load_yaml, require
		from util import Reporter, require

		# Import this way to produce a more useful error message.
		try:
		@@ -103,7 +103,8 @@ def get_repo_url(source_dir, repo_url):

		# Guess.
		cmd = 'git remote -v'
		p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, close_fds=True, universal_newlines=True)
		p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE,
		close_fds=True, universal_newlines=True)
		stdout_data, stderr_data = p.communicate()
		stdout_data = stdout_data.split('\n')
		matches = [P_GIT_REMOTE.match(line) for line in stdout_data]
		@@ -112,10 +113,12 @@ def get_repo_url(source_dir, repo_url):
		'Unexpected output from git remote command: "{0}"'.format(matches))

		username = matches[0].group(1)
		require(username, 'empty username in git remote output {0}'.format(matches[0]))
		require(
		username, 'empty username in git remote output {0}'.format(matches[0]))

		project_name = matches[0].group(2)
		require(username, 'empty project name in git remote output {0}'.format(matches[0]))
		require(
		username, 'empty project name in git remote output {0}'.format(matches[0]))

		url = F_REPO_URL.format(username, project_name)
		return url
		@@ -154,13 +157,15 @@ def get_labels(repo_url):
		"""

		m = P_REPO_URL.match(repo_url)
		require(m, 'repository URL {0} does not match expected pattern'.format(repo_url))
		require(
		m, 'repository URL {0} does not match expected pattern'.format(repo_url))

		username = m.group(1)
		require(username, 'empty username in repository URL {0}'.format(repo_url))

		project_name = m.group(2)
		require(username, 'empty project name in repository URL {0}'.format(repo_url))
		require(
		username, 'empty project name in repository URL {0}'.format(repo_url))

		url = F_API_URL.format(username, project_name)
		r = requests.get(url)

bin/test_lesson_check.py

+3 −1

Original line number	Diff line number	Diff line
		@@ -3,9 +3,10 @@ import unittest
		import lesson_check
		import util


		class TestFileList(unittest.TestCase):
		def setUp(self):
		self.reporter = util.Reporter() ## TODO: refactor reporter class.
		self.reporter = util.Reporter() # TODO: refactor reporter class.

		def test_file_list_has_expected_entries(self):
		# For first pass, simply assume that all required files are present
		@@ -15,5 +16,6 @@ class TestFileList(unittest.TestCase):
		lesson_check.check_fileset('', self.reporter, all_filenames)
		self.assertEqual(len(self.reporter.messages), 0)


		if __name__ == "__main__":
		unittest.main()

bin/util.py

+14 −12

Original line number	Diff line number	Diff line
		from __future__ import print_function

		import sys
		import os
		import json
		@@ -29,6 +29,7 @@ UNWANTED_FILES = [
		# (Can't use 'None' because that might be a legitimate value.)
		REPORTER_NOT_SET = []


		class Reporter(object):
		"""Collect and report errors."""

		@@ -38,7 +39,6 @@ class Reporter(object):
		super(Reporter, self).__init__()
		self.messages = []


		def check_field(self, filename, name, values, key, expected=REPORTER_NOT_SET):
		"""Check that a dictionary has an expected value."""

		@@ -48,10 +48,11 @@ class Reporter(object):
		pass
		elif type(expected) in (tuple, set, list):
		if values[key] not in expected:
		self.add(filename, '{0} {1} value {2} is not in {3}', name, key, values[key], expected)
		self.add(
		filename, '{0} {1} value {2} is not in {3}', name, key, values[key], expected)
		elif values[key] != expected:
		self.add(filename, '{0} {1} is {2} not {3}', name, key, values[key], expected)

		self.add(filename, '{0} {1} is {2} not {3}',
		name, key, values[key], expected)

		def check(self, condition, location, fmt, *args):
		"""Append error if condition not met."""
		@@ -59,13 +60,11 @@ class Reporter(object):
		if not condition:
		self.add(location, fmt, *args)


		def add(self, location, fmt, *args):
		"""Append error unilaterally."""

		self.messages.append((location, fmt.format(*args)))


		def report(self, stream=sys.stdout):
		"""Report all messages in order."""

		@@ -111,11 +110,13 @@ def read_markdown(parser, path):

		# Split into lines.
		metadata_len = 0 if metadata_raw is None else metadata_raw.count('\n')
		lines = [(metadata_len+i+1, line, len(line)) for (i, line) in enumerate(body.split('\n'))]
		lines = [(metadata_len+i+1, line, len(line))
		for (i, line) in enumerate(body.split('\n'))]

		# Parse Markdown.
		cmd = 'ruby {0}'.format(parser)
		p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, close_fds=True, universal_newlines=True)
		p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE,
		close_fds=True, universal_newlines=True)
		stdout_data, stderr_data = p.communicate(body)
		doc = json.loads(stdout_data)

		@@ -136,7 +137,6 @@ def split_metadata(path, text):

		metadata_raw = None
		metadata_yaml = None
		metadata_len = None

		pieces = text.split('---', 2)
		if len(pieces) == 3:
		@@ -145,7 +145,8 @@ def split_metadata(path, text):
		try:
		metadata_yaml = yaml.load(metadata_raw)
		except yaml.YAMLError as e:
		print('Unable to parse YAML header in {0}:\n{1}'.format(path, e), file=sys.stderr)
		print('Unable to parse YAML header in {0}:\n{1}'.format(
		path, e), file=sys.stderr)
		sys.exit(1)

		return metadata_raw, metadata_yaml, text
		@@ -161,7 +162,8 @@ def load_yaml(filename):
		with open(filename, 'r') as reader:
		return yaml.load(reader)
		except (yaml.YAMLError, IOError) as e:
		print('Unable to load YAML file {0}:\n{1}'.format(filename, e), file=sys.stderr)
		print('Unable to load YAML file {0}:\n{1}'.format(
		filename, e), file=sys.stderr)
		sys.exit(1)