Commit c4bc887e authored by Shin, Woong's avatar Shin, Woong
Browse files

initial commit



* Initial import from prior project
Signed-off-by: Shin, Woong's avatarWoong Shin <shinw@ornl.gov>
parents
.pdo-venv
__pycache__
*.swp
.*.swp
*.pyc
tmp
*.egg-info
.PHONY: help
help:
-@echo "[pdo make]"
-@echo "development:"
-@echo " init|fini: initialize / finalize development environment"
-@echo " env: printout environment to activate the virtualenv"
-@echo " test: run unittests"
VENV=./.pdo-venv
.PHONY: init fini env
init:
-@echo "@ initializing development environment"
if [ ! -e $(VENV) ]; then python3 -m venv $(VENV) && pip install --upgrade pip; fi
$(VENV)/bin/pip3 install -r ./requirements.txt
$(VENV)/bin/pip3 install -r ./requirements_dev.txt
$(VENV)/bin/pip3 install -e .
-@echo "@ virtual environment setup finished"
-@echo '@ . ./.pdo-venv/bin/activate'
fini:
-rm -rf $(VENV)
test:
$(VENV)/bin/pytest -sv
pdo toolkit
===========
Tools for container based development orchestration
import os
import sys
import logging
from .environ import *
from .exceptions import *
# Logger object
LOG_LEVEL = os.environ.get('PDO_LOG_LEVEL', 'INFO')
logging.basicConfig(
level=LOG_LEVEL.upper(),
format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
handlers=[logging.StreamHandler(sys.stdout)]
)
# Version from the scm
__version__ = 'tbd'
try:
from .version import __version__
except ImportError:
pass
from .cli import pdo_cli
"""
adm.py - Administrator commands
"""
import sys
import click
import logging
from .utils import AliasedGroup
logger = logging.getLogger(__name__)
@click.group(cls=AliasedGroup)
@click.pass_context
def cli(ctx):
pass
@cli.group()
@click.pass_context
def adm(ctx):
"""Admin commands"""
pass
@adm.command('project')
@click.option('--delete', '-D', is_flag=True, default=False)
@click.option('--add', '-A', is_flag=True, default=False)
@click.argument('name', default=None, required=False)
@click.pass_context
def adm_project(ctx, delete, add, name):
"""Example command"""
sys.exit(0)
"""
cli.py - CLI entrypoint for mts users
"""
import os
import click
import json
from pdo.cli.porcelain import cli as porcelain_cli
from pdo.cli.adm import cli as adm_cli
import logging
import logging.config
from .utils import AliasedGroup
from .utils import get_dotcontext_dir
logger = logging.getLogger(__name__)
# Composite for the main mtsctl client
cli_source = [
porcelain_cli,
adm_cli,
]
def find_context():
context = get_dotcontext_dir(dotprefix='.pdo')
if context is None:
context = click.get_app_dir('pdo', force_posix=True)
return context
def prepare_dot_context(ctx, machine, log_file, log_level, context):
"""Common context preparation function"""
# Load an initialized context if file does not exist
ctx.allow_extra_args = True
@click.command(cls=click.CommandCollection, sources=cli_source)
# Duplication of 'option' section here is almost unavoidable
# We just rely on careful inspection and replication
@click.option('--machine', '-m', is_flag=True, default=False,
help="Produce machine readable output")
@click.option('--log_file', '-l', default=os.environ.get('DJ_LOGFILE', 'dj.log'),
help="Designate a logfile for output")
@click.option('--log_level', '-L', default=os.environ.get('DJ_LOGLEVEL', 'INFO'),
help="Log level (DEBUG, *INFO*, WARNING, ERROR, CRITICAL)")
@click.option('--context', '-c',
default=lambda: os.path.join(find_context(), 'config'),
help="Context config file location (default: ~/.dj/config)"
)
@click.pass_context
def pdo_cli(ctx, machine, log_file, log_level, context):
"""
pdo Toolkit
"""
ctx.obj = None
"""
porcelain.py - convenience commands
"""
import os
import sys
import click
import logging
import yaml
import subprocess
from datetime import datetime
from click.testing import CliRunner
from ..environ import *
from .. import __version__
from .utils import AliasedGroup
logger = logging.getLogger(__name__)
@click.group(cls=AliasedGroup)
@click.pass_context
def cli(ctx):
pass
@cli.command()
def version():
"""Version"""
click.echo(__version__)
return 0
@cli.command()
@click.option('--url', '-U', default=None)
@click.option('--organization', '-o', default=None)
@click.option('--project', '-p', default=None)
@click.option('--username', '-u', default=None)
@click.option('--site-config', '-c', default=None)
@click.pass_context
def init(ctx, url, organization, project, username, site_config):
"""Create and initialize workspace"""
pass
"""
site.py - Site configuration commands
"""
import sys
import click
import logging
from socket import getfqdn
from .utils import AliasedGroup
from .utils import edit_yaml_object
from ..resources.site import SiteConfig, Site
from ..resources.ops import load_site_config_raw, dump_site_config_raw
from ..resources.ops import get_current_cluster
from .utils import pp_details
logger = logging.getLogger(__name__)
def site_and_cluster(ctx):
site = ctx.obj.load_site_config()
cluster = get_current_cluster(site)
return site, cluster
@click.group(cls=AliasedGroup)
@click.pass_context
def cli(ctx):
pass
@cli.group()
@click.pass_context
def site(ctx):
"""Site configuration management"""
pass
@site.command('up')
@click.pass_context
def site_up(ctx):
site, cluster = site_and_cluster(ctx)
click.echo(f'Configuring site {cluster.name} to an UP state')
click.echo(f'Starting agent01')
click.echo(f'Starting agent02')
click.echo(f'Submitting agent_job_01')
click.echo(f'Starting agent03')
ctx.exit(0)
@site.command('down')
@click.pass_context
def site_down(ctx):
site, cluster = site_and_cluster(ctx)
click.echo(f'Configuring site {cluster.name} to a DOWN state')
click.echo(f'Stopping agent01')
click.echo(f'Stopping agent02')
click.echo(f'Canceling agent_job_01')
click.echo(f'Stopping agent03')
ctx.exit(0)
@site.command('status')
@click.pass_context
def site_status(ctx):
site, cluster = site_and_cluster(ctx)
data = {
'current': {
'node': getfqdn(),
'cluster': cluster.name,
'domains': cluster.domains,
},
'agents': [
'agent01',
'agent02',
'agent03',
],
}
pp_details(data, machine=ctx.obj.machine)
ctx.exit(0)
@site.command('edit')
@click.pass_context
def site_edit(ctx):
"""Edit site configuration
Opens the site configuration file using the editor stated in the
$EDITOR environment variable
"""
try:
data = load_site_config_raw(ctx.obj.current_context_filename())
schema = SiteConfig()
site_cfg_payload = edit_yaml_object(data, ctx, schema=schema, raise_exc=False)
dump_site_config_raw(ctx.obj.current_context_filename(), site_cfg_payload)
except Exception as e:
click.echo('Error: {}'.format(e))
ctx.exit(1)
ctx.exit(0)
"""
utils.py - CLI utilities
"""
import os
import sys
import pwd
import yaml
import click
import json
import yaml
from terminaltables import AsciiTable, SingleTable
class AliasedGroup(click.Group):
"""Click extension that guesse the commands form shortcuts
"""
def get_command(self, ctx, cmd_name):
"""Shortcuts to the commands
"""
rv = click.Group.get_command(self, ctx, cmd_name)
if rv is not None:
return rv
matches = [x for x in self.list_commands(ctx)
if x.startswith(cmd_name)]
if not matches:
return None
elif len(matches) == 1:
return click.Group.get_command(self, ctx, matches[0])
ctx.fail('Too many matches: %s' % ', '.join(sorted(matches)))
def get_dotcontext_dir(workdir=os.getcwd(), dotprefix='.dj'):
"""Try to search for a workspace context starting from 'workdir'"""
while not (workdir == '' or workdir == '/'):
ctxdir = os.path.join(workdir, dotprefix)
if os.access(ctxdir, os.F_OK) is True:
return ctxdir
workdir = os.path.dirname(workdir)
ctxdir = os.path.join(workdir, dotprefix)
if os.access(ctxdir, os.F_OK) is True:
return ctxdir
return None
def edit_yaml_object(data, ctx=sys, schema=None, header='', raise_exc=False):
"""Invoke edits for an object and get a validated object
Performs use interaction with an editor specified by the user
Loops until user aborts the edit and passes through validation
:param data: python dictionary
:param ctx: sys object or a click context
:param schema: Marshmallow schema object to verify contents
:param raise_exc: Raises an exception instead of interacting with the user
:returns: the parsed & verified dictionary from the user
"""
payload = header + yaml.dump(data)
while True:
ret = click.edit(payload, extension='.yml')
if not ret:
if raise_exc:
raise RuntimeError('No edits')
click.echo('No edits: abort? [y] - otherwise retry')
c = click.getchar()
if c == 'y':
ctx.exit(1)
else:
continue
payload = ret
# Parse the input and validate using the schema if provided
# schema object is meant to be a marshmallow object
try:
data = yaml.safe_load(payload)
if schema:
schema.load(data)
except Exception as e:
if raise_exc:
raise e
#click.clear()
click.echo(e)
click.echo("Error in edits: abort? [y] - otherwise retry")
if click.getchar() == 'y':
click.echo ('Abort edit')
ctx.exit(1)
# Pressed any key.. reedit
continue
click.echo('Edits accepted')
break
return data
def table_out(data, title=None):
table = SingleTable(
data,
title=title,
)
table.inner_column_border = False
table.inner_footing_row_border = False
table.inner_heading_row_border = False
table.inner_row_border = False
table.outer_border = False
table.justify_columns = {}
table.padding_left = 1
table.padding_right = 1
click.echo(table.table)
def table_out_json(data):
click.echo(json.dumps(data))
def pp_table(data: list, header=[], machine=False, nohead=False):
"""Prints out data to the console"""
out_data = list()
# If header is provided, it will be used as the header
if not nohead and header:
out_data.append(header)
# Data should be a list of lists
out_data += data
if machine:
table_out_json(out_data)
else:
table_out(out_data)
def pp_records(data: list, header=[], machine=False, nohead=False):
"""Prints out data to the console"""
out_data = list()
# If header is provided, it will be used as the header
if not nohead and header:
out_data.append(list(map(str.upper, header)))
# Data should be a list of lists
for rec in data:
if type(rec) is not dict:
continue
if not header:
header = rec.keys()
if not nohead:
out_data.append(list(map(str.upper, header)))
continue
row = list()
for fld in header:
row.append(rec[fld])
out_data.append(row)
if machine:
table_out_json(out_data)
else:
table_out(out_data)
def pp_details(data: dict, table=None, machine=False):
"""Printout details, human or json"""
out_data = dict()
out_data.update(data)
if table and type(table) is list:
out_data['items'] = table
if not machine:
click.echo(yaml.dump(out_data, default_flow_style=False))
else:
click.echo(json.dumps(out_data))
"""
environ.py - environment imports
"""
import os
import sys
#
# Exceptions
#
class UnauthorizedError(Exception):
"""Unauthorized error"""
class ConflictError(Exception):
"""Conflict error"""
class NotFoundError(Exception):
"""Not found error"""
class UpdateConflictError(ConflictError):
"""Update failed due to a conflict"""
class LockError(Exception):
"""Errors upon locking"""
class LockConflictError(ConflictError):
"""Somebody else's lock was held"""
class UnlockError(Exception):
"""Cannot unlock document"""
class InvalidPath(Exception):
"""The route path was invalid"""
class ValidationError(Exception):
"""Validation error occured"""
"""
fixtures.py - test support for rails
"""
import os
import time
import pytest
import yaml
import subprocess
from click.testing import CliRunner
from pdo.cli.cli import find_context
#
# Helper functions for tests
#
def assert_json_response(response, status=None, **kwargs):
"""Common check functions normal json response objects"""
assert response.status_code == status
assert response.headers['content-type'] == 'application/json'
try:
payload = response.get_json()
except:
payload = response.json()
for k, v in kwargs.items():
assert payload[k] == v
return payload
def assert_json_problem_report(response, status=None, title=None, detail=None, type=None,
instance=None, headers=None, ext=None):
"""Common check functions for connexion ProblemException exceptions"""
assert response.status_code == status
assert response.headers['content-type'] == 'application/problem+json'
try:
payload = response.get_json()
except:
payload = response.json()
cnt = 0
if status:
cnt += 1
assert payload['status'] == status
if title:
cnt += 1
assert payload['title'] == title
if detail:
cnt += 1
assert payload['detail'] == detail
if type:
cnt += 1
assert payload['type'] == type
if instance:
cnt += 1
assert payload['instance'] == headers
if ext:
cnt += 1
assert payload['ext'] == ext
# At least one details should be provided
assert cnt > 0
# Return the payload for further assertions
return payload
#
# Schema and models
#
@pytest.fixture
def mock_apigw():
instance = ApiGWMockServer()
yield instance
del instance
#
# Integration test support
#
@pytest.fixture
def valid_testcreds_auth():
"""
Fetch credentials from the test environment
"""
username = os.environ['OLCFAUTH_TEST_USER']
password = os.environ['OLCFAUTH_TEST_PASSWD']
organization = 'olcf'
project = os.environ['OLCFAUTH_TEST_PROJECTS']
yield username, password, organization, project
@pytest.fixture
def valid_loopback_apikey_auth():
"""
Fetch the testing APIKEY for loopback test
"""
apikey = os.environ['APIGW_TEST_APIKEY']
username = os.environ['OLCFAUTH_TEST_USER']
password = os.environ['OLCFAUTH_TEST_PASSWD']
organization = 'olcf'
project = os.environ['OLCFAUTH_TEST_PROJECTS']
yield apikey, username, password, organization, project
TASK_END_STATES = [
'success',
'failed',
'cancel',