diff --git a/.env.ci b/.env.ci index d80e981088bcb9252d906cc98850d91b64a78e45..53031808d2eb3b24d9b2a573a461eee2ae10421b 100644 --- a/.env.ci +++ b/.env.ci @@ -20,7 +20,7 @@ # TEST_SERVER_HOSTNAME=192.168.0.1 # TEST_SERVER_HOSTNAME=example.com # -TEST_SERVER_HOSTNAME=foobar.invalid +TEST_SERVER_HOSTNAME=djrems # The SSH port of the server to connect to. This will almost always be port 22, # but it could be different. @@ -37,7 +37,7 @@ TEST_SERVER_PORT=22 # Example(s): # TEST_REMOTE_USER=johnsmith # -TEST_REMOTE_USER=foo +TEST_REMOTE_USER=testuser # The remote user's password. This will be used for initial connections during # the tests as well as testing the functionality of the public key transfers. @@ -46,7 +46,7 @@ TEST_REMOTE_USER=foo # Example(s): # TEST_REMOTE_PASSWORD=p4ssw0rd # -TEST_REMOTE_PASSWORD=bar +TEST_REMOTE_PASSWORD=p4ssw0rd # The remote directory to store scripts in. This directory should already exist # as the tests won't create it automatically. @@ -69,7 +69,7 @@ TEST_REMOTE_DIRECTORY=/tmp/ # Example(s): # TEST_REMOTE_FILENAME=foobar.py # -TEST_REMOTE_FILENAME=foobar.py +TEST_REMOTE_FILENAME=jobtest.py # The path to the Python interpreter to use. If the path is not absolute, the # $PATH variable will be searched to find the right executable. This can be diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8b38159967920df93fc9fde406ebbe53164729d4..ac5bccdddd00694ec1e0261dfa0ef8c3f35009f1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,6 +19,9 @@ stages: - docker system prune -f -a --volumes # get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} + # download docker-compose executable + - curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o ./docker-compose + - chmod +x ./docker-compose .docker-teardown: &docker-teardown - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs @@ -42,14 +45,6 @@ flake8: tags: - dev -before_script: - - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines -# get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) - - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} - -after_script: - - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs - dockerbuild: stage: build before_script: @@ -64,14 +59,16 @@ dockerbuild: - dev # identify which runners can run this job # Tests that do not require a remote worker -noremote: +test: stage: test before_script: - *docker-setup script: - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry - # pytest with coverage, creating a volume to retrieve the coverage output from the container - - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "make test-noremote" + - docker tag ${IMAGE_NAMETAG} djrems # retag the image + - mv .env.ci .env + # create a container where to run pytest of remote tests with coverage + - ./docker-compose up after_script: - *docker-teardown artifacts: diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 3a7bb7ab9d0130346c25570d8537416b6ce1a216..ee46f2ba2d9883cbc7c628ddb2c349368586e61a 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -156,6 +156,6 @@ create a conda environment, although a virtualenv should be equally acceptable 5. Fire up the conda environment and invoke pytest:: - $ conda activate djroms + $ conda activate djrems (djrems) pytest -v ./tests # run all tests (djrems) pytest -v ./tests/test_tasks.py::test_deploy_and_delete_key # singe test function diff --git a/Dockerfile b/Dockerfile index 27c954664610b9d4b54a4d8e709b02e26cc8e7ff..193403bf3c10146305341fbd0f0bece1f1bddf5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,3 +18,10 @@ RUN apt-get update && apt-get install -y emacs iputils-ping less make mlocate ne RUN conda install --name base -c conda-forge mamba RUN mamba env update --name base --file conda.environment/environment-dev.yml + +# Will start conda environment when $HOME/.bashrc is invoked +# Critical to prepend .bashrc, otherwise the conda environment +# will not be activated when executing commands via remote SSH +RUN echo "$(echo "conda activate base"; cat /etc/skel/.bashrc)" > /etc/skel/.bashrc +RUN echo "$(echo "source /etc/profile.d/conda.sh"; cat /etc/skel/.bashrc)" > /etc/skel/.bashrc +RUN echo "$(echo "# Activate Conda Environment base"; cat /etc/skel/.bashrc)" > /etc/skel/.bashrc diff --git a/Makefile b/Makefile index fb8bef1746b98e81577391ee9aeefd64f892d222..3d5aa1c3286fda5f8fc7aff14870b1e95672d5b1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean-pyc clean-build condadev docs help +.PHONY: clean-pyc clean-build condadev coverage dockerbuilddev dockercleanall docs test-noremote test-remote test help SHELL := bash .DEFAULT_GOAL := help define BROWSER_PYSCRIPT @@ -36,12 +36,15 @@ clean-pyc: ## remove Python file artifacts doclint: ## check documentation style with flake8 flake8 --select=D --ignore=F django_remote_submission -test: ## run tests quickly with the default Python - pytest -v - test-noremote: ## run tests skipping those requiring a remote worker pytest -v -m "not remote_required" --cov-report html:artifacts/htmlcov --cov-report term --cov-report xml:artifacts/coverage.xml --cov=django_remote_submission tests/ +test-remote: ## run tests skipping those requiring a remote worker + pytest -v -m "remote_required" --cov-report html:artifacts/htmlcov_r --cov-report term --cov-report xml:artifacts/coverage.xml --cov=django_remote_submission tests/ + +test: ## run tests with coverage + pytest -v --cov-report html:artifacts/htmlcov --cov-report term --cov-report xml:artifacts/coverage.xml --cov=django_remote_submission tests/ + coverage: ## check code coverage quickly with the default Python py.test --cov=django_remote_submission tests/ @@ -77,3 +80,10 @@ condadev: ## create conda environment for developing, and install pre-commit hoo conda install -y --name djrems -c conda-forge mamba mamba env update -y --name djrems --file ./conda.environment/environment-dev.yml $(CONDA_ACTIVATE) djrems; python setup.py develop; pre-commit install + +dockerbuilddev: ## create docker image for testing + /bin/cp .env.ci .env + docker build --network host -t djrems . + +dockercleanall: ## delete ALL containers, images, and volumes + docker system prune -f -a --volumes diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..0001a1de93aa81a387a43823879193f1643fde3c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,21 @@ +#use for local development + +version: '3' + +services: + + djrems: + restart: always + image: djrems + hostname: djrems + expose: + - "22" + env_file: + - .env + volumes: + - ./docker-entrypoint.sh:/usr/bin/docker-entrypoint.sh + - artifacts:/opt/django_remote_submission/artifacts + command: /usr/bin/docker-entrypoint.sh + +volumes: + artifacts: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000000000000000000000000000000000000..d3642c19d94ec255dac1cf1e6846533d9ecceddb --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e # stop and exit script if any command or pipeline returns + +echo "OPENSSL VERSION = $(openssl version)" + +# create account for test user +if id "${TEST_REMOTE_USER}" &>/dev/null; then + echo "${TEST_REMOTE_PASSWORD} already exists" +else + useradd -m -p $(openssl passwd -1 ${TEST_REMOTE_PASSWORD}) -s /bin/bash -G users ${TEST_REMOTE_USER} +fi + +# start the ssh daemon +/etc/init.d/ssh start + +# run remote tests +make test + +# hack preventing container from exiting +# uncomment only for debugging purposes +# tail -f /dev/null diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 5d6861f6990a8ab457cb27c9e5eacde45af8c6b2..0eefe9620ef21cb1edb16bcfe737f60b564c33f9 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -30,17 +30,30 @@ def pairwise(iterable): return zip(a, b) -EnvBase = collections.namedtuple('Env', [ - 'server_hostname', 'server_port', 'remote_directory', 'remote_filename', - 'remote_user', 'remote_password', 'python_path', 'python_arguments', -]) +EnvBase = collections.namedtuple( + 'Env', + [ + 'server_hostname', + 'server_port', + 'remote_directory', + 'remote_filename', + 'remote_user', + 'remote_password', + 'python_path', + 'python_arguments', + ], +) class Env(EnvBase): def __repr__(self): - return super(Env, self).__repr__().replace( - 'remote_password={!r}'.format(self.remote_password), - 'remote_password={!r}'.format('******'), + return ( + super(Env, self) + .__repr__() + .replace( + 'remote_password={!r}'.format(self.remote_password), + 'remote_password={!r}'.format('******'), + ) ) @@ -166,11 +179,11 @@ def job_model_saved(mocker): @pytest.fixture(params=[True, False], ids=["Remote", "Local"]) def runs_remotely(request): - ''' + """ params == True: Uses RemoteWrapper params == False: Uses LocalWrapper if it's running with command line marker -m "not remote_required" skip it - ''' + """ skipif_remote_required = "not remote_required" in request.config.getoption('-m') if not request.param: @@ -184,13 +197,15 @@ def runs_remotely(request): @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time for i in range(5): print("line: {}".format(i)) time.sleep(0.1) -''') +''' +) def test_submit_job_normal_usage(env, job, job_model_saved, runs_remotely): from django_remote_submission.models import Job, Log from django_remote_submission.tasks import submit_job_to_server @@ -216,15 +231,16 @@ def test_submit_job_normal_usage(env, job, job_model_saved, runs_remotely): @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ hello world -''') +''' +) def test_copy_job(env, job, job_model_saved, runs_remotely): from django_remote_submission.models import Job, Log from django_remote_submission.tasks import copy_job_to_server - results = copy_job_to_server( - job.pk, env.remote_password, remote=runs_remotely - ) + + results = copy_job_to_server(job.pk, env.remote_password, remote=runs_remotely) assert len(results) == 0 @@ -239,14 +255,16 @@ def test_copy_job(env, job, job_model_saved, runs_remotely): @pytest.mark.remote_required @pytest.mark.parametrize("runs_remotely", ["Remote"]) @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time import sys for i in range(5): print("line: {}".format(i), file=sys.stdout if i % 2 == 0 else sys.stderr) time.sleep(0.1) -''') +''' +) def test_submit_job_multiple_streams(env, job, runs_remotely): ''' This one cannot run in local as the Live Log is not real ''' from django_remote_submission.models import Log @@ -272,10 +290,12 @@ def test_submit_job_multiple_streams(env, job, runs_remotely): @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ import sys sys.exit(1) -''') +''' +) def test_submit_job_failure(env, job, runs_remotely): from django_remote_submission.models import Job from django_remote_submission.tasks import submit_job_to_server @@ -287,20 +307,21 @@ def test_submit_job_failure(env, job, runs_remotely): @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time import sys for i in range(5): print('line: {}'.format(i), file=sys.stdout) time.sleep(0.1) -''') +''' +) def test_submit_job_log_policy_log_total(env, job, runs_remotely): from django_remote_submission.models import Log from django_remote_submission.tasks import submit_job_to_server, LogPolicy - submit_job_to_server(job.pk, env.remote_password, remote=runs_remotely, - log_policy=LogPolicy.LOG_TOTAL) + submit_job_to_server(job.pk, env.remote_password, remote=runs_remotely, log_policy=LogPolicy.LOG_TOTAL) assert Log.objects.count() == 1 log = Log.objects.get() @@ -309,41 +330,42 @@ def test_submit_job_log_policy_log_total(env, job, runs_remotely): @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time import sys for i in range(5): print('line: {}'.format(i), file=sys.stdout) time.sleep(0.1) -''') +''' +) def test_submit_job_log_policy_log_none(env, job, runs_remotely): from django_remote_submission.models import Log from django_remote_submission.tasks import submit_job_to_server, LogPolicy - submit_job_to_server(job.pk, env.remote_password, remote=runs_remotely, - log_policy=LogPolicy.LOG_NONE) + submit_job_to_server(job.pk, env.remote_password, remote=runs_remotely, log_policy=LogPolicy.LOG_NONE) assert Log.objects.count() == 0 @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time import sys for i in range(5): print('line: {}'.format(i)) time.sleep(0.35) -''') +''' +) def test_submit_job_timeout(env, job, runs_remotely): from django_remote_submission.models import Job, Log from django_remote_submission.tasks import submit_job_to_server import datetime - submit_job_to_server(job.pk, env.remote_password, - remote=runs_remotely, - timeout=datetime.timedelta(seconds=1)) + submit_job_to_server(job.pk, env.remote_password, remote=runs_remotely, timeout=datetime.timedelta(seconds=1)) assert Log.objects.count() == 3 @@ -352,7 +374,8 @@ def test_submit_job_timeout(env, job, runs_remotely): @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time import sys @@ -360,35 +383,32 @@ for i in range(5): with open('{}.txt'.format(i), 'w') as f: print('line: {}'.format(i), file=f) time.sleep(0.1) -''') +''' +) def test_submit_job_modified_files(env, job, runs_remotely): from django_remote_submission.models import Result from django_remote_submission.tasks import submit_job_to_server import re - results = submit_job_to_server(job.pk, env.remote_password, - remote=runs_remotely) + results = submit_job_to_server(job.pk, env.remote_password, remote=runs_remotely) assert len(results) == 5 - assert sorted(results.keys()) == \ - ['0.txt', '1.txt', '2.txt', '3.txt', '4.txt'] + assert sorted(results.keys()) == ['0.txt', '1.txt', '2.txt', '3.txt', '4.txt'] for (result_fname, result_pk) in results.items(): result = Result.objects.get(pk=result_pk) i = int(re.match(r'^([0-9])\.txt', result_fname).group(1)) - assert result.local_file.read().decode('utf-8') == \ - 'line: {}\n'.format(i) + assert result.local_file.read().decode('utf-8') == 'line: {}\n'.format(i) - matcher = re.compile( - r'results/{}/[0-4].txt'.format(job.uuid) - ) + matcher = re.compile(r'results/{}/[0-4].txt'.format(job.uuid)) assert matcher.match(result.local_file.name) is not None @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time import sys @@ -396,30 +416,30 @@ for i in range(5): with open('{}.txt'.format(i), 'w') as f: print('line: {}'.format(i), file=f) time.sleep(0.1) -''') +''' +) def test_submit_job_modified_files_positive_pattern(env, job, runs_remotely): from django_remote_submission.models import Result from django_remote_submission.tasks import submit_job_to_server import re - results = submit_job_to_server(job.pk, env.remote_password, - remote=runs_remotely, - store_results=['0.txt', '[12].txt']) + results = submit_job_to_server( + job.pk, env.remote_password, remote=runs_remotely, store_results=['0.txt', '[12].txt'] + ) assert len(results) == 3 - assert sorted(results.keys()) == \ - ['0.txt', '1.txt', '2.txt'] + assert sorted(results.keys()) == ['0.txt', '1.txt', '2.txt'] for (result_fname, result_pk) in results.items(): result = Result.objects.get(pk=result_pk) i = int(re.match(r'^([0-9])\.txt', result_fname).group(1)) - assert result.local_file.read().decode('utf-8') == \ - 'line: {}\n'.format(i) + assert result.local_file.read().decode('utf-8') == 'line: {}\n'.format(i) @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time import sys @@ -427,31 +447,29 @@ for i in range(5): with open('{}.txt'.format(i), 'w') as f: print('line: {}'.format(i), file=f) time.sleep(0.1) -''') +''' +) def test_submit_job_modified_files_negative_pattern(env, job, runs_remotely): from django_remote_submission.models import Result from django_remote_submission.tasks import submit_job_to_server import re - results = submit_job_to_server(job.pk, env.remote_password, - remote=runs_remotely, - store_results=['*', '![34].txt']) + results = submit_job_to_server(job.pk, env.remote_password, remote=runs_remotely, store_results=['*', '![34].txt']) assert len(results) == 3 - assert sorted(results.keys()) == \ - ['0.txt', '1.txt', '2.txt'] + assert sorted(results.keys()) == ['0.txt', '1.txt', '2.txt'] for (result_fname, result_pk) in results.items(): result = Result.objects.get(pk=result_pk) i = int(re.match(r'^([0-9])\.txt', result_fname).group(1)) - assert result.local_file.read().decode('utf-8') == \ - 'line: {}\n'.format(i) + assert result.local_file.read().decode('utf-8') == 'line: {}\n'.format(i) @pytest.mark.django_db def test_submit_job_deploy_key(env, job_gen, interpreter_gen, runs_remotely): from django_remote_submission.tasks import submit_job_to_server + try: from shlex import quote as cmd_quote except ImportError: @@ -470,20 +488,32 @@ def test_submit_job_deploy_key(env, job_gen, interpreter_gen, runs_remotely): arguments=['python', '-u'], ) """ + sshdir = Path("/tmp") + id_file = IdentityFile(sshdir=sshdir) # temporary RSA identity file + + # Connect with password drop the key + wrapper = RemoteWrapper( + hostname=env.server_hostname, + username=env.remote_user, + port=env.server_port, + ) + with wrapper.connect(password=env.remote_password): + wrapper.deploy_key_if_it_does_not_exist(id_file.public) - id_file = IdentityFile(sshdir=Path("/tmp")) # temporary RSA identity file + # delete the key with open(id_file.public, 'rt') as f: key = f.read().strip() remove_existing_key_job = job_gen( program='''\ sed -i.bak -e /{key}/d ~/.ssh/authorized_keys - '''.format(key=cmd_quote(key.replace(r'/', r'\/'))), + '''.format( + key=cmd_quote(key.replace(r'/', r'\/')) + ), interpreter=sh, ) - submit_job_to_server(remove_existing_key_job.pk, env.remote_password, - remote=runs_remotely) + submit_job_to_server(remove_existing_key_job.pk, env.remote_password, remote=runs_remotely) add_key_job = job_gen( program='''\ @@ -492,15 +522,14 @@ def test_submit_job_deploy_key(env, job_gen, interpreter_gen, runs_remotely): interpreter=sh, ) - submit_job_to_server(add_key_job.pk, env.remote_password, - remote=runs_remotely) + # run the job using not the key (it was removed) but the password + submit_job_to_server(add_key_job.pk, env.remote_password, remote=runs_remotely) @pytest.mark.remote_required @pytest.mark.django_db def test_delete_key_old_way(env): - from django_remote_submission.wrapper.remote import RemoteWrapper wrapper = RemoteWrapper( hostname=env.server_hostname, username=env.remote_user, @@ -561,8 +590,14 @@ def test_deploy_and_delete_key(env): ) # delete the key from server passing the private key file for credentials - copy_key_to_server(public_key_filename=id_file.public, username=env.remote_user, password=env.remote_password, - hostname=env.server_hostname, port=env.server_port, remote=runs_remotely) + copy_key_to_server( + public_key_filename=id_file.public, + username=env.remote_user, + password=env.remote_password, + hostname=env.server_hostname, + port=env.server_port, + remote=runs_remotely, + ) with wrapper.connect(key_filename=id_file.private): pass delete_key_from_server(