From 5eb0dcb1c9fd5986d45ddc6615be91b89158c217 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Thu, 17 Feb 2022 10:32:18 -0500 Subject: [PATCH 01/24] add test-remote target in Makefile Signed-off-by: Jose Borreguero --- Makefile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index fb8bef1..b342f95 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean-pyc clean-build condadev docs help +.PHONY: clean-pyc clean-build condadev docs test-noremote test-remote help SHELL := bash .DEFAULT_GOAL := help define BROWSER_PYSCRIPT @@ -36,12 +36,15 @@ clean-pyc: ## remove Python file artifacts doclint: ## check documentation style with flake8 flake8 --select=D --ignore=F django_remote_submission -test: ## run tests quickly with the default Python - pytest -v - test-noremote: ## run tests skipping those requiring a remote worker pytest -v -m "not remote_required" --cov-report html:artifacts/htmlcov --cov-report term --cov-report xml:artifacts/coverage.xml --cov=django_remote_submission tests/ +test-remote: ## run tests skipping those requiring a remote worker + pytest -v -m "remote_required" --cov-report html:artifacts/htmlcov_r --cov-report term --cov-report xml:artifacts/coverage.xml --cov=django_remote_submission tests/ + +test: ## run tests quickly with the default Python + pytest -v --cov-report html:artifacts/htmlcov --cov-report term --cov-report xml:artifacts/coverage.xml --cov=django_remote_submission tests/ + coverage: ## check code coverage quickly with the default Python py.test --cov=django_remote_submission tests/ -- GitLab From 718bafd6b77c14ca2dfa8b446f0db2b7877fcdc5 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 18 Feb 2022 07:52:07 -0500 Subject: [PATCH 02/24] fix spelling typo Signed-off-by: Jose Borreguero --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 3a7bb7a..ee46f2b 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -156,6 +156,6 @@ create a conda environment, although a virtualenv should be equally acceptable 5. Fire up the conda environment and invoke pytest:: - $ conda activate djroms + $ conda activate djrems (djrems) pytest -v ./tests # run all tests (djrems) pytest -v ./tests/test_tasks.py::test_deploy_and_delete_key # singe test function -- GitLab From f04399f858780bbff5eed372885d5f68c5f0ddcf Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 18 Feb 2022 07:52:56 -0500 Subject: [PATCH 03/24] specify remote user in environment file Signed-off-by: Jose Borreguero --- .env.ci | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.env.ci b/.env.ci index d80e981..5303180 100644 --- a/.env.ci +++ b/.env.ci @@ -20,7 +20,7 @@ # TEST_SERVER_HOSTNAME=192.168.0.1 # TEST_SERVER_HOSTNAME=example.com # -TEST_SERVER_HOSTNAME=foobar.invalid +TEST_SERVER_HOSTNAME=djrems # The SSH port of the server to connect to. This will almost always be port 22, # but it could be different. @@ -37,7 +37,7 @@ TEST_SERVER_PORT=22 # Example(s): # TEST_REMOTE_USER=johnsmith # -TEST_REMOTE_USER=foo +TEST_REMOTE_USER=testuser # The remote user's password. This will be used for initial connections during # the tests as well as testing the functionality of the public key transfers. @@ -46,7 +46,7 @@ TEST_REMOTE_USER=foo # Example(s): # TEST_REMOTE_PASSWORD=p4ssw0rd # -TEST_REMOTE_PASSWORD=bar +TEST_REMOTE_PASSWORD=p4ssw0rd # The remote directory to store scripts in. This directory should already exist # as the tests won't create it automatically. @@ -69,7 +69,7 @@ TEST_REMOTE_DIRECTORY=/tmp/ # Example(s): # TEST_REMOTE_FILENAME=foobar.py # -TEST_REMOTE_FILENAME=foobar.py +TEST_REMOTE_FILENAME=jobtest.py # The path to the Python interpreter to use. If the path is not absolute, the # $PATH variable will be searched to find the right executable. This can be -- GitLab From 386a2cfb2e7d24f27332bf14b63f5705d26ff814 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 18 Feb 2022 07:55:46 -0500 Subject: [PATCH 04/24] enable conda environment over ssh remote jobs connection Signed-off-by: Jose Borreguero --- Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Dockerfile b/Dockerfile index 27c9546..193403b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,3 +18,10 @@ RUN apt-get update && apt-get install -y emacs iputils-ping less make mlocate ne RUN conda install --name base -c conda-forge mamba RUN mamba env update --name base --file conda.environment/environment-dev.yml + +# Will start conda environment when $HOME/.bashrc is invoked +# Critical to prepend .bashrc, otherwise the conda environment +# will not be activated when executing commands via remote SSH +RUN echo "$(echo "conda activate base"; cat /etc/skel/.bashrc)" > /etc/skel/.bashrc +RUN echo "$(echo "source /etc/profile.d/conda.sh"; cat /etc/skel/.bashrc)" > /etc/skel/.bashrc +RUN echo "$(echo "# Activate Conda Environment base"; cat /etc/skel/.bashrc)" > /etc/skel/.bashrc -- GitLab From 7e4fe16855cb9edf0ecca0c10c9c9a05beb44e42 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 18 Feb 2022 07:56:38 -0500 Subject: [PATCH 05/24] docker compose to set up the "remote" worker Signed-off-by: Jose Borreguero --- docker-compose.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6f1d20d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,17 @@ +#use for local development + +version: '3' + +services: + + djrems: + restart: always + image: djrems + hostname: djrems + expose: + - "22" + env_file: + - .env + volumes: + - artifacts:/opt/django_remote_submission/artifacts + command: /opt/django_remote_submission/docker-entrypoint.sh -- GitLab From a25e7716f076ee3c19a81d69edeebc10717bb3a2 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 18 Feb 2022 07:57:20 -0500 Subject: [PATCH 06/24] docker entry point to add a test user and run the remote tests Signed-off-by: Jose Borreguero --- docker-entrypoint.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 docker-entrypoint.sh diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..f781d76 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e # stop and exit script if any command or pipeline returns + +# crate account for test user +if id "${TEST_REMOTE_USER}" &>/dev/null; then + echo "${TEST_REMOTE_PASSWORD} already exists" +else + useradd -m -p $(openssl passwd -crypt ${TEST_REMOTE_PASSWORD}) -s /bin/bash -G users ${TEST_REMOTE_USER} +fi + +# start the ssh daemon +/etc/init.d/ssh start + +# run remote tests +make test-remote -- GitLab From e12fe7a1ae68dd9356e8f7aeeae151a3386be896 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 18 Feb 2022 07:57:38 -0500 Subject: [PATCH 07/24] new test jobs in CI for remote tests Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8b38159..3269305 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ variables: stages: - static_analysis - build + - testremote - test - deploy @@ -63,6 +64,29 @@ dockerbuild: tags: - dev # identify which runners can run this job +# Tests that do not require a remote worker +remote: + stage: testremote + before_script: + - *docker-setup + script: + - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry + - docker tag ${IMAGE_NAMETAG} djrems # retag the image + - mv .env.ci .env + - docker-compose up -d + # pytest with coverage, creating a volume to retrieve the coverage output from the container + - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "make test-remote" + after_script: + - *docker-teardown + artifacts: + paths: + - artifacts/htmlcov/ # detailed coverage report htmlcov/index.html + reports: + cobertura: artifacts/coverage.xml # show coverage changes in GitLab's merge-request diff view + expire_in: 1 week # remove artifacts for this Job after one week + tags: + - dev + # Tests that do not require a remote worker noremote: stage: test -- GitLab From 34857b9506840fb2ace6fe8409a081ad431c46e6 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 18 Feb 2022 08:40:53 -0500 Subject: [PATCH 08/24] download docker-compose Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3269305..c1e3db1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,6 +47,8 @@ before_script: - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines # get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} + - curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o ./docker-compose + - chmod +x ./docker-compose after_script: - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs @@ -73,7 +75,7 @@ remote: - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry - docker tag ${IMAGE_NAMETAG} djrems # retag the image - mv .env.ci .env - - docker-compose up -d + - ./docker-compose up -d # pytest with coverage, creating a volume to retrieve the coverage output from the container - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "make test-remote" after_script: -- GitLab From eeb2b8cd242238da2d0ae54fad1b0c44154bd8f3 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 18 Feb 2022 09:09:49 -0500 Subject: [PATCH 09/24] download docker-compose Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c1e3db1..cff047f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,9 @@ stages: - docker system prune -f -a --volumes # get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} + # download docker-compose executable + - curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o ./docker-compose + - chmod +x ./docker-compose .docker-teardown: &docker-teardown - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs @@ -47,8 +50,6 @@ before_script: - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines # get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} - - curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o ./docker-compose - - chmod +x ./docker-compose after_script: - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs -- GitLab From d2a5fde3b69229fb375074f8b0c4c0eec135f26d Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 18 Feb 2022 09:26:24 -0500 Subject: [PATCH 10/24] declaration of "artifacts" in the volumes section Signed-off-by: Jose Borreguero --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 6f1d20d..8aaaf0f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,3 +15,6 @@ services: volumes: - artifacts:/opt/django_remote_submission/artifacts command: /opt/django_remote_submission/docker-entrypoint.sh + +volumes: + artifacts: -- GitLab From 25d21a644a37f9ea35bf83bcc07a3eb158bdfcd5 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 20 Feb 2022 09:07:44 -0500 Subject: [PATCH 11/24] clean up unused instructions Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cff047f..e86bfe5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,14 +46,6 @@ flake8: tags: - dev -before_script: - - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines -# get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) - - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} - -after_script: - - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs - dockerbuild: stage: build before_script: @@ -76,9 +68,8 @@ remote: - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry - docker tag ${IMAGE_NAMETAG} djrems # retag the image - mv .env.ci .env + # create a container where to run pytest of remote tests with coverage - ./docker-compose up -d - # pytest with coverage, creating a volume to retrieve the coverage output from the container - - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "make test-remote" after_script: - *docker-teardown artifacts: -- GitLab From 73bb1c4fe13ad489ca2c8c9e9a3a89f6813980db Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 20 Feb 2022 09:27:46 -0500 Subject: [PATCH 12/24] for local dev purposes, change entrypoint without rebuilding the image Signed-off-by: Jose Borreguero --- docker-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8aaaf0f..b9c462f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,9 @@ services: - .env volumes: - artifacts:/opt/django_remote_submission/artifacts - command: /opt/django_remote_submission/docker-entrypoint.sh + # DEBUG remove this mount and change command: to /opt/django_remote_submission/docker-entrypoint.sh + - ./docker-entrypoint.sh:/usr/bin/docker-entrypoint.sh + command: /usr/bin/docker-entrypoint.sh volumes: artifacts: -- GitLab From 31675f3241f1022bf55b791bb276b7e4a87582cb Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 20 Feb 2022 09:31:29 -0500 Subject: [PATCH 13/24] openssl >= 3.X deprecated option "-crypt" Signed-off-by: Jose Borreguero --- docker-entrypoint.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index f781d76..15a9bb7 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,15 +1,22 @@ #!/bin/bash set -e # stop and exit script if any command or pipeline returns -# crate account for test user +echo "OPENSSL VERSION = $(openssl version)" + +# create account for test user if id "${TEST_REMOTE_USER}" &>/dev/null; then echo "${TEST_REMOTE_PASSWORD} already exists" else - useradd -m -p $(openssl passwd -crypt ${TEST_REMOTE_PASSWORD}) -s /bin/bash -G users ${TEST_REMOTE_USER} + useradd -m -p $(openssl passwd -1 ${TEST_REMOTE_PASSWORD}) -s /bin/bash -G users ${TEST_REMOTE_USER} fi # start the ssh daemon /etc/init.d/ssh start # run remote tests -make test-remote +# DEBUG uncomment this once debugging's done +make test + +# hack preventing container from exiting +# DEBUG comment this once debugging's done +#tail -f /dev/null -- GitLab From a65b83d1c92ad48b67ca2551c636682ba6d840fb Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 20 Feb 2022 09:31:58 -0500 Subject: [PATCH 14/24] perform remote and no-remote tests Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e86bfe5..a4cd207 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,6 @@ variables: stages: - static_analysis - build - - testremote - test - deploy @@ -60,8 +59,8 @@ dockerbuild: - dev # identify which runners can run this job # Tests that do not require a remote worker -remote: - stage: testremote +test: + stage: test before_script: - *docker-setup script: @@ -82,24 +81,24 @@ remote: - dev # Tests that do not require a remote worker -noremote: - stage: test - before_script: - - *docker-setup - script: - - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry - # pytest with coverage, creating a volume to retrieve the coverage output from the container - - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "make test-noremote" - after_script: - - *docker-teardown - artifacts: - paths: - - artifacts/htmlcov/ # detailed coverage report htmlcov/index.html - reports: - cobertura: artifacts/coverage.xml # show coverage changes in GitLab's merge-request diff view - expire_in: 1 week # remove artifacts for this Job after one week - tags: - - dev +#noremote: +# stage: test +# before_script: +# - *docker-setup +# script: +# - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry +# # pytest with coverage, creating a volume to retrieve the coverage output from the container +# - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "make test-noremote" +# after_script: +# - *docker-teardown +# artifacts: +# paths: +# - artifacts/htmlcov/ # detailed coverage report htmlcov/index.html +# reports: +# cobertura: artifacts/coverage.xml # show coverage changes in GitLab's merge-request diff view +# expire_in: 1 week # remove artifacts for this Job after one week +# tags: +# - dev pythonwheel: stage: deploy -- GitLab From a7860ac1655158182ff55864779cbb301cc1582f Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 20 Feb 2022 09:47:53 -0500 Subject: [PATCH 15/24] remove DEBUG comments and instructions Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 20 -------------------- Makefile | 2 +- docker-compose.yml | 4 +--- docker-entrypoint.sh | 5 ++--- 4 files changed, 4 insertions(+), 27 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a4cd207..e1335dd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -80,26 +80,6 @@ test: tags: - dev -# Tests that do not require a remote worker -#noremote: -# stage: test -# before_script: -# - *docker-setup -# script: -# - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry -# # pytest with coverage, creating a volume to retrieve the coverage output from the container -# - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "make test-noremote" -# after_script: -# - *docker-teardown -# artifacts: -# paths: -# - artifacts/htmlcov/ # detailed coverage report htmlcov/index.html -# reports: -# cobertura: artifacts/coverage.xml # show coverage changes in GitLab's merge-request diff view -# expire_in: 1 week # remove artifacts for this Job after one week -# tags: -# - dev - pythonwheel: stage: deploy only: diff --git a/Makefile b/Makefile index b342f95..e456510 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ test-noremote: ## run tests skipping those requiring a remote worker test-remote: ## run tests skipping those requiring a remote worker pytest -v -m "remote_required" --cov-report html:artifacts/htmlcov_r --cov-report term --cov-report xml:artifacts/coverage.xml --cov=django_remote_submission tests/ -test: ## run tests quickly with the default Python +test: ## run tests with coverage pytest -v --cov-report html:artifacts/htmlcov --cov-report term --cov-report xml:artifacts/coverage.xml --cov=django_remote_submission tests/ coverage: ## check code coverage quickly with the default Python diff --git a/docker-compose.yml b/docker-compose.yml index b9c462f..8aaaf0f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,9 +14,7 @@ services: - .env volumes: - artifacts:/opt/django_remote_submission/artifacts - # DEBUG remove this mount and change command: to /opt/django_remote_submission/docker-entrypoint.sh - - ./docker-entrypoint.sh:/usr/bin/docker-entrypoint.sh - command: /usr/bin/docker-entrypoint.sh + command: /opt/django_remote_submission/docker-entrypoint.sh volumes: artifacts: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 15a9bb7..d3642c1 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -14,9 +14,8 @@ fi /etc/init.d/ssh start # run remote tests -# DEBUG uncomment this once debugging's done make test # hack preventing container from exiting -# DEBUG comment this once debugging's done -#tail -f /dev/null +# uncomment only for debugging purposes +# tail -f /dev/null -- GitLab From c420c475b218ac3b611edaa704f400155b98ae17 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 20 Feb 2022 09:50:12 -0500 Subject: [PATCH 16/24] add "test" and "coverage" to the list of PHONY targets Signed-off-by: Jose Borreguero --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e456510..6e4577c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean-pyc clean-build condadev docs test-noremote test-remote help +.PHONY: clean-pyc clean-build condadev coverage docs test-noremote test-remote test elp SHELL := bash .DEFAULT_GOAL := help define BROWSER_PYSCRIPT -- GitLab From 7f64152fad127a61aef4472fed6380f95806a2bd Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 20 Feb 2022 10:15:58 -0500 Subject: [PATCH 17/24] DEBUG printouts to verify we're running the tests Signed-off-by: Jose Borreguero --- docker-entrypoint.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index d3642c1..4de08cd 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -14,8 +14,22 @@ fi /etc/init.d/ssh start # run remote tests + +# DEBUG delete these printouts +echo "##########################" +echo "##########################" +echo " ABOUT TO RUN THE TESTS" +echo "##########################" +echo "##########################" + make test +echo "##########################" +echo "##########################" +echo " TESTS HAVE RUN" +echo "##########################" +echo "##########################" + # hack preventing container from exiting # uncomment only for debugging purposes # tail -f /dev/null -- GitLab From 4467404a32602b26e8a9b0845a7da3d9514e5771 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 20 Feb 2022 10:16:20 -0500 Subject: [PATCH 18/24] Make target to build the local dev image Signed-off-by: Jose Borreguero --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6e4577c..7b28cbc 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean-pyc clean-build condadev coverage docs test-noremote test-remote test elp +.PHONY: clean-pyc clean-build condadev coverage dockerbuilddev docs test-noremote test-remote test help SHELL := bash .DEFAULT_GOAL := help define BROWSER_PYSCRIPT @@ -80,3 +80,7 @@ condadev: ## create conda environment for developing, and install pre-commit hoo conda install -y --name djrems -c conda-forge mamba mamba env update -y --name djrems --file ./conda.environment/environment-dev.yml $(CONDA_ACTIVATE) djrems; python setup.py develop; pre-commit install + +dockerbuilddev: ## create docker image for testing + /bin/cp .env.ci .env + docker build --network host -t djrems . -- GitLab From dd80d36cb5952dc63aa050e9bcdfb1a59f68818f Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 20 Feb 2022 10:51:10 -0500 Subject: [PATCH 19/24] don't run in the background, otherwise pytest messages won't show Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e1335dd..ac5bccd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -68,7 +68,7 @@ test: - docker tag ${IMAGE_NAMETAG} djrems # retag the image - mv .env.ci .env # create a container where to run pytest of remote tests with coverage - - ./docker-compose up -d + - ./docker-compose up after_script: - *docker-teardown artifacts: -- GitLab From c861c4d2d56b2fca1449cdf74c690a6b178caaec Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 20 Feb 2022 11:18:27 -0500 Subject: [PATCH 20/24] bugfix, use the temp sshdir Signed-off-by: Jose Borreguero --- tests/test_tasks.py | 169 +++++++++++++++++++++++++------------------- 1 file changed, 97 insertions(+), 72 deletions(-) diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 5d6861f..e108f55 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -30,17 +30,30 @@ def pairwise(iterable): return zip(a, b) -EnvBase = collections.namedtuple('Env', [ - 'server_hostname', 'server_port', 'remote_directory', 'remote_filename', - 'remote_user', 'remote_password', 'python_path', 'python_arguments', -]) +EnvBase = collections.namedtuple( + 'Env', + [ + 'server_hostname', + 'server_port', + 'remote_directory', + 'remote_filename', + 'remote_user', + 'remote_password', + 'python_path', + 'python_arguments', + ], +) class Env(EnvBase): def __repr__(self): - return super(Env, self).__repr__().replace( - 'remote_password={!r}'.format(self.remote_password), - 'remote_password={!r}'.format('******'), + return ( + super(Env, self) + .__repr__() + .replace( + 'remote_password={!r}'.format(self.remote_password), + 'remote_password={!r}'.format('******'), + ) ) @@ -166,11 +179,11 @@ def job_model_saved(mocker): @pytest.fixture(params=[True, False], ids=["Remote", "Local"]) def runs_remotely(request): - ''' + """ params == True: Uses RemoteWrapper params == False: Uses LocalWrapper if it's running with command line marker -m "not remote_required" skip it - ''' + """ skipif_remote_required = "not remote_required" in request.config.getoption('-m') if not request.param: @@ -184,13 +197,15 @@ def runs_remotely(request): @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time for i in range(5): print("line: {}".format(i)) time.sleep(0.1) -''') +''' +) def test_submit_job_normal_usage(env, job, job_model_saved, runs_remotely): from django_remote_submission.models import Job, Log from django_remote_submission.tasks import submit_job_to_server @@ -216,15 +231,16 @@ def test_submit_job_normal_usage(env, job, job_model_saved, runs_remotely): @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ hello world -''') +''' +) def test_copy_job(env, job, job_model_saved, runs_remotely): from django_remote_submission.models import Job, Log from django_remote_submission.tasks import copy_job_to_server - results = copy_job_to_server( - job.pk, env.remote_password, remote=runs_remotely - ) + + results = copy_job_to_server(job.pk, env.remote_password, remote=runs_remotely) assert len(results) == 0 @@ -239,14 +255,16 @@ def test_copy_job(env, job, job_model_saved, runs_remotely): @pytest.mark.remote_required @pytest.mark.parametrize("runs_remotely", ["Remote"]) @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time import sys for i in range(5): print("line: {}".format(i), file=sys.stdout if i % 2 == 0 else sys.stderr) time.sleep(0.1) -''') +''' +) def test_submit_job_multiple_streams(env, job, runs_remotely): ''' This one cannot run in local as the Live Log is not real ''' from django_remote_submission.models import Log @@ -272,10 +290,12 @@ def test_submit_job_multiple_streams(env, job, runs_remotely): @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ import sys sys.exit(1) -''') +''' +) def test_submit_job_failure(env, job, runs_remotely): from django_remote_submission.models import Job from django_remote_submission.tasks import submit_job_to_server @@ -287,20 +307,21 @@ def test_submit_job_failure(env, job, runs_remotely): @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time import sys for i in range(5): print('line: {}'.format(i), file=sys.stdout) time.sleep(0.1) -''') +''' +) def test_submit_job_log_policy_log_total(env, job, runs_remotely): from django_remote_submission.models import Log from django_remote_submission.tasks import submit_job_to_server, LogPolicy - submit_job_to_server(job.pk, env.remote_password, remote=runs_remotely, - log_policy=LogPolicy.LOG_TOTAL) + submit_job_to_server(job.pk, env.remote_password, remote=runs_remotely, log_policy=LogPolicy.LOG_TOTAL) assert Log.objects.count() == 1 log = Log.objects.get() @@ -309,41 +330,42 @@ def test_submit_job_log_policy_log_total(env, job, runs_remotely): @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time import sys for i in range(5): print('line: {}'.format(i), file=sys.stdout) time.sleep(0.1) -''') +''' +) def test_submit_job_log_policy_log_none(env, job, runs_remotely): from django_remote_submission.models import Log from django_remote_submission.tasks import submit_job_to_server, LogPolicy - submit_job_to_server(job.pk, env.remote_password, remote=runs_remotely, - log_policy=LogPolicy.LOG_NONE) + submit_job_to_server(job.pk, env.remote_password, remote=runs_remotely, log_policy=LogPolicy.LOG_NONE) assert Log.objects.count() == 0 @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time import sys for i in range(5): print('line: {}'.format(i)) time.sleep(0.35) -''') +''' +) def test_submit_job_timeout(env, job, runs_remotely): from django_remote_submission.models import Job, Log from django_remote_submission.tasks import submit_job_to_server import datetime - submit_job_to_server(job.pk, env.remote_password, - remote=runs_remotely, - timeout=datetime.timedelta(seconds=1)) + submit_job_to_server(job.pk, env.remote_password, remote=runs_remotely, timeout=datetime.timedelta(seconds=1)) assert Log.objects.count() == 3 @@ -352,7 +374,8 @@ def test_submit_job_timeout(env, job, runs_remotely): @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time import sys @@ -360,35 +383,32 @@ for i in range(5): with open('{}.txt'.format(i), 'w') as f: print('line: {}'.format(i), file=f) time.sleep(0.1) -''') +''' +) def test_submit_job_modified_files(env, job, runs_remotely): from django_remote_submission.models import Result from django_remote_submission.tasks import submit_job_to_server import re - results = submit_job_to_server(job.pk, env.remote_password, - remote=runs_remotely) + results = submit_job_to_server(job.pk, env.remote_password, remote=runs_remotely) assert len(results) == 5 - assert sorted(results.keys()) == \ - ['0.txt', '1.txt', '2.txt', '3.txt', '4.txt'] + assert sorted(results.keys()) == ['0.txt', '1.txt', '2.txt', '3.txt', '4.txt'] for (result_fname, result_pk) in results.items(): result = Result.objects.get(pk=result_pk) i = int(re.match(r'^([0-9])\.txt', result_fname).group(1)) - assert result.local_file.read().decode('utf-8') == \ - 'line: {}\n'.format(i) + assert result.local_file.read().decode('utf-8') == 'line: {}\n'.format(i) - matcher = re.compile( - r'results/{}/[0-4].txt'.format(job.uuid) - ) + matcher = re.compile(r'results/{}/[0-4].txt'.format(job.uuid)) assert matcher.match(result.local_file.name) is not None @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time import sys @@ -396,30 +416,30 @@ for i in range(5): with open('{}.txt'.format(i), 'w') as f: print('line: {}'.format(i), file=f) time.sleep(0.1) -''') +''' +) def test_submit_job_modified_files_positive_pattern(env, job, runs_remotely): from django_remote_submission.models import Result from django_remote_submission.tasks import submit_job_to_server import re - results = submit_job_to_server(job.pk, env.remote_password, - remote=runs_remotely, - store_results=['0.txt', '[12].txt']) + results = submit_job_to_server( + job.pk, env.remote_password, remote=runs_remotely, store_results=['0.txt', '[12].txt'] + ) assert len(results) == 3 - assert sorted(results.keys()) == \ - ['0.txt', '1.txt', '2.txt'] + assert sorted(results.keys()) == ['0.txt', '1.txt', '2.txt'] for (result_fname, result_pk) in results.items(): result = Result.objects.get(pk=result_pk) i = int(re.match(r'^([0-9])\.txt', result_fname).group(1)) - assert result.local_file.read().decode('utf-8') == \ - 'line: {}\n'.format(i) + assert result.local_file.read().decode('utf-8') == 'line: {}\n'.format(i) @pytest.mark.django_db -@pytest.mark.job_program('''\ +@pytest.mark.job_program( + '''\ from __future__ import print_function import time import sys @@ -427,31 +447,29 @@ for i in range(5): with open('{}.txt'.format(i), 'w') as f: print('line: {}'.format(i), file=f) time.sleep(0.1) -''') +''' +) def test_submit_job_modified_files_negative_pattern(env, job, runs_remotely): from django_remote_submission.models import Result from django_remote_submission.tasks import submit_job_to_server import re - results = submit_job_to_server(job.pk, env.remote_password, - remote=runs_remotely, - store_results=['*', '![34].txt']) + results = submit_job_to_server(job.pk, env.remote_password, remote=runs_remotely, store_results=['*', '![34].txt']) assert len(results) == 3 - assert sorted(results.keys()) == \ - ['0.txt', '1.txt', '2.txt'] + assert sorted(results.keys()) == ['0.txt', '1.txt', '2.txt'] for (result_fname, result_pk) in results.items(): result = Result.objects.get(pk=result_pk) i = int(re.match(r'^([0-9])\.txt', result_fname).group(1)) - assert result.local_file.read().decode('utf-8') == \ - 'line: {}\n'.format(i) + assert result.local_file.read().decode('utf-8') == 'line: {}\n'.format(i) @pytest.mark.django_db def test_submit_job_deploy_key(env, job_gen, interpreter_gen, runs_remotely): from django_remote_submission.tasks import submit_job_to_server + try: from shlex import quote as cmd_quote except ImportError: @@ -470,20 +488,21 @@ def test_submit_job_deploy_key(env, job_gen, interpreter_gen, runs_remotely): arguments=['python', '-u'], ) """ - - id_file = IdentityFile(sshdir=Path("/tmp")) # temporary RSA identity file + sshdir = Path("/tmp") + id_file = IdentityFile(sshdir=sshdir) # temporary RSA identity file with open(id_file.public, 'rt') as f: key = f.read().strip() remove_existing_key_job = job_gen( program='''\ - sed -i.bak -e /{key}/d ~/.ssh/authorized_keys - '''.format(key=cmd_quote(key.replace(r'/', r'\/'))), + sed -i.bak -e /{key}/d {sshdir}/authorized_keys + '''.format( + key=cmd_quote(key.replace(r'/', r'\/')), sshdir=str(sshdir) + ), interpreter=sh, ) - submit_job_to_server(remove_existing_key_job.pk, env.remote_password, - remote=runs_remotely) + submit_job_to_server(remove_existing_key_job.pk, env.remote_password, remote=runs_remotely) add_key_job = job_gen( program='''\ @@ -492,8 +511,7 @@ def test_submit_job_deploy_key(env, job_gen, interpreter_gen, runs_remotely): interpreter=sh, ) - submit_job_to_server(add_key_job.pk, env.remote_password, - remote=runs_remotely) + submit_job_to_server(add_key_job.pk, env.remote_password, remote=runs_remotely) @pytest.mark.remote_required @@ -501,6 +519,7 @@ def test_submit_job_deploy_key(env, job_gen, interpreter_gen, runs_remotely): def test_delete_key_old_way(env): from django_remote_submission.wrapper.remote import RemoteWrapper + wrapper = RemoteWrapper( hostname=env.server_hostname, username=env.remote_user, @@ -561,8 +580,14 @@ def test_deploy_and_delete_key(env): ) # delete the key from server passing the private key file for credentials - copy_key_to_server(public_key_filename=id_file.public, username=env.remote_user, password=env.remote_password, - hostname=env.server_hostname, port=env.server_port, remote=runs_remotely) + copy_key_to_server( + public_key_filename=id_file.public, + username=env.remote_user, + password=env.remote_password, + hostname=env.server_hostname, + port=env.server_port, + remote=runs_remotely, + ) with wrapper.connect(key_filename=id_file.private): pass delete_key_from_server( -- GitLab From f112852023b91b8f39a51f9b18b0ec66d47dfe97 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 20 Feb 2022 11:19:13 -0500 Subject: [PATCH 21/24] mount the entrypoint for easier debugging Signed-off-by: Jose Borreguero --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8aaaf0f..0001a1d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,8 +13,9 @@ services: env_file: - .env volumes: + - ./docker-entrypoint.sh:/usr/bin/docker-entrypoint.sh - artifacts:/opt/django_remote_submission/artifacts - command: /opt/django_remote_submission/docker-entrypoint.sh + command: /usr/bin/docker-entrypoint.sh volumes: artifacts: -- GitLab From 028ee55d4c09aa2700220604964b40d8cc25b3cc Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 20 Feb 2022 11:20:33 -0500 Subject: [PATCH 22/24] remote DEBUG printouts Signed-off-by: Jose Borreguero --- docker-entrypoint.sh | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 4de08cd..d3642c1 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -14,22 +14,8 @@ fi /etc/init.d/ssh start # run remote tests - -# DEBUG delete these printouts -echo "##########################" -echo "##########################" -echo " ABOUT TO RUN THE TESTS" -echo "##########################" -echo "##########################" - make test -echo "##########################" -echo "##########################" -echo " TESTS HAVE RUN" -echo "##########################" -echo "##########################" - # hack preventing container from exiting # uncomment only for debugging purposes # tail -f /dev/null -- GitLab From b53efc072e686721c22c6ebe1ef3587995725aca Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 20 Feb 2022 11:56:09 -0500 Subject: [PATCH 23/24] add key before removing it Signed-off-by: Jose Borreguero --- tests/test_tasks.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/test_tasks.py b/tests/test_tasks.py index e108f55..0eefe96 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -490,14 +490,25 @@ def test_submit_job_deploy_key(env, job_gen, interpreter_gen, runs_remotely): """ sshdir = Path("/tmp") id_file = IdentityFile(sshdir=sshdir) # temporary RSA identity file + + # Connect with password drop the key + wrapper = RemoteWrapper( + hostname=env.server_hostname, + username=env.remote_user, + port=env.server_port, + ) + with wrapper.connect(password=env.remote_password): + wrapper.deploy_key_if_it_does_not_exist(id_file.public) + + # delete the key with open(id_file.public, 'rt') as f: key = f.read().strip() remove_existing_key_job = job_gen( program='''\ - sed -i.bak -e /{key}/d {sshdir}/authorized_keys + sed -i.bak -e /{key}/d ~/.ssh/authorized_keys '''.format( - key=cmd_quote(key.replace(r'/', r'\/')), sshdir=str(sshdir) + key=cmd_quote(key.replace(r'/', r'\/')) ), interpreter=sh, ) @@ -511,6 +522,7 @@ def test_submit_job_deploy_key(env, job_gen, interpreter_gen, runs_remotely): interpreter=sh, ) + # run the job using not the key (it was removed) but the password submit_job_to_server(add_key_job.pk, env.remote_password, remote=runs_remotely) @@ -518,8 +530,6 @@ def test_submit_job_deploy_key(env, job_gen, interpreter_gen, runs_remotely): @pytest.mark.django_db def test_delete_key_old_way(env): - from django_remote_submission.wrapper.remote import RemoteWrapper - wrapper = RemoteWrapper( hostname=env.server_hostname, username=env.remote_user, -- GitLab From da8351ad14bfc8288ca8d0a4953d96da251d0f90 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 20 Feb 2022 11:56:48 -0500 Subject: [PATCH 24/24] Make target to erase all images Signed-off-by: Jose Borreguero --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7b28cbc..3d5aa1c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean-pyc clean-build condadev coverage dockerbuilddev docs test-noremote test-remote test help +.PHONY: clean-pyc clean-build condadev coverage dockerbuilddev dockercleanall docs test-noremote test-remote test help SHELL := bash .DEFAULT_GOAL := help define BROWSER_PYSCRIPT @@ -84,3 +84,6 @@ condadev: ## create conda environment for developing, and install pre-commit hoo dockerbuilddev: ## create docker image for testing /bin/cp .env.ci .env docker build --network host -t djrems . + +dockercleanall: ## delete ALL containers, images, and volumes + docker system prune -f -a --volumes -- GitLab