From e2defffffe87f6f23de85d027a8ad2430d802ef8 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Wed, 9 Feb 2022 13:02:48 -0500 Subject: [PATCH 01/16] stub job for testing Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 236c354..6dd79a0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,12 +5,15 @@ variables: stages: - build - static_analysis + - test +# Job setup before_script: - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines # get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} +# Job teardown after_script: - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs @@ -29,3 +32,12 @@ flake8: - docker run ${IMAGE_NAMETAG} bash -c "flake8 ./django_remote_submission ./tests" # entrypoint is ${INSTALL_DIR} tags: - dev + +# Tests that do not require a remote worker +noremote: + state: test + script: + - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry + - echo "reserved for testing" + tags: + - dev -- GitLab From 52dc20295b9bb29ffc13259f30ddb384065db36c Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Wed, 9 Feb 2022 13:08:01 -0500 Subject: [PATCH 02/16] fix silly typo Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6dd79a0..cb6ce42 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,7 @@ flake8: # Tests that do not require a remote worker noremote: - state: test + stage: test script: - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry - echo "reserved for testing" -- GitLab From ece4be7def291504671649385d3517949bf555ef Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Wed, 9 Feb 2022 13:09:23 -0500 Subject: [PATCH 03/16] pytest-cov already requires coverage Signed-off-by: Jose Borreguero --- conda.environment/environment-dev.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/conda.environment/environment-dev.yml b/conda.environment/environment-dev.yml index 85f3c42..5b8c5a4 100644 --- a/conda.environment/environment-dev.yml +++ b/conda.environment/environment-dev.yml @@ -24,7 +24,6 @@ dependencies: # from here on packages strictly for testing - black - bumpversion - - coverage - flake8 - mock - mypy -- GitLab From e4e685ce47e73edfb48ae192704c8b3bd3ad9ad4 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Wed, 9 Feb 2022 21:42:33 -0500 Subject: [PATCH 04/16] set django version same as that of web_reflectivity Signed-off-by: Jose Borreguero --- conda.environment/environment-dev.yml | 2 +- conda.environment/environment.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda.environment/environment-dev.yml b/conda.environment/environment-dev.yml index 5b8c5a4..2b2454a 100644 --- a/conda.environment/environment-dev.yml +++ b/conda.environment/environment-dev.yml @@ -7,7 +7,7 @@ dependencies: - python=3.8 - asgiref - celery - - django + - django=3.2 - django-channels - django-environ - django-filter diff --git a/conda.environment/environment.yml b/conda.environment/environment.yml index fb57c4b..b6f7a51 100644 --- a/conda.environment/environment.yml +++ b/conda.environment/environment.yml @@ -9,7 +9,7 @@ dependencies: - six - paramiko - celery - - django + - django=3.2 - django-filter - django-model-utils - djangorestframework -- GitLab From 91ae435aadf0b0935c40bf66b4d6ee0b6d946d21 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Wed, 9 Feb 2022 21:43:16 -0500 Subject: [PATCH 05/16] add no-remote test in CI invoking a make target Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 2 +- Dockerfile | 2 +- Makefile | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cb6ce42..96d5c7a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,6 +38,6 @@ noremote: stage: test script: - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry - - echo "reserved for testing" + - docker run ${IMAGE_NAMETAG} bash -c "make test-noremote" # invoke pytest with coverage tags: - dev diff --git a/Dockerfile b/Dockerfile index 0471cf6..27c9546 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ COPY . . # This line will throw error if using an old miniconda3 image. # To correct, one must build a miniconda3 image while piling all apt-get statements together into one # https://hub.docker.com/r/continuumio/miniconda3/dockerfile -RUN apt-get update && apt-get install -y vim emacs less mlocate iputils-ping net-tools openssh-client openssh-server +RUN apt-get update && apt-get install -y emacs iputils-ping less make mlocate net-tools openssh-client openssh-server vim RUN conda install --name base -c conda-forge mamba RUN mamba env update --name base --file conda.environment/environment-dev.yml diff --git a/Makefile b/Makefile index a68202f..d6dc47a 100644 --- a/Makefile +++ b/Makefile @@ -34,8 +34,8 @@ doclint: ## check documentation style with flake8 test: ## run tests quickly with the default Python pytest -v -test-ci: ## run tests skipping those requiring a remote worker - pytest -m "not remote_required" -v +test-noremote: ## run tests skipping those requiring a remote worker + pytest -v -m "not remote_required" --cov=django_remote_submission tests/ test-all: ## run tests on every Python version with tox tox -- GitLab From fa4b92fcf5e6973dee2b8d42069c9c23d49a724d Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Thu, 10 Feb 2022 08:36:18 -0500 Subject: [PATCH 06/16] HTML coverage report as artifact Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 20 +++++++++++--------- Makefile | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 96d5c7a..93c53f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,15 +7,14 @@ stages: - static_analysis - test -# Job setup -before_script: - - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines -# get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) - - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} - -# Job teardown -after_script: - - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs +# "setup" and "teardown" for all jobs +default: + before_script: + - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines + # get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) + - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} + after_script: + - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs dockerbuild: stage: build @@ -39,5 +38,8 @@ noremote: script: - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry - docker run ${IMAGE_NAMETAG} bash -c "make test-noremote" # invoke pytest with coverage + artifacts: + paths: + - htmlcov # coverage report htmlcov/index.html as an artifact tags: - dev diff --git a/Makefile b/Makefile index d6dc47a..985cde7 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ test: ## run tests quickly with the default Python pytest -v test-noremote: ## run tests skipping those requiring a remote worker - pytest -v -m "not remote_required" --cov=django_remote_submission tests/ + pytest -v -m "not remote_required" --cov-report html --cov-report term --cov=django_remote_submission tests/ test-all: ## run tests on every Python version with tox tox -- GitLab From 0321ccb94b92b9a7ad4970c8aa0d7ab094588adc Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Thu, 10 Feb 2022 08:58:20 -0500 Subject: [PATCH 07/16] register pytest markings to avoid warnings Signed-off-by: Jose Borreguero --- pytest.ini | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 5c60171..49da52b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,8 @@ [pytest] +markers = + remote_required: test requiring a remote worker for fitting + job_program: script to be run remotely (as a string) testpaths = tests - -# Avoid reading-in django_remote_submission/migrations/0001_initial.py when we're modifying/adding models +# Avoid reading-in migrations/ when modifying or adding models # https://pytest-django.readthedocs.io/en/latest/database.html#no-migrations-disable-django-migrations addopts = --no-migrations -- GitLab From d26c46b89cc90ad7cc018d546949f58bc997fdd9 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Thu, 10 Feb 2022 09:09:55 -0500 Subject: [PATCH 08/16] docker volume for the coverage output Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 93c53f4..2826ed1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,9 +37,11 @@ noremote: stage: test script: - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry - - docker run ${IMAGE_NAMETAG} bash -c "make test-noremote" # invoke pytest with coverage + # pytest with coverage, creating a volume to retrieve the coverage output from the container + - docker run -v $PWD/htmlcov:${INSTALL_DIR}/htmlcov ${IMAGE_NAMETAG} bash -c "make test-noremote" artifacts: paths: - - htmlcov # coverage report htmlcov/index.html as an artifact + - htmlcov/ # coverage report htmlcov/index.html as an artifact + expire_in: 1 week # remove artifacts for this Job after one week tags: - dev -- GitLab From 0fdec7cd6188bfb6ba129fb68cb26e140e9a3b3d Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Thu, 10 Feb 2022 10:04:07 -0500 Subject: [PATCH 09/16] use commit slug instead of name Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2826ed1..09a40b9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ variables: INSTALL_DIR: "/opt/django_remote_submission" - IMAGE_NAMETAG: "code.ornl.gov:4567/reflectometry/django-remote-submission/${CI_COMMIT_REF_NAME}" + IMAGE_NAMETAG: "code.ornl.gov:4567/reflectometry/django-remote-submission/${CI_COMMIT_REF_SLUG}" stages: - build -- GitLab From 13a3e9cd6cf32a837483e84b7509ca059a47e4f0 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Thu, 10 Feb 2022 10:33:38 -0500 Subject: [PATCH 10/16] try XML coverage Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 6 ++++-- Makefile | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 09a40b9..9069714 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,10 +38,12 @@ noremote: script: - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry # pytest with coverage, creating a volume to retrieve the coverage output from the container - - docker run -v $PWD/htmlcov:${INSTALL_DIR}/htmlcov ${IMAGE_NAMETAG} bash -c "make test-noremote" + - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "make test-noremote" artifacts: paths: - - htmlcov/ # coverage report htmlcov/index.html as an artifact + - artifacts/htmlcov/ # coverage report htmlcov/index.html + reports: + cobertura: artifacts/coverage.xml expire_in: 1 week # remove artifacts for this Job after one week tags: - dev diff --git a/Makefile b/Makefile index 985cde7..27d7dfd 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ test: ## run tests quickly with the default Python pytest -v test-noremote: ## run tests skipping those requiring a remote worker - pytest -v -m "not remote_required" --cov-report html --cov-report term --cov=django_remote_submission tests/ + pytest -v -m "not remote_required" --cov-report html:artifacts/htmlcov --cov-report term --cov-report xml:artifacts/coverage.xml --cov=django_remote_submission tests/ test-all: ## run tests on every Python version with tox tox -- GitLab From 7a5695ca937c5b380978a5b6439b9e698dea1c17 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Thu, 10 Feb 2022 11:58:31 -0500 Subject: [PATCH 11/16] some more docs Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9069714..682ce3e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,9 +41,9 @@ noremote: - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "make test-noremote" artifacts: paths: - - artifacts/htmlcov/ # coverage report htmlcov/index.html + - artifacts/htmlcov/ # detailed coverage report htmlcov/index.html reports: - cobertura: artifacts/coverage.xml + cobertura: artifacts/coverage.xml # show coverage changes in GitLab's merge-request diff view expire_in: 1 week # remove artifacts for this Job after one week tags: - dev -- GitLab From d4a5ad250197dafcbe8e1855d030c5559b5f9b8d Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Thu, 10 Feb 2022 12:06:25 -0500 Subject: [PATCH 12/16] environment file available in CI Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 682ce3e..43269a8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ default: - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines # get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} + - mv .env.ci .env after_script: - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs -- GitLab From a93e707a108b833eca010f2dcb10bf604365d712 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Thu, 10 Feb 2022 12:08:16 -0500 Subject: [PATCH 13/16] the .env file should be created only for the job that generates the image Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 43269a8..0a6defa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,13 +13,13 @@ default: - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines # get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} - - mv .env.ci .env after_script: - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs dockerbuild: stage: build script: + - mv .env.ci .env - docker build -t ${IMAGE_NAMETAG} . # create image with default Dockerfile - docker push ${IMAGE_NAMETAG} # upload to the Container Registry tags: -- GitLab From 20d3a06c1147c8f0f50eac036bd46adf1ab96ac3 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Thu, 10 Feb 2022 12:10:39 -0500 Subject: [PATCH 14/16] remame environment file for CI Signed-off-by: Jose Borreguero --- .env.travis => .env.ci | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .env.travis => .env.ci (100%) diff --git a/.env.travis b/.env.ci similarity index 100% rename from .env.travis rename to .env.ci -- GitLab From 4d970b189368345edf6ffb6c64bb695bf2764525 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Thu, 10 Feb 2022 12:29:55 -0500 Subject: [PATCH 15/16] suppres not useful warning Signed-off-by: Jose Borreguero --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest.ini b/pytest.ini index 49da52b..0fb1e03 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,3 +6,5 @@ testpaths = tests # Avoid reading-in migrations/ when modifying or adding models # https://pytest-django.readthedocs.io/en/latest/database.html#no-migrations-disable-django-migrations addopts = --no-migrations +filterwarnings = + ignore::django.utils.deprecation.RemovedInDjango20Warning -- GitLab From 0422363a7f0c69f7180031ea2f19bf466014ad1f Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Thu, 10 Feb 2022 13:05:02 -0500 Subject: [PATCH 16/16] Fix the django version in the warning to supress Signed-off-by: Jose Borreguero --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 0fb1e03..85be8d5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,4 +7,4 @@ testpaths = tests # https://pytest-django.readthedocs.io/en/latest/database.html#no-migrations-disable-django-migrations addopts = --no-migrations filterwarnings = - ignore::django.utils.deprecation.RemovedInDjango20Warning + ignore::django.utils.deprecation.RemovedInDjango40Warning -- GitLab