From f60a0410cd701a23fc466a443c3995ddcfdb834d Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 8 Feb 2022 12:09:11 -0500 Subject: [PATCH 01/34] bring docs requirements into the conda environment Signed-off-by: Jose Borreguero --- conda.environment/environment-dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda.environment/environment-dev.yml b/conda.environment/environment-dev.yml index 2b2454a..ec0d3f8 100644 --- a/conda.environment/environment-dev.yml +++ b/conda.environment/environment-dev.yml @@ -24,6 +24,7 @@ dependencies: # from here on packages strictly for testing - black - bumpversion + - coverage - flake8 - mock - mypy -- GitLab From 550504ab301af0b97661b786ffbe7369c86262e9 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 8 Feb 2022 21:14:09 -0500 Subject: [PATCH 02/34] store the docker image in the project's Container Registry Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a6defa..701c9c0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,13 @@ default: after_script: - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs +before_script: + - docker system prune -f -a --volumes # remove containers, images, volumes, networks + - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} + +after_script: + - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs + dockerbuild: stage: build script: -- GitLab From 27e0bcd558cde7c7709ecc4e572fdce159d82a73 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 8 Feb 2022 21:32:29 -0500 Subject: [PATCH 03/34] better documentation for gitlab-ci.yml Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 701c9c0..4d18eb8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,7 +17,8 @@ default: - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs before_script: - - docker system prune -f -a --volumes # remove containers, images, volumes, networks + - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines +# get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} after_script: -- GitLab From d1ee3ac5c8ec301c9f387a696324d832861a36cb Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Thu, 10 Feb 2022 13:42:59 -0500 Subject: [PATCH 04/34] creating a wheel with hardoded package version and archaich setup.py Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d18eb8..2929d3a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,11 +1,13 @@ variables: INSTALL_DIR: "/opt/django_remote_submission" IMAGE_NAMETAG: "code.ornl.gov:4567/reflectometry/django-remote-submission/${CI_COMMIT_REF_SLUG}" + PACKAGE_VERSION: "2.1.0" stages: - build - static_analysis - test + - deploy # "setup" and "teardown" for all jobs default: @@ -56,3 +58,14 @@ noremote: expire_in: 1 week # remove artifacts for this Job after one week tags: - dev + +wheel: + stage: deploy + script: + - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry + - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "python setup.py bdist_wheel --dist-dir artifacts" + artifacts: + paths: + - artifacts/django_remote_submission-${PACKAGE_VERSION}-py2.py3-none-any.whl + tags: + - dev -- GitLab From 323bbc7be54500ed94313da3d83aca1d98573f37 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 13 Feb 2022 14:28:05 -0500 Subject: [PATCH 05/34] updating requirements environment file Signed-off-by: Jose Borreguero --- conda.environment/environment.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/conda.environment/environment.yml b/conda.environment/environment.yml index b6f7a51..70e5d1f 100644 --- a/conda.environment/environment.yml +++ b/conda.environment/environment.yml @@ -4,9 +4,6 @@ channels: - pyplan dependencies: - python=3.8 - - setuptools - - wheel - - six - paramiko - celery - django=3.2 -- GitLab From f4e7cac48435900e931ca8668365c261ea0ccb55 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 13 Feb 2022 14:28:37 -0500 Subject: [PATCH 06/34] updating requirements-dev environment file Signed-off-by: Jose Borreguero --- conda.environment/environment-dev.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conda.environment/environment-dev.yml b/conda.environment/environment-dev.yml index ec0d3f8..9eaa0ab 100644 --- a/conda.environment/environment-dev.yml +++ b/conda.environment/environment-dev.yml @@ -6,6 +6,7 @@ channels: dependencies: - python=3.8 - asgiref + - bump2version - celery - django=3.2 - django-channels @@ -15,9 +16,6 @@ dependencies: - djangorestframework - paramiko - pip - - setuptools - - six - - wheel # from here on packages strictly for documentation - docutils - sphinx @@ -33,4 +31,6 @@ dependencies: - pytest-cov - pytest-django - pytest-mock + - setuptools + - wheel - tox -- GitLab From 83e1ed6235fee311471f515b7755098b9f5c5646 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 13 Feb 2022 14:29:06 -0500 Subject: [PATCH 07/34] move info to setup.cfg Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 2 +- setup.cfg | 47 ++++++++++++++++++++++++++++++++++------ setup.py | 59 +------------------------------------------------- 3 files changed, 42 insertions(+), 66 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2929d3a..8a15ddc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -66,6 +66,6 @@ wheel: - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "python setup.py bdist_wheel --dist-dir artifacts" artifacts: paths: - - artifacts/django_remote_submission-${PACKAGE_VERSION}-py2.py3-none-any.whl + - artifacts/django_remote_submission-${PACKAGE_VERSION}-py3-none-any.whl tags: - dev diff --git a/setup.cfg b/setup.cfg index fa69555..fd79de1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,15 +1,48 @@ +[metadata] +name = django_remote_submission +description = A Django application to manage long running job submission, including starting the job, saving logs, and storing results +long_description = file: README.rst, LICENSE +version = attr: django_remote_submission.__version__ +author = T. Hobson, M. Doucet and R. M. Ferraz Leal +author_email = ferrazlealrm@ornl.gov +url = https://code.ornl.gov/reflectometry/django-remote-submission +keywords = django, ssh, django-remote-submission +license = ISC License +zip_safe = False +classifiers = + Development Status :: 3 - Alpha + Framework :: Django + Framework :: Django :: 3.2 + Intended Audience :: Developers + License :: OSI Approved :: ISC License + Natural Language :: English + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + +[options] +python_requires = >= 3.6 +install_requires = + paramiko + celery + django==3.2 + django-filter + django-model-utils + djangorestframework + django-channels + django-environ + asgiref +include_package_data = True +packages = django_remote_submission + [bumpversion] current_version = 2.1.0 -commit = True -tag = True - -[bumpversion:file:setup.py] +commit = False +tag = False [bumpversion:file:django_remote_submission/__init__.py] -[wheel] -universal = 1 - [flake8] exclude = __pycache__, diff --git a/setup.py b/setup.py index 28f6c4c..36f7fe7 100755 --- a/setup.py +++ b/setup.py @@ -1,61 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -import os -import sys -import pathlib from setuptools import setup -import pkg_resources -version = '2.1.0' - -# load dependencies from file requirements.txt -with pathlib.Path('requirements.txt').open() as requirements_txt: - install_requires = [str(requirement) for requirement in pkg_resources.parse_requirements(requirements_txt)] - -if sys.argv[-1] == 'publish': - try: - import wheel - print("Wheel version: ", wheel.__version__) - except ImportError: - print('Wheel library missing. Please run "pip install wheel"') - sys.exit() - os.system('python setup.py sdist upload') - os.system('python setup.py bdist_wheel upload') - sys.exit() - -if sys.argv[-1] == 'tag': - print("Tagging the version on git:") - os.system("git tag -a %s -m 'version %s'" % (version, version)) - os.system("git push --tags") - sys.exit() - -readme = open('README.rst').read() -history = open('HISTORY.rst').read().replace('.. :changelog:', '') - -setup( - name='django-remote-submission', - version=version, - description="""A Django application to manage long running job submission, including starting the job, saving logs, and storing results.""", - long_description=readme + '\n\n' + history, - author='T. Hobson, M. Doucet and R. M. Ferraz Leal', - author_email='ferrazlealrm@ornl.gov', - url='https://github.com/ornl-ndav/django-remote-submission', - packages=[ - 'django_remote_submission', - ], - include_package_data=True, - install_requires=install_requires, - license="ISCL", - zip_safe=False, - keywords='django-remote-submission', - classifiers=[ - 'Development Status :: 3 - Alpha', - 'Framework :: Django', - 'Framework :: Django :: 2.0', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', - 'Natural Language :: English', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - ], -) +setup() -- GitLab From bac34e5f5908f608a4d8ec771b9211f8adefff6e Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 13 Feb 2022 14:29:36 -0500 Subject: [PATCH 08/34] tentative make command for bump2version Signed-off-by: Jose Borreguero --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 27d7dfd..21f1e40 100644 --- a/Makefile +++ b/Makefile @@ -75,3 +75,6 @@ condadev: ## create conda environment for developing, and install pre-commit hoo conda install --name djrems -c conda-forge mamba mamba env update --name djrems --file ./conda.environment/environment-dev.yml $(CONDA_ACTIVATE) djrems; python setup.py develop; pre-commit install +bump2version: # change version, e.g "make bump2version version=2.3.0" + echo $version + bump2version --new-version $$version -- GitLab From 21de7961833389ccc2067dee4688b8e5d8ce6a6c Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 13 Feb 2022 14:33:48 -0500 Subject: [PATCH 09/34] update setup.cfg Signed-off-by: Jose Borreguero --- setup.cfg | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/setup.cfg b/setup.cfg index fd79de1..a8e12af 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,29 +10,29 @@ keywords = django, ssh, django-remote-submission license = ISC License zip_safe = False classifiers = - Development Status :: 3 - Alpha - Framework :: Django - Framework :: Django :: 3.2 - Intended Audience :: Developers - License :: OSI Approved :: ISC License - Natural Language :: English - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 + Development Status :: 3 - Alpha + Framework :: Django + Framework :: Django :: 3.2 + Intended Audience :: Developers + License :: OSI Approved :: ISC License + Natural Language :: English + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 [options] python_requires = >= 3.6 install_requires = - paramiko - celery - django==3.2 - django-filter - django-model-utils - djangorestframework - django-channels - django-environ - asgiref + paramiko + celery + django==3.2 + django-filter + django-model-utils + djangorestframework + django-channels + django-environ + asgiref include_package_data = True packages = django_remote_submission @@ -49,4 +49,4 @@ exclude = build, dist, migrations -max-line-length=119 +max-line-length = 119 -- GitLab From d74e1cd8ba879215276c843f884eb21b2b603199 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 13 Feb 2022 14:35:57 -0500 Subject: [PATCH 10/34] update setup.cfg Signed-off-by: Jose Borreguero --- setup.cfg | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.cfg b/setup.cfg index a8e12af..1b9a7f5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,10 @@ +[bumpversion] +current_version = 2.1.0 +commit = False +tag = False + +[bumpversion:file:django_remote_submission/__init__.py] + [metadata] name = django_remote_submission description = A Django application to manage long running job submission, including starting the job, saving logs, and storing results @@ -36,13 +43,6 @@ install_requires = include_package_data = True packages = django_remote_submission -[bumpversion] -current_version = 2.1.0 -commit = False -tag = False - -[bumpversion:file:django_remote_submission/__init__.py] - [flake8] exclude = __pycache__, -- GitLab From ffefc107bb9d938df8e9d7cf18f0678309d30bf6 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 13 Feb 2022 17:37:06 -0500 Subject: [PATCH 11/34] bumpversion command update in the makefile Signed-off-by: Jose Borreguero --- Makefile | 8 +++++--- setup.cfg | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 21f1e40..713736b 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,11 @@ help: clean: clean-build clean-pyc +bumpversion: ## increase the version, e.g "make bumpversion bump=minor". Allowed bumps are "major", "minor", "patch" + bumpversion $$bump + @echo "Don't forget to issue the following command:" + @tag=`grep current_version setup.cfg | cut -d '=' -f 2 | tr -d ' '`; echo "git push tag v$$tag" + clean-build: ## remove build artifacts rm -fr build/ rm -fr dist/ @@ -75,6 +80,3 @@ condadev: ## create conda environment for developing, and install pre-commit hoo conda install --name djrems -c conda-forge mamba mamba env update --name djrems --file ./conda.environment/environment-dev.yml $(CONDA_ACTIVATE) djrems; python setup.py develop; pre-commit install -bump2version: # change version, e.g "make bump2version version=2.3.0" - echo $version - bump2version --new-version $$version diff --git a/setup.cfg b/setup.cfg index 1b9a7f5..9fa984b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [bumpversion] current_version = 2.1.0 -commit = False -tag = False +commit = True +tag = True [bumpversion:file:django_remote_submission/__init__.py] -- GitLab From 77216f393524f7a2322f7ac3320e124c7c8f0d4d Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 13 Feb 2022 17:48:22 -0500 Subject: [PATCH 12/34] bumpversion conf file Signed-off-by: Jose Borreguero --- .bumpversion | 6 ++++++ setup.cfg | 7 ------- 2 files changed, 6 insertions(+), 7 deletions(-) create mode 100644 .bumpversion diff --git a/.bumpversion b/.bumpversion new file mode 100644 index 0000000..6c203f8 --- /dev/null +++ b/.bumpversion @@ -0,0 +1,6 @@ +[bumpversion] +current_version = 2.1.0 +commit = True +tag = True + +[bumpversion:file:django_remote_submission/__init__.py] diff --git a/setup.cfg b/setup.cfg index 9fa984b..92f7f88 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,10 +1,3 @@ -[bumpversion] -current_version = 2.1.0 -commit = True -tag = True - -[bumpversion:file:django_remote_submission/__init__.py] - [metadata] name = django_remote_submission description = A Django application to manage long running job submission, including starting the job, saving logs, and storing results -- GitLab From ed5fe18944ec7066139b5154df3dd74bfee6da6c Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 13 Feb 2022 17:51:17 -0500 Subject: [PATCH 13/34] add extension to bumpversion file Signed-off-by: Jose Borreguero --- .bumpversion => .bumpversion.cfg | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .bumpversion => .bumpversion.cfg (100%) diff --git a/.bumpversion b/.bumpversion.cfg similarity index 100% rename from .bumpversion rename to .bumpversion.cfg -- GitLab From 790f288e51ae7f8eccea10dd2711239cea4f2aaa Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 13 Feb 2022 17:54:41 -0500 Subject: [PATCH 14/34] grep bumped version from the .bumpversion file Signed-off-by: Jose Borreguero --- .bumpversion.cfg | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6c203f8..61fcaad 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.1.0 +current_version = 2.1.1 commit = True tag = True diff --git a/Makefile b/Makefile index 713736b..588eb98 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ clean: clean-build clean-pyc bumpversion: ## increase the version, e.g "make bumpversion bump=minor". Allowed bumps are "major", "minor", "patch" bumpversion $$bump @echo "Don't forget to issue the following command:" - @tag=`grep current_version setup.cfg | cut -d '=' -f 2 | tr -d ' '`; echo "git push tag v$$tag" + @tag=`grep current_version .bumpversion.cfg | cut -d '=' -f 2 | tr -d ' '`; echo "git push tag v$$tag" clean-build: ## remove build artifacts rm -fr build/ -- GitLab From 66985a099f8861539599354e4f5b7258516dc917 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sun, 13 Feb 2022 21:38:46 -0500 Subject: [PATCH 15/34] correct version in .bumpversion.cfg file --- .bumpversion.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 61fcaad..6c203f8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.1.1 +current_version = 2.1.0 commit = True tag = True -- GitLab From 73e0a6ff30c897b352721ee779c0d451a58391b6 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Mon, 14 Feb 2022 08:23:37 -0500 Subject: [PATCH 16/34] first draft of script to publish wheel --- .gitlab-ci.yml | 9 +++++---- scripts/gitlab/publish_wheel.sh | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 scripts/gitlab/publish_wheel.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8a15ddc..9e6149b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,13 +59,14 @@ noremote: tags: - dev -wheel: +latest-wheel: stage: deploy + only: + # DEBUG: change to main and next + - onwheels script: - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "python setup.py bdist_wheel --dist-dir artifacts" - artifacts: - paths: - - artifacts/django_remote_submission-${PACKAGE_VERSION}-py3-none-any.whl + - sudo bash ./scripts/gitlab/publish_wheel.sh tags: - dev diff --git a/scripts/gitlab/publish_wheel.sh b/scripts/gitlab/publish_wheel.sh new file mode 100644 index 0000000..4e5b6e9 --- /dev/null +++ b/scripts/gitlab/publish_wheel.sh @@ -0,0 +1,23 @@ +#! /bin/bash + +echo "############################" +echo "############################" +echo " The branch name is ${CI_COMMIT_REF_NAME}" + + +# Append "-dev" if branch is "next" +# DEBUG: replace "onwheels" with "next" +#if [ ${CI_COMMIT_REF_NAME} == "onwheels" ]; then +# FILENAME_OLD=`ls artifacts/django_remote_submission-*-py3-none-any.whl` +# VERSION=`echo ${FILENAME_OLD} | grep -oP "[0-9]+\.[0-9]+\.[0-9]+"` +# FILENAME_NEW=`echo ${FILENAME_OLD} | sed "s/$VERSION/$VERSION-dev/g"` +# /bin/mv ${FILENAME_OLD} ${FILENAME_NEW} +#fi + +FILENAME=`ls artifacts/django_remote_submission-*-py3-none-any.whl` +echo $FILENAME + +#python -m twine upload --repository gitlab $FILENAME + +echo "############################" +echo "############################" -- GitLab From 86f2b8fdaa1dff8caa72ced0788e7ba0dbf0ce7f Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Mon, 14 Feb 2022 16:42:08 -0500 Subject: [PATCH 17/34] publish wheel command Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 6 ++++-- scripts/gitlab/amend_wheel_filename.sh | 14 ++++++++++++++ scripts/gitlab/publish_wheel.sh | 23 ----------------------- 3 files changed, 18 insertions(+), 25 deletions(-) create mode 100644 scripts/gitlab/amend_wheel_filename.sh delete mode 100644 scripts/gitlab/publish_wheel.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9e6149b..28635ac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,7 +59,7 @@ noremote: tags: - dev -latest-wheel: +pythonwheel: stage: deploy only: # DEBUG: change to main and next @@ -67,6 +67,8 @@ latest-wheel: script: - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "python setup.py bdist_wheel --dist-dir artifacts" - - sudo bash ./scripts/gitlab/publish_wheel.sh + - sed -i "s|CI_COMMIT_REF_NAME|${CI_COMMIT_REF_NAME}|" ./scripts/gitlab/amend_wheel_filename.sh + - sudo bash ./scripts/gitlab/amend_wheel_filename.sh + - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi artifacts/django_remote_submission-*-py3-none-any.whl tags: - dev diff --git a/scripts/gitlab/amend_wheel_filename.sh b/scripts/gitlab/amend_wheel_filename.sh new file mode 100644 index 0000000..7cddde7 --- /dev/null +++ b/scripts/gitlab/amend_wheel_filename.sh @@ -0,0 +1,14 @@ +#! /bin/bash +# +# Placeholder CI_COMMIT_REF_NAME must be replaced +# with its actual value prior to running this script +# + +# Append "-dev" to the wheel name if the branch is "next" +# DEBUG: replace "onwheels" with "next" +if [ "CI_COMMIT_REF_NAME" == "onwheels" ]; then + FILENAME_OLD=`ls artifacts/django_remote_submission-*-py3-none-any.whl` + VERSION=`echo ${FILENAME_OLD} | grep -oP "[0-9]+\.[0-9]+\.[0-9]+"` # e.g. "2.1.0" + FILENAME_NEW=`echo ${FILENAME_OLD} | sed "s/$VERSION/$VERSION.dev/g"` + /bin/mv ${FILENAME_OLD} ${FILENAME_NEW} +fi diff --git a/scripts/gitlab/publish_wheel.sh b/scripts/gitlab/publish_wheel.sh deleted file mode 100644 index 4e5b6e9..0000000 --- a/scripts/gitlab/publish_wheel.sh +++ /dev/null @@ -1,23 +0,0 @@ -#! /bin/bash - -echo "############################" -echo "############################" -echo " The branch name is ${CI_COMMIT_REF_NAME}" - - -# Append "-dev" if branch is "next" -# DEBUG: replace "onwheels" with "next" -#if [ ${CI_COMMIT_REF_NAME} == "onwheels" ]; then -# FILENAME_OLD=`ls artifacts/django_remote_submission-*-py3-none-any.whl` -# VERSION=`echo ${FILENAME_OLD} | grep -oP "[0-9]+\.[0-9]+\.[0-9]+"` -# FILENAME_NEW=`echo ${FILENAME_OLD} | sed "s/$VERSION/$VERSION-dev/g"` -# /bin/mv ${FILENAME_OLD} ${FILENAME_NEW} -#fi - -FILENAME=`ls artifacts/django_remote_submission-*-py3-none-any.whl` -echo $FILENAME - -#python -m twine upload --repository gitlab $FILENAME - -echo "############################" -echo "############################" -- GitLab From ef2177f2a7fe62da7ffc8d1126f73a9f2b2cc08c Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Mon, 14 Feb 2022 17:51:10 -0500 Subject: [PATCH 18/34] YAML anchors for multiple before_script and after_script Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 71 ++++++++++++++++++-------- scripts/gitlab/amend_wheel_filename.sh | 2 +- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 28635ac..f798fbb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,22 +1,49 @@ variables: INSTALL_DIR: "/opt/django_remote_submission" IMAGE_NAMETAG: "code.ornl.gov:4567/reflectometry/django-remote-submission/${CI_COMMIT_REF_SLUG}" - PACKAGE_VERSION: "2.1.0" stages: - - build - static_analysis + - build - test - deploy -# "setup" and "teardown" for all jobs -default: - before_script: - - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines +# +# YAML anchors Section +# to reuse setup and teardown commands +# + +.docker-setup: &docker-setup + - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines # get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) - - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} - after_script: - - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs + - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} + +.docker-teardown: &docker-teardown + - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs + +.conda-setup: &conda-setup + - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda install --name base -c conda-forge mamba + - mamba env update --name base --file ./conda.environment/environment-dev.yml + - mamba env install --name base twine + +# +# Jobs Section +# + +flake8: + stage: static_analysis + before_script: + - *conda-setup + script: + - source activate base + - flake8 ./django_remote_submission ./tests + tags: + - dev before_script: - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines @@ -28,28 +55,28 @@ after_script: dockerbuild: stage: build + before_script: + - *docker-setup script: - mv .env.ci .env - docker build -t ${IMAGE_NAMETAG} . # create image with default Dockerfile - docker push ${IMAGE_NAMETAG} # upload to the Container Registry + after_script: + - *docker-teardown tags: - dev # identify which runners can run this job -flake8: - stage: static_analysis - script: - - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry - - docker run ${IMAGE_NAMETAG} bash -c "flake8 ./django_remote_submission ./tests" # entrypoint is ${INSTALL_DIR} - tags: - - dev - # Tests that do not require a remote worker noremote: stage: test + before_script: + - *docker-setup script: - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry # pytest with coverage, creating a volume to retrieve the coverage output from the container - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "make test-noremote" + after_script: + - *docker-teardown artifacts: paths: - artifacts/htmlcov/ # detailed coverage report htmlcov/index.html @@ -64,11 +91,13 @@ pythonwheel: only: # DEBUG: change to main and next - onwheels + before_script: + - *conda-setup script: - - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry - - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "python setup.py bdist_wheel --dist-dir artifacts" + - source activate base + - python setup.py bdist_wheel - sed -i "s|CI_COMMIT_REF_NAME|${CI_COMMIT_REF_NAME}|" ./scripts/gitlab/amend_wheel_filename.sh - - sudo bash ./scripts/gitlab/amend_wheel_filename.sh - - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi artifacts/django_remote_submission-*-py3-none-any.whl + - sudo bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" + - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl tags: - dev diff --git a/scripts/gitlab/amend_wheel_filename.sh b/scripts/gitlab/amend_wheel_filename.sh index 7cddde7..6fb6c51 100644 --- a/scripts/gitlab/amend_wheel_filename.sh +++ b/scripts/gitlab/amend_wheel_filename.sh @@ -7,7 +7,7 @@ # Append "-dev" to the wheel name if the branch is "next" # DEBUG: replace "onwheels" with "next" if [ "CI_COMMIT_REF_NAME" == "onwheels" ]; then - FILENAME_OLD=`ls artifacts/django_remote_submission-*-py3-none-any.whl` + FILENAME_OLD=`ls dist/django_remote_submission-*-py3-none-any.whl` VERSION=`echo ${FILENAME_OLD} | grep -oP "[0-9]+\.[0-9]+\.[0-9]+"` # e.g. "2.1.0" FILENAME_NEW=`echo ${FILENAME_OLD} | sed "s/$VERSION/$VERSION.dev/g"` /bin/mv ${FILENAME_OLD} ${FILENAME_NEW} -- GitLab From d5d6f1f5a625e0c18d0326c7555f6767bf648948 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Mon, 14 Feb 2022 18:02:59 -0500 Subject: [PATCH 19/34] install twine only for deployment Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f798fbb..1f3f4c6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,8 @@ stages: # .docker-setup: &docker-setup - - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines + # remove containers, images, volumes, and networks left over from previous pipelines + - docker system prune -f -a --volumes # get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} @@ -29,7 +30,6 @@ stages: - conda config --set always_yes yes --set changeps1 no - conda install --name base -c conda-forge mamba - mamba env update --name base --file ./conda.environment/environment-dev.yml - - mamba env install --name base twine # # Jobs Section @@ -40,7 +40,7 @@ flake8: before_script: - *conda-setup script: - - source activate base + - conda activate base - flake8 ./django_remote_submission ./tests tags: - dev @@ -94,7 +94,8 @@ pythonwheel: before_script: - *conda-setup script: - - source activate base + - conda activate base + - mamba install twine - python setup.py bdist_wheel - sed -i "s|CI_COMMIT_REF_NAME|${CI_COMMIT_REF_NAME}|" ./scripts/gitlab/amend_wheel_filename.sh - sudo bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" -- GitLab From 9766e5d48c8eda28e316a6ac9a5d3d1385c1a793 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Mon, 14 Feb 2022 18:09:52 -0500 Subject: [PATCH 20/34] add conda init in before_script Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1f3f4c6..14a43aa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,6 +27,7 @@ stages: - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - hash -r + - conda init bash - conda config --set always_yes yes --set changeps1 no - conda install --name base -c conda-forge mamba - mamba env update --name base --file ./conda.environment/environment-dev.yml -- GitLab From b7fd1073632adedf1d0c9314de34077c6a66410d Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Mon, 14 Feb 2022 22:05:10 -0500 Subject: [PATCH 21/34] clean up directory from previous job Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 14a43aa..8449c26 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,7 @@ stages: .conda-setup: &conda-setup - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + - if [ -d "$HOME/miniconda" ];then /bin/rm -r $HOME/miniconda;fi # clean-up - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - hash -r -- GitLab From 3c91a275cb97951c34c92aa2c0585e753921cdb7 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Mon, 14 Feb 2022 23:22:49 -0500 Subject: [PATCH 22/34] source the activate script Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8449c26..316ba90 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,7 +42,7 @@ flake8: before_script: - *conda-setup script: - - conda activate base + - source activate base - flake8 ./django_remote_submission ./tests tags: - dev -- GitLab From 6166e3c5cbc27f066ef23979999bb93849f2bf33 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 15 Feb 2022 07:47:46 -0500 Subject: [PATCH 23/34] setup conda with script Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 20 ++++++++------------ scripts/gitlab/setup_conda.sh | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 12 deletions(-) create mode 100644 scripts/gitlab/setup_conda.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 316ba90..4cca013 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,5 @@ variables: + CONDA_ENV: "djrems" INSTALL_DIR: "/opt/django_remote_submission" IMAGE_NAMETAG: "code.ornl.gov:4567/reflectometry/django-remote-submission/${CI_COMMIT_REF_SLUG}" @@ -23,15 +24,8 @@ stages: - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs .conda-setup: &conda-setup - - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - - if [ -d "$HOME/miniconda" ];then /bin/rm -r $HOME/miniconda;fi # clean-up - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$PATH" - - hash -r - - conda init bash - - conda config --set always_yes yes --set changeps1 no - - conda install --name base -c conda-forge mamba - - mamba env update --name base --file ./conda.environment/environment-dev.yml + - bash ./scripts/gitlab/setup_conda.sh + - mamba env update --name ${CONDA_ENV} --file ./conda.environment/environment-dev.yml # # Jobs Section @@ -42,7 +36,8 @@ flake8: before_script: - *conda-setup script: - - source activate base + - echo "DEBUG notice, CONDA INSTALL ="; echo `which conda` + - conda activate ${CONDA_ENV} - flake8 ./django_remote_submission ./tests tags: - dev @@ -96,11 +91,12 @@ pythonwheel: before_script: - *conda-setup script: - - conda activate base + - echo "DEBUG notice, CONDA INSTALL ="; echo `which conda` + - conda activate ${CONDA_ENV} - mamba install twine - python setup.py bdist_wheel - sed -i "s|CI_COMMIT_REF_NAME|${CI_COMMIT_REF_NAME}|" ./scripts/gitlab/amend_wheel_filename.sh - - sudo bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" + - bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl tags: - dev diff --git a/scripts/gitlab/setup_conda.sh b/scripts/gitlab/setup_conda.sh new file mode 100644 index 0000000..482e82e --- /dev/null +++ b/scripts/gitlab/setup_conda.sh @@ -0,0 +1,21 @@ +#! /bin/bash +# +# Placeholders CONDA_ENV, RUNNER_HOME must be replaced +# with their actual value prior to running this script +# + +# Does the miniconda directory already exists? +if [ ! -d "$HOME/miniconda" ]; then + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + bash miniconda.sh -b -p $HOME/miniconda +fi +export PATH="$HOME/miniconda/bin:$PATH" + +# Has the conda environment being created already? +if ! source activate "${CONDA_ENV}";then + conda create --name "${CONDA_ENV}" + conda install --name "${CONDA_ENV}" -c conda-forge mamba +fi + +hash -r +conda config --set always_yes yes --set changeps1 no -- GitLab From b7739d314def7dfecb3bd16c2662b9dc3c147415 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 15 Feb 2022 07:55:46 -0500 Subject: [PATCH 24/34] export PATH variable Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 4 ++-- scripts/gitlab/setup_conda.sh | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4cca013..e7630d0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,6 +25,7 @@ stages: .conda-setup: &conda-setup - bash ./scripts/gitlab/setup_conda.sh + - export PATH="$HOME/miniconda/bin:$PATH" - mamba env update --name ${CONDA_ENV} --file ./conda.environment/environment-dev.yml # @@ -95,8 +96,7 @@ pythonwheel: - conda activate ${CONDA_ENV} - mamba install twine - python setup.py bdist_wheel - - sed -i "s|CI_COMMIT_REF_NAME|${CI_COMMIT_REF_NAME}|" ./scripts/gitlab/amend_wheel_filename.sh - - bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" + - CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl tags: - dev diff --git a/scripts/gitlab/setup_conda.sh b/scripts/gitlab/setup_conda.sh index 482e82e..5879bff 100644 --- a/scripts/gitlab/setup_conda.sh +++ b/scripts/gitlab/setup_conda.sh @@ -4,12 +4,20 @@ # with their actual value prior to running this script # +echo "########################" +echo "########################" +echo "HOME=$HOME" +echo "CONDA_ENV=${CONDA_ENV}" +echo "PATH=$PATH" +echo "########################" +echo "########################" + # Does the miniconda directory already exists? if [ ! -d "$HOME/miniconda" ]; then wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; bash miniconda.sh -b -p $HOME/miniconda fi -export PATH="$HOME/miniconda/bin:$PATH" +PATH="$HOME/miniconda/bin:$PATH" # Has the conda environment being created already? if ! source activate "${CONDA_ENV}";then -- GitLab From 113147e3d1b77e602bf63f02e14f4ca2d1477ba6 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 15 Feb 2022 08:11:38 -0500 Subject: [PATCH 25/34] try with conda instead of mamba Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 2 +- scripts/gitlab/setup_conda.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e7630d0..b7a7db0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,7 +26,7 @@ stages: .conda-setup: &conda-setup - bash ./scripts/gitlab/setup_conda.sh - export PATH="$HOME/miniconda/bin:$PATH" - - mamba env update --name ${CONDA_ENV} --file ./conda.environment/environment-dev.yml + - conda env update --name ${CONDA_ENV} --file ./conda.environment/environment-dev.yml # # Jobs Section diff --git a/scripts/gitlab/setup_conda.sh b/scripts/gitlab/setup_conda.sh index 5879bff..71e5731 100644 --- a/scripts/gitlab/setup_conda.sh +++ b/scripts/gitlab/setup_conda.sh @@ -22,7 +22,7 @@ PATH="$HOME/miniconda/bin:$PATH" # Has the conda environment being created already? if ! source activate "${CONDA_ENV}";then conda create --name "${CONDA_ENV}" - conda install --name "${CONDA_ENV}" -c conda-forge mamba + #conda install --name "${CONDA_ENV}" -c conda-forge mamba fi hash -r -- GitLab From 27325fb4f2c32e35bf12308f464e057e1663d4b4 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 15 Feb 2022 08:17:15 -0500 Subject: [PATCH 26/34] use source instead of conda command Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b7a7db0..a1de8f6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,7 +38,7 @@ flake8: - *conda-setup script: - echo "DEBUG notice, CONDA INSTALL ="; echo `which conda` - - conda activate ${CONDA_ENV} + - source activate ${CONDA_ENV} - flake8 ./django_remote_submission ./tests tags: - dev @@ -93,8 +93,8 @@ pythonwheel: - *conda-setup script: - echo "DEBUG notice, CONDA INSTALL ="; echo `which conda` - - conda activate ${CONDA_ENV} - - mamba install twine + - source activate ${CONDA_ENV} + - conda install twine - python setup.py bdist_wheel - CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl -- GitLab From 63084249930f1dbeaa0c556d0018d416eba337cf Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 15 Feb 2022 09:02:22 -0500 Subject: [PATCH 27/34] temporary debug stage Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 28 ++++++++++++++++++++++---- scripts/gitlab/amend_wheel_filename.sh | 16 ++++++++++----- scripts/gitlab/setup_conda.sh | 9 --------- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a1de8f6..cde6ccb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,11 +3,14 @@ variables: INSTALL_DIR: "/opt/django_remote_submission" IMAGE_NAMETAG: "code.ornl.gov:4567/reflectometry/django-remote-submission/${CI_COMMIT_REF_SLUG}" +#stages: +# - static_analysis +# - build +# - test +# - deploy + stages: - - static_analysis - - build - - test - - deploy + - debug # # YAML anchors Section @@ -32,6 +35,23 @@ stages: # Jobs Section # +debug: + stage: debug + only: + # DEBUG: change to main and next + - onwheels + before_script: + - *conda-setup + script: + - echo "DEBUG notice, CONDA INSTALL ="; echo `which conda` + - source activate ${CONDA_ENV} + - conda install twine + - python setup.py bdist_wheel + - CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" + - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl + tags: + - dev + flake8: stage: static_analysis before_script: diff --git a/scripts/gitlab/amend_wheel_filename.sh b/scripts/gitlab/amend_wheel_filename.sh index 6fb6c51..25d283e 100644 --- a/scripts/gitlab/amend_wheel_filename.sh +++ b/scripts/gitlab/amend_wheel_filename.sh @@ -6,9 +6,15 @@ # Append "-dev" to the wheel name if the branch is "next" # DEBUG: replace "onwheels" with "next" -if [ "CI_COMMIT_REF_NAME" == "onwheels" ]; then - FILENAME_OLD=`ls dist/django_remote_submission-*-py3-none-any.whl` - VERSION=`echo ${FILENAME_OLD} | grep -oP "[0-9]+\.[0-9]+\.[0-9]+"` # e.g. "2.1.0" - FILENAME_NEW=`echo ${FILENAME_OLD} | sed "s/$VERSION/$VERSION.dev/g"` - /bin/mv ${FILENAME_OLD} ${FILENAME_NEW} +if [ "${CI_COMMIT_REF_NAME}" == "onwheels" ]; then + FILENAME_OLD=$(ls dist/django_remote_submission-*-py3-none-any.whl) + VERSION=$(echo "${FILENAME_OLD}" | grep -oP "[0-9]+\.[0-9]+\.[0-9]+") # e.g. "2.1.0" + FILENAME_NEW=$(echo "${FILENAME_OLD}" | sed "s/$VERSION/$VERSION.dev/g") + /bin/mv "${FILENAME_OLD}" "${FILENAME_NEW}" +else + echo "########################" + echo "########################" + echo "CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME}" + echo "########################" + echo "########################" fi diff --git a/scripts/gitlab/setup_conda.sh b/scripts/gitlab/setup_conda.sh index 71e5731..3296a8f 100644 --- a/scripts/gitlab/setup_conda.sh +++ b/scripts/gitlab/setup_conda.sh @@ -4,14 +4,6 @@ # with their actual value prior to running this script # -echo "########################" -echo "########################" -echo "HOME=$HOME" -echo "CONDA_ENV=${CONDA_ENV}" -echo "PATH=$PATH" -echo "########################" -echo "########################" - # Does the miniconda directory already exists? if [ ! -d "$HOME/miniconda" ]; then wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; @@ -22,7 +14,6 @@ PATH="$HOME/miniconda/bin:$PATH" # Has the conda environment being created already? if ! source activate "${CONDA_ENV}";then conda create --name "${CONDA_ENV}" - #conda install --name "${CONDA_ENV}" -c conda-forge mamba fi hash -r -- GitLab From f35cbc44b7ca5d47a2021a5021e70dfa0bcb7933 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 15 Feb 2022 09:04:16 -0500 Subject: [PATCH 28/34] all jobs must be accounted for in stages Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cde6ccb..84c6979 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,14 +3,12 @@ variables: INSTALL_DIR: "/opt/django_remote_submission" IMAGE_NAMETAG: "code.ornl.gov:4567/reflectometry/django-remote-submission/${CI_COMMIT_REF_SLUG}" -#stages: -# - static_analysis -# - build -# - test -# - deploy - stages: - debug + - static_analysis + - build + - test + - deploy # # YAML anchors Section -- GitLab From 2c243aab13d9812710fbdfa20301938bb3881a6e Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 15 Feb 2022 09:26:41 -0500 Subject: [PATCH 29/34] add verbose to investigate proble Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 4 ++-- scripts/gitlab/amend_wheel_filename.sh | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 84c6979..728e96d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,7 +46,7 @@ debug: - conda install twine - python setup.py bdist_wheel - CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" - - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl + - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl tags: - dev @@ -115,6 +115,6 @@ pythonwheel: - conda install twine - python setup.py bdist_wheel - CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" - - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl + - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl tags: - dev diff --git a/scripts/gitlab/amend_wheel_filename.sh b/scripts/gitlab/amend_wheel_filename.sh index 25d283e..31a1226 100644 --- a/scripts/gitlab/amend_wheel_filename.sh +++ b/scripts/gitlab/amend_wheel_filename.sh @@ -12,9 +12,17 @@ if [ "${CI_COMMIT_REF_NAME}" == "onwheels" ]; then FILENAME_NEW=$(echo "${FILENAME_OLD}" | sed "s/$VERSION/$VERSION.dev/g") /bin/mv "${FILENAME_OLD}" "${FILENAME_NEW}" else + # DEBUG echo "########################" echo "########################" echo "CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME}" echo "########################" echo "########################" fi + + # DEBUG + echo "########################" + echo "########################" + echo $(ls dist/*) + echo "########################" + echo "########################" -- GitLab From 917d7f918bd0788c3d5af6c0ca46c5f055e2da51 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 15 Feb 2022 10:39:07 -0500 Subject: [PATCH 30/34] use the pipeline id as build number Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 2 +- scripts/gitlab/amend_wheel_filename.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 728e96d..5fd7249 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -45,7 +45,7 @@ debug: - source activate ${CONDA_ENV} - conda install twine - python setup.py bdist_wheel - - CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" + - CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} CI_PIPELINE_IID=${CI_PIPELINE_IID} bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl tags: - dev diff --git a/scripts/gitlab/amend_wheel_filename.sh b/scripts/gitlab/amend_wheel_filename.sh index 31a1226..c0098a0 100644 --- a/scripts/gitlab/amend_wheel_filename.sh +++ b/scripts/gitlab/amend_wheel_filename.sh @@ -9,7 +9,7 @@ if [ "${CI_COMMIT_REF_NAME}" == "onwheels" ]; then FILENAME_OLD=$(ls dist/django_remote_submission-*-py3-none-any.whl) VERSION=$(echo "${FILENAME_OLD}" | grep -oP "[0-9]+\.[0-9]+\.[0-9]+") # e.g. "2.1.0" - FILENAME_NEW=$(echo "${FILENAME_OLD}" | sed "s/$VERSION/$VERSION.dev/g") + FILENAME_NEW=$(echo "${FILENAME_OLD}" | sed "s/$VERSION/${CI_PIPELINE_IID}/g") /bin/mv "${FILENAME_OLD}" "${FILENAME_NEW}" else # DEBUG @@ -23,6 +23,7 @@ fi # DEBUG echo "########################" echo "########################" + echo "CI_PIPELINE_IID=${CI_PIPELINE_IID}" echo $(ls dist/*) echo "########################" echo "########################" -- GitLab From 6c48d7bf44934177ed6038ebae9f8d7b37588689 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 15 Feb 2022 10:53:38 -0500 Subject: [PATCH 31/34] follow PYPI practice for versioning a development release Signed-off-by: Jose Borreguero --- scripts/gitlab/amend_wheel_filename.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/gitlab/amend_wheel_filename.sh b/scripts/gitlab/amend_wheel_filename.sh index c0098a0..a66ee35 100644 --- a/scripts/gitlab/amend_wheel_filename.sh +++ b/scripts/gitlab/amend_wheel_filename.sh @@ -9,7 +9,8 @@ if [ "${CI_COMMIT_REF_NAME}" == "onwheels" ]; then FILENAME_OLD=$(ls dist/django_remote_submission-*-py3-none-any.whl) VERSION=$(echo "${FILENAME_OLD}" | grep -oP "[0-9]+\.[0-9]+\.[0-9]+") # e.g. "2.1.0" - FILENAME_NEW=$(echo "${FILENAME_OLD}" | sed "s/$VERSION/${CI_PIPELINE_IID}/g") + # suggested development releases versioning: https://the-hitchhikers-guide-to-packaging.readthedocs.io/en/latest/specification.html#development-releases + FILENAME_NEW=$(echo "${FILENAME_OLD}" | sed "s/$VERSION/$VERSION.dev${CI_PIPELINE_IID}/g") /bin/mv "${FILENAME_OLD}" "${FILENAME_NEW}" else # DEBUG -- GitLab From 8db131b735ea812949b8a6a40434e71232a17a28 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 15 Feb 2022 11:04:07 -0500 Subject: [PATCH 32/34] delete debug statements and conditions Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 26 +++----------------------- scripts/gitlab/amend_wheel_filename.sh | 20 ++------------------ 2 files changed, 5 insertions(+), 41 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5fd7249..c38e516 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,6 @@ variables: IMAGE_NAMETAG: "code.ornl.gov:4567/reflectometry/django-remote-submission/${CI_COMMIT_REF_SLUG}" stages: - - debug - static_analysis - build - test @@ -33,29 +32,11 @@ stages: # Jobs Section # -debug: - stage: debug - only: - # DEBUG: change to main and next - - onwheels - before_script: - - *conda-setup - script: - - echo "DEBUG notice, CONDA INSTALL ="; echo `which conda` - - source activate ${CONDA_ENV} - - conda install twine - - python setup.py bdist_wheel - - CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} CI_PIPELINE_IID=${CI_PIPELINE_IID} bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" - - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl - tags: - - dev - flake8: stage: static_analysis before_script: - *conda-setup script: - - echo "DEBUG notice, CONDA INSTALL ="; echo `which conda` - source activate ${CONDA_ENV} - flake8 ./django_remote_submission ./tests tags: @@ -105,16 +86,15 @@ noremote: pythonwheel: stage: deploy only: - # DEBUG: change to main and next - - onwheels + - main + - next before_script: - *conda-setup script: - - echo "DEBUG notice, CONDA INSTALL ="; echo `which conda` - source activate ${CONDA_ENV} - conda install twine - python setup.py bdist_wheel - CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" - - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl + - TWINE_PASSWORD=${CI_JOB_TOKEN} CI_PIPELINE_IID=${CI_PIPELINE_IID} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl tags: - dev diff --git a/scripts/gitlab/amend_wheel_filename.sh b/scripts/gitlab/amend_wheel_filename.sh index a66ee35..bbbd7e1 100644 --- a/scripts/gitlab/amend_wheel_filename.sh +++ b/scripts/gitlab/amend_wheel_filename.sh @@ -4,27 +4,11 @@ # with its actual value prior to running this script # -# Append "-dev" to the wheel name if the branch is "next" -# DEBUG: replace "onwheels" with "next" -if [ "${CI_COMMIT_REF_NAME}" == "onwheels" ]; then +# Append "-devXXX" to the wheel name if the branch is "next" +if [ "${CI_COMMIT_REF_NAME}" == "next" ]; then FILENAME_OLD=$(ls dist/django_remote_submission-*-py3-none-any.whl) VERSION=$(echo "${FILENAME_OLD}" | grep -oP "[0-9]+\.[0-9]+\.[0-9]+") # e.g. "2.1.0" # suggested development releases versioning: https://the-hitchhikers-guide-to-packaging.readthedocs.io/en/latest/specification.html#development-releases FILENAME_NEW=$(echo "${FILENAME_OLD}" | sed "s/$VERSION/$VERSION.dev${CI_PIPELINE_IID}/g") /bin/mv "${FILENAME_OLD}" "${FILENAME_NEW}" -else - # DEBUG - echo "########################" - echo "########################" - echo "CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME}" - echo "########################" - echo "########################" fi - - # DEBUG - echo "########################" - echo "########################" - echo "CI_PIPELINE_IID=${CI_PIPELINE_IID}" - echo $(ls dist/*) - echo "########################" - echo "########################" -- GitLab From 5cc7d8751648a06b143fa666bf32393f6df48290 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 15 Feb 2022 14:23:29 -0500 Subject: [PATCH 33/34] check build_number options Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c38e516..072b835 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -88,13 +88,15 @@ pythonwheel: only: - main - next + # DEBUG remove "onwheels" item + - onwheels before_script: - *conda-setup script: - source activate ${CONDA_ENV} - conda install twine - - python setup.py bdist_wheel - - CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} bash ./scripts/gitlab/amend_wheel_filename.sh # only does something for branch "next" + # DEBUG substitute "onwheels" for "next" and delete file "amend_wheel_filename.sh" + - if [ "${CI_COMMIT_REF_NAME}" == "onwheels" ]; then python setup.py bdist_wheel --build-number ${CI_PIPELINE_IID}; else python setup.py bdist_wheel;fi - TWINE_PASSWORD=${CI_JOB_TOKEN} CI_PIPELINE_IID=${CI_PIPELINE_IID} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl tags: - dev -- GitLab From e055bae32db65c90ec7da41edb3de01d67369dad Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Tue, 15 Feb 2022 14:36:31 -0500 Subject: [PATCH 34/34] delete DEBUG conditions Signed-off-by: Jose Borreguero --- .gitlab-ci.yml | 5 +---- scripts/gitlab/amend_wheel_filename.sh | 14 -------------- 2 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 scripts/gitlab/amend_wheel_filename.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 072b835..8b38159 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -88,15 +88,12 @@ pythonwheel: only: - main - next - # DEBUG remove "onwheels" item - - onwheels before_script: - *conda-setup script: - source activate ${CONDA_ENV} - conda install twine - # DEBUG substitute "onwheels" for "next" and delete file "amend_wheel_filename.sh" - - if [ "${CI_COMMIT_REF_NAME}" == "onwheels" ]; then python setup.py bdist_wheel --build-number ${CI_PIPELINE_IID}; else python setup.py bdist_wheel;fi + - if [ "${CI_COMMIT_REF_NAME}" == "next" ]; then python setup.py bdist_wheel --build-number ${CI_PIPELINE_IID}; else python setup.py bdist_wheel;fi - TWINE_PASSWORD=${CI_JOB_TOKEN} CI_PIPELINE_IID=${CI_PIPELINE_IID} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl tags: - dev diff --git a/scripts/gitlab/amend_wheel_filename.sh b/scripts/gitlab/amend_wheel_filename.sh deleted file mode 100644 index bbbd7e1..0000000 --- a/scripts/gitlab/amend_wheel_filename.sh +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/bash -# -# Placeholder CI_COMMIT_REF_NAME must be replaced -# with its actual value prior to running this script -# - -# Append "-devXXX" to the wheel name if the branch is "next" -if [ "${CI_COMMIT_REF_NAME}" == "next" ]; then - FILENAME_OLD=$(ls dist/django_remote_submission-*-py3-none-any.whl) - VERSION=$(echo "${FILENAME_OLD}" | grep -oP "[0-9]+\.[0-9]+\.[0-9]+") # e.g. "2.1.0" - # suggested development releases versioning: https://the-hitchhikers-guide-to-packaging.readthedocs.io/en/latest/specification.html#development-releases - FILENAME_NEW=$(echo "${FILENAME_OLD}" | sed "s/$VERSION/$VERSION.dev${CI_PIPELINE_IID}/g") - /bin/mv "${FILENAME_OLD}" "${FILENAME_NEW}" -fi -- GitLab