diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000000000000000000000000000000000000..6c203f8f0b4390eb4c19e7ae46441567d38d9613 --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,6 @@ +[bumpversion] +current_version = 2.1.0 +commit = True +tag = True + +[bumpversion:file:django_remote_submission/__init__.py] diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a6defaedbae865472c47ef2de2e7b35d018d06b..8b38159967920df93fc9fde406ebbe53164729d4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,45 +1,79 @@ variables: + CONDA_ENV: "djrems" INSTALL_DIR: "/opt/django_remote_submission" IMAGE_NAMETAG: "code.ornl.gov:4567/reflectometry/django-remote-submission/${CI_COMMIT_REF_SLUG}" stages: - - build - static_analysis + - build - test + - deploy -# "setup" and "teardown" for all jobs -default: - before_script: - - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines +# +# YAML anchors Section +# to reuse setup and teardown commands +# + +.docker-setup: &docker-setup + # remove containers, images, volumes, and networks left over from previous pipelines + - docker system prune -f -a --volumes # get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) - - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} - after_script: - - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs + - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} + +.docker-teardown: &docker-teardown + - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs + +.conda-setup: &conda-setup + - bash ./scripts/gitlab/setup_conda.sh + - export PATH="$HOME/miniconda/bin:$PATH" + - conda env update --name ${CONDA_ENV} --file ./conda.environment/environment-dev.yml + +# +# Jobs Section +# + +flake8: + stage: static_analysis + before_script: + - *conda-setup + script: + - source activate ${CONDA_ENV} + - flake8 ./django_remote_submission ./tests + tags: + - dev + +before_script: + - docker system prune -f -a --volumes # remove containers, images, volumes, and networks left over from previous pipelines +# get access to the Container Registry (https://code.ornl.gov/reflectometry/django-remote-submission/container_registry) + - docker login --username=${CI_REGISTRY_USER} --password=${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} + +after_script: + - sudo chown -R gitlab-runner . # allows "docker system prune" next time the pipeline runs dockerbuild: stage: build + before_script: + - *docker-setup script: - mv .env.ci .env - docker build -t ${IMAGE_NAMETAG} . # create image with default Dockerfile - docker push ${IMAGE_NAMETAG} # upload to the Container Registry + after_script: + - *docker-teardown tags: - dev # identify which runners can run this job -flake8: - stage: static_analysis - script: - - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry - - docker run ${IMAGE_NAMETAG} bash -c "flake8 ./django_remote_submission ./tests" # entrypoint is ${INSTALL_DIR} - tags: - - dev - # Tests that do not require a remote worker noremote: stage: test + before_script: + - *docker-setup script: - docker pull ${IMAGE_NAMETAG} # pull from the Container Registry # pytest with coverage, creating a volume to retrieve the coverage output from the container - docker run -v $PWD/artifacts:${INSTALL_DIR}/artifacts ${IMAGE_NAMETAG} bash -c "make test-noremote" + after_script: + - *docker-teardown artifacts: paths: - artifacts/htmlcov/ # detailed coverage report htmlcov/index.html @@ -48,3 +82,18 @@ noremote: expire_in: 1 week # remove artifacts for this Job after one week tags: - dev + +pythonwheel: + stage: deploy + only: + - main + - next + before_script: + - *conda-setup + script: + - source activate ${CONDA_ENV} + - conda install twine + - if [ "${CI_COMMIT_REF_NAME}" == "next" ]; then python setup.py bdist_wheel --build-number ${CI_PIPELINE_IID}; else python setup.py bdist_wheel;fi + - TWINE_PASSWORD=${CI_JOB_TOKEN} CI_PIPELINE_IID=${CI_PIPELINE_IID} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/django_remote_submission-*-py3-none-any.whl + tags: + - dev diff --git a/Makefile b/Makefile index 27d7dfd72589c05e878cb3fc3e170e4a86756302..588eb98cae3df55c6f5ade70fa058f4c1f5ae7c5 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,11 @@ help: clean: clean-build clean-pyc +bumpversion: ## increase the version, e.g "make bumpversion bump=minor". Allowed bumps are "major", "minor", "patch" + bumpversion $$bump + @echo "Don't forget to issue the following command:" + @tag=`grep current_version .bumpversion.cfg | cut -d '=' -f 2 | tr -d ' '`; echo "git push tag v$$tag" + clean-build: ## remove build artifacts rm -fr build/ rm -fr dist/ diff --git a/conda.environment/environment-dev.yml b/conda.environment/environment-dev.yml index 2b2454afad6ba383732621ea97e87b374ee72adb..9eaa0ab97ecfe6e72f72df7cdd837e22260577b0 100644 --- a/conda.environment/environment-dev.yml +++ b/conda.environment/environment-dev.yml @@ -6,6 +6,7 @@ channels: dependencies: - python=3.8 - asgiref + - bump2version - celery - django=3.2 - django-channels @@ -15,15 +16,13 @@ dependencies: - djangorestframework - paramiko - pip - - setuptools - - six - - wheel # from here on packages strictly for documentation - docutils - sphinx # from here on packages strictly for testing - black - bumpversion + - coverage - flake8 - mock - mypy @@ -32,4 +31,6 @@ dependencies: - pytest-cov - pytest-django - pytest-mock + - setuptools + - wheel - tox diff --git a/conda.environment/environment.yml b/conda.environment/environment.yml index b6f7a51ed6382eba41a9a9e29edcd8033fe3bcf4..70e5d1f39b19d0eb184004bf9b81895d181d486e 100644 --- a/conda.environment/environment.yml +++ b/conda.environment/environment.yml @@ -4,9 +4,6 @@ channels: - pyplan dependencies: - python=3.8 - - setuptools - - wheel - - six - paramiko - celery - django=3.2 diff --git a/scripts/gitlab/setup_conda.sh b/scripts/gitlab/setup_conda.sh new file mode 100644 index 0000000000000000000000000000000000000000..3296a8ff2c305f4d0e2c991243989d0b776f548e --- /dev/null +++ b/scripts/gitlab/setup_conda.sh @@ -0,0 +1,20 @@ +#! /bin/bash +# +# Placeholders CONDA_ENV, RUNNER_HOME must be replaced +# with their actual value prior to running this script +# + +# Does the miniconda directory already exists? +if [ ! -d "$HOME/miniconda" ]; then + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + bash miniconda.sh -b -p $HOME/miniconda +fi +PATH="$HOME/miniconda/bin:$PATH" + +# Has the conda environment being created already? +if ! source activate "${CONDA_ENV}";then + conda create --name "${CONDA_ENV}" +fi + +hash -r +conda config --set always_yes yes --set changeps1 no diff --git a/setup.cfg b/setup.cfg index fa69555dc51f707cd06dde05a5c4ab6954cc8c91..92f7f881ca20417c0aae274dd11877c73b45e134 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,14 +1,40 @@ -[bumpversion] -current_version = 2.1.0 -commit = True -tag = True +[metadata] +name = django_remote_submission +description = A Django application to manage long running job submission, including starting the job, saving logs, and storing results +long_description = file: README.rst, LICENSE +version = attr: django_remote_submission.__version__ +author = T. Hobson, M. Doucet and R. M. Ferraz Leal +author_email = ferrazlealrm@ornl.gov +url = https://code.ornl.gov/reflectometry/django-remote-submission +keywords = django, ssh, django-remote-submission +license = ISC License +zip_safe = False +classifiers = + Development Status :: 3 - Alpha + Framework :: Django + Framework :: Django :: 3.2 + Intended Audience :: Developers + License :: OSI Approved :: ISC License + Natural Language :: English + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 -[bumpversion:file:setup.py] - -[bumpversion:file:django_remote_submission/__init__.py] - -[wheel] -universal = 1 +[options] +python_requires = >= 3.6 +install_requires = + paramiko + celery + django==3.2 + django-filter + django-model-utils + djangorestframework + django-channels + django-environ + asgiref +include_package_data = True +packages = django_remote_submission [flake8] exclude = @@ -16,4 +42,4 @@ exclude = build, dist, migrations -max-line-length=119 +max-line-length = 119 diff --git a/setup.py b/setup.py index 28f6c4c9a50ef5b4970e27ed5eba327ccdfcf322..36f7fe77d71ee14ca8f7dc19f56f1aaa8cebf953 100755 --- a/setup.py +++ b/setup.py @@ -1,61 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -import os -import sys -import pathlib from setuptools import setup -import pkg_resources -version = '2.1.0' - -# load dependencies from file requirements.txt -with pathlib.Path('requirements.txt').open() as requirements_txt: - install_requires = [str(requirement) for requirement in pkg_resources.parse_requirements(requirements_txt)] - -if sys.argv[-1] == 'publish': - try: - import wheel - print("Wheel version: ", wheel.__version__) - except ImportError: - print('Wheel library missing. Please run "pip install wheel"') - sys.exit() - os.system('python setup.py sdist upload') - os.system('python setup.py bdist_wheel upload') - sys.exit() - -if sys.argv[-1] == 'tag': - print("Tagging the version on git:") - os.system("git tag -a %s -m 'version %s'" % (version, version)) - os.system("git push --tags") - sys.exit() - -readme = open('README.rst').read() -history = open('HISTORY.rst').read().replace('.. :changelog:', '') - -setup( - name='django-remote-submission', - version=version, - description="""A Django application to manage long running job submission, including starting the job, saving logs, and storing results.""", - long_description=readme + '\n\n' + history, - author='T. Hobson, M. Doucet and R. M. Ferraz Leal', - author_email='ferrazlealrm@ornl.gov', - url='https://github.com/ornl-ndav/django-remote-submission', - packages=[ - 'django_remote_submission', - ], - include_package_data=True, - install_requires=install_requires, - license="ISCL", - zip_safe=False, - keywords='django-remote-submission', - classifiers=[ - 'Development Status :: 3 - Alpha', - 'Framework :: Django', - 'Framework :: Django :: 2.0', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', - 'Natural Language :: English', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - ], -) +setup()