Commit dde2c700 authored by Walsh, Michael's avatar Walsh, Michael
Browse files

Merge branch 'ref120_upgrade_python_django_postgresql' into 'next'

Ref120 upgrade python django postgresql

See merge request !11
parents 9328a0d0 33a373b5
Pipeline #180120 failed with stages
in 12 minutes and 12 seconds
......@@ -10,17 +10,46 @@ without actually executing the Refl1D jobs.
Running a test server
You can easily deploy and test this application using Conda environments. The ``webrefl_env.yml`` file describes
an environment where we added the ``refl1d`` dependency for local computations, and where we use ``sqlite`` as our database::
You can easily deploy and test this application using Conda environments. The ``webrefl_env_py3.yml`` file describes
an environment where we added the ``refl1d`` dependency for local computations, and where we use ``postgresql`` as our database::
cd test/environment
cd web_reflectivity/test/environment
conda env create -f webrefl_env.yml
conda env create -f webrefl_env_py3.yml
source activate webrefl
source activate webrefl_py3
The application depends on `redis <>`_, which you may have to install separately.
You will also want to install Django separately.
According to `the documentation of Django 3.2 <>`_, the latest version of PostgreSQL Django supports is 9.6
Once you have Django 9.6 installed, you will then want to add a user with database create permissions, and a database to your local install using the cli tool psql.
These will be used in your
.. code-block:: python
'default': {
'ENGINE': 'django.db.backends.postgresql',
'NAME': os.environ.get('DATABASE_NAME'),
'USER': os.environ.get('DATABASE_USER'),
'PASSWORD': os.environ.get('DATABASE_PASS'),
'HOST': 'localhost',
'PORT': '5432',
Additionally in your you will also want to set
.. code-block:: python
CELERY_BROKER_URL = 'redis://'
SECRET_KEY = `some random value, cannot be blank
This is so it points to your local Celery instance without DNS, and the secret key is a hard requirement by the frameworks even if it isnt used on your local.
The default installation directory is ``/var/www/web_reflectivity``.
Make sure you can write in the installation directory::
......@@ -30,18 +59,18 @@ Make sure ``datahandler`` is one of the INSTALLED_APPS in ````.
Install the code::
cd web_reflectivity/src
make install
Create a test user::
cd /var/www/web_reflectivity/app; python createsuperuser --username testuser
make create_test_user
Start redis and celery, and the test server::
cd /var/www/web_reflectivity/app; celery -A fitting.celery worker --loglevel=debug
cd /var/www/web_reflectivity/app; python runserver
make start_test_server
Using Refl1D
......@@ -61,7 +90,7 @@ Running system tests
You can run the system tests included in the code by running::
python test
make test
.. toctree::
web_reflectivity/ export-subst
include web_reflectivity/
prefix = /var/www/web_reflectivity
prefix = $(REFL_INSTALL_DIR)
app_dir := .
ifndef PYTHON
PYTHON := python
@echo "Run make install to install the live data server"
# Check dependencies
@python -c "import django" || echo "\nERROR: Django is not installed:\n"
@python -c "import psycopg2" || echo "\nWARNING: psycopg2 is not installed:\n"
@python -c "import cython" || echo "\nWARNING: cython is not installed\n"
@python -c "import pandas" || echo "\nWARNING: pandas is not installed\n"
@python -c "import plotly" || echo "\nWARNING: plotly is not installed\n"
@python -c "import plotly.offline" || echo "\nWARNING: plotly.offline is not installed\n"
$(PYTHON) -m pip install -r requirements_py3.txt
install: webapp
# Make sure the install directories exist
test -d $(prefix) || mkdir -m 0755 -p $(prefix)
test -d $(prefix)/app || mkdir -m 0755 $(prefix)/app
test -d $(prefix)/app/db || mkdir -m 0755 $(prefix)/app/db
test -d $(prefix)/static || mkdir -m 0755 $(prefix)/static
test -d $(prefix)/media || mkdir -m 0755 $(prefix)/media
# Install application code
/bin/cp $(app_dir)/ $(prefix)/app
/bin/cp -R $(app_dir)/web_reflectivity $(prefix)/app
/bin/cp -R $(app_dir)/templates $(prefix)/app
/bin/cp -R $(app_dir)/fitting $(prefix)/app
/bin/cp -R $(app_dir)/datahandler $(prefix)/app
/bin/cp -R $(app_dir)/users $(prefix)/app
/bin/cp -R $(app_dir)/tools $(prefix)/app
/bin/cp -R $(app_dir)/static $(prefix)/app
# Copy test data for automated build/tests
/bin/cp -R ../test/test_data.txt $(prefix)/app
webapp: webapp/core
# Collect the static files and install them
cd $(prefix)/app; python3 collectstatic --noinput
# Create migrations and apply them
cd $(prefix)/app; python3 makemigrations
cd $(prefix)/app; python3 migrate
# Prepare web monitor cache: RUN THIS ONCE BY HAND
#cd $(prefix)/app; python3 createcachetable webcache
@echo "\n\nReady to go: run apachectl restart\n"
first_install: webapp/core
# Modify and copy the wsgi configuration
/bin/cp apache/apache_django_wsgi.conf /etc/apache2/sites-available/
apachectl restart
/sbin/service redis restart
/sbin/service celery restart
# Create deploy directory as root since it's in /var/www
test -d $(prefix) || sudo mkdir -m 0755 -p $(prefix); sudo chown $(shell whoami) $(prefix);
cd $(prefix)/app; celery -A fitting.celery worker --loglevel=debug
cd $(prefix)/app; python runserver
cd /var/www/web_reflectivity/app; python3 createsuperuser --username testuser
cd $(prefix)/app; python test
docker build -t webi:v0.1 -f Dockerfile_local .
docker volume create webi_db
docker run -p 8000:8000 --mount source=webi_db,target=/var/www/web_reflectivity/app/db webi:v0.1
.PHONY: start
.PHONY: check
.PHONY: install
.PHONY: webapp
.PHONY: webapp/core
.PHONY: first_install
.PHONY: test
.PHONY: create_app_dir
.PHONY: start_test_server
\ No newline at end of file
......@@ -3,7 +3,7 @@
import json
import tempfile
from django.test import TestCase
from django.test import TransactionTestCase
from django.test import Client
from django.contrib.auth.models import User
from django.forms import model_to_dict
......@@ -16,7 +16,8 @@ from . import job_handling
from .parsing import refl1d, refl1d_err_model, refl1d_simultaneous
from .simultaneous import model_handling
class UserTestCase(TestCase):
class UserTestCase(TransactionTestCase):
reset_sequences = True
def setUp(self):
# Every test needs a client.
self.client = Client()
......@@ -47,14 +48,17 @@ class UserTestCase(TestCase):
self.assertEqual(response.status_code, 301)
class TestDataHandling(TestCase):
class TestDataHandling(TransactionTestCase):
reset_sequences = True
def test_key(self):
""" Test the generation of secret key for a data set """
with self.settings(LIVE_PLOT_SECRET_KEY='1234'):
url_with_key = dh.append_key('/', 'refl', 1)
self.assertEqual(url_with_key, '/?key=3b9a06c28c5b1c0ae87c2bd05ea8603b9b9c0c31')
class FileTestCase(TestCase):
class FileTestCase(TransactionTestCase):
reset_sequences = True
""" File handling tests """
def setUp(self):
# Every test needs a client.
......@@ -111,7 +115,8 @@ class FileTestCase(TestCase):
user_data = UserData.objects.get(user=self.user, file_id=1)
self.assertEqual(user_data.tags, "bunch of tags")
class FitSubmitTestCase(TestCase):
class FitSubmitTestCase(TransactionTestCase):
reset_sequences = True
""" Test submission of a model and file handling """
def setUp(self):
# Every test needs a client.
......@@ -163,7 +168,8 @@ class FitSubmitTestCase(TestCase):
# We get a 302 because of a redirect
self.assertEqual(response.status_code, 302)
class SimultaneousViewsTestCase(TestCase):
class SimultaneousViewsTestCase(TransactionTestCase):
reset_sequences = True
""" Test simultaneous fitting """
def setUp(self):
# Every test needs a client.
......@@ -235,7 +241,8 @@ class SimultaneousViewsTestCase(TestCase):
script = items[0].get_constraint(sample_name='sample')
self.assertEqual(script, "sample1['material'].material.rho = sample2['material'].material.rho")
class FitProblemViewsTestCase(TestCase):
class FitProblemViewsTestCase(TransactionTestCase):
reset_sequences = True
""" Test functionality related to fits """
def setUp(self):
# Every test needs a client.
......@@ -431,7 +438,8 @@ class FitProblemViewsTestCase(TestCase):
self.assertEqual(fit_problem.reflectivity_model.back_sld, 0)
class FitterOptionsTestCase(TestCase):
class FitterOptionsTestCase(TransactionTestCase):
reset_sequences = True
""" Test fitter options """
def setUp(self):
# Every test needs a client.
......@@ -453,7 +461,8 @@ class FitterOptionsTestCase(TestCase):
option_obj = FitterOptions.objects.get(user=self.user)
self.assertEqual(option_obj.steps, 500)
class ParsingTestCase(TestCase):
class ParsingTestCase(TransactionTestCase):
reset_sequences = True
""" Test refl1d result parsers """
def setUp(self):
# Every test needs a client.
......@@ -584,7 +593,8 @@ Done: 2.40629 sec
data, _ = refl1d_err_model.parse_slabs(self.log)
self.assertEqual(data[0][0]['chi2'], '108.1844')
class SimultaneousParsingTestCase(TestCase):
class SimultaneousParsingTestCase(TransactionTestCase):
reset_sequences = True
""" Test refl1d result parsers for simultaneous fits"""
def setUp(self):
self.log = """SIMULTANEOUS ["ref_l/154461","ref_l/154461","ref_l/157294"]
......@@ -818,7 +828,8 @@ Done: 15.7372 sec
self.assertEqual(len(data), 3)
class ToolsTestCase(TestCase):
class ToolsTestCase(TransactionTestCase):
reset_sequences = True
""" Test the SLD and capacity tools """
def setUp(self):
# Every test needs a client.
......@@ -846,7 +857,8 @@ class ToolsTestCase(TestCase):
self.assertEqual(response.context['capacity'], ' 0.786')
class JsonParsingTestCase(TestCase):
class JsonParsingTestCase(TransactionTestCase):
reset_sequences = True
""" Test JSON parsing """
def setUp(self):
# Every test needs a client.
......@@ -902,14 +914,16 @@ Done: 0.981189 sec
self.assertTrue(len(r_plot) > 0)
class CatalogTestCase(TestCase):
class CatalogTestCase(TransactionTestCase):
reset_sequences = True
def test_oncat(self):
from . import catalog
self.assertTrue(catalog.decode_time('2018-10-12T00:00:00-06:39') is not None)
self.assertEqual(catalog.get_run_info('john', 1), {})
class ICATTestCase(TestCase):
class ICATTestCase(TransactionTestCase):
reset_sequences = True
def test_icat(self):
from . import icat_server_communication
self.assertEqual(icat_server_communication.get_run_info('john', 1), {})
VCS = git
style = pep440
versionfile_source = web_reflectivity/
versionfile_build = web_reflectivity/
tag_prefix = v
statistics = True
count = True
from setuptools import setup
import versioneer
\ No newline at end of file
This diff is collapsed.
__version__ = "1.2.0"
\ No newline at end of file
from . import _version
__version__ = _version.get_versions()['version']
This diff is collapsed.
name: webrefl_py3
- python==3.7.0
- numpy
- sqlite
- redis
- celery[redis]
- pip==21.3.1
- pip:
- django
- celery
- redis
- django-celery-results
- git+
- django-celery-results==2.2.0
- git+
- cython
- pandas
- plotly
- psycopg2-binary>=2.8
- scipy
- numba
- requests
- periodictable
- refl1d
- matplotlib
- channels_redis
- channels-redis
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment