added job queue for image upload using redis server

9decf048 · josh · 98ce7a15 · 9decf048 · 9decf048 · 9decf048
Commit 9decf048 authored Jan 28, 2021 by josh
10 changed files
--- a/HDRData.py
+++ b/HDRData.py
 import os
+import shutil

 import numpy as np

@@ -6,6 +7,10 @@ from PIL import Image
 from abc import ABC
 from pathlib import Path

+from redis import Redis
+import rq
+from rq import get_current_job
+

 '''
 HDR App database interface
@@ -211,6 +216,26 @@ class HDRFileDB( HDRDatabaseInterface ):



+    def importAndProcessImage( self, imagePath: Path ):
+        print( imagePath )
+        '''
+        - arg1: path to the image
+        - arg2: path to database
+        - arg3: new name for image 
+        '''
+        imageName = imagePath.name.split( "." )[0]
+
+        #os.system( "python ai4hdr_backend/Scripts/importAndProcessImage.py {} {} {}".format( imagePath, self.dbBasePath, imageName ) )    
+
+        imgPath = str( imagePath )
+        dbPath = str( self.dbBasePath )
+        newName = imageName
+        #importAndProcess( imgPath=imgPath, dbPath=dbPath, newImgName=newName )
+
+        queue = rq.Queue( "hdr-tasks", connection=Redis.from_url( "redis://" ) )
+        job = queue.enqueue( "tasks.importAndProcessImageTask", imgPath, dbPath, newName )
+
+
    '''
    PRIVATE METHODS
    '''

--- a/__pycache__/HDRData.cpython-36.pyc
+++ b/__pycache__/HDRData.cpython-36.pyc
--- a/__pycache__/tasks.cpython-36.pyc
+++ b/__pycache__/tasks.cpython-36.pyc
--- a/app.py
+++ b/app.py
@@ -100,14 +100,22 @@ def uploadImage():
            return redirect( request.url )
        if file and fileTypeAllowed( file.filename, ALLOWED_EXTENSIONS ):
            filename = secure_filename( file.filename )
-            tmpPath  = TEMP_LOC.joinpath( filename.split( "." )[0] )
-            os.mkdir( tmpPath )
-            # LAUNCH PROCESSING - OUTPUT -> filenameTemp
-            file.save( os.path.join( str( tmpPath ), filename ) )
-            #os.system( "importAndProcessImage.py " )
-            # SAVE RESULTS TO DATABASE
-            #imageDB.saveImage( file, filename )
-            shutil.rmtree( tmpPath )
+            tmpDir  = TEMP_LOC.joinpath( filename.split( "." )[0] )
+
+            # Add id to folder name to avoid collisions
+            ids = [int( f.name.split( "-" )[0] ) for f in TEMP_LOC.iterdir() ]
+            uId = 0  
+            if len( ids ) != 0:
+                uId = max( ids ) + 1 
+
+            # Save image to temporary dir
+            tmpDir = tmpDir.parent.joinpath( "{}-{}".format( uId, tmpDir.name ) )
+            os.mkdir( tmpDir )
+            file.save( os.path.join( str( tmpDir ), filename ) )
+
+            # Launch import and process
+            imageDB.importAndProcessImage( tmpDir.joinpath( filename ).resolve() )
+            print( "BACK TO APP" )
    return render_template( "imageUpload.html" )



--- a/launchApp.sh
+++ b/launchApp.sh
+python app.py ../ImageDatabase
\ No newline at end of file
--- a/launchWorker.sh
+++ b/launchWorker.sh
+rq worker hdr-tasks
\ No newline at end of file
--- a/misc/updateSubModule.txt
+++ b/misc/updateSubModule.txt
+cd /ai4hdr_backend
+git pull
+cd ..
+git add ai4hdr_backend
+git commit -m "Upgraded ai4hdr_backend"
\ No newline at end of file
--- a/requirements.txt
+++ b/requirements.txt
+absl-py==0.10.0
+aiohttp==3.6.2
+alabaster==0.7.12
+alpha-vantage==2.1.2
+anaconda-client==1.7.2
+anaconda-project==0.8.3
+appdirs==1.4.3
+argcomplete==1.11.1
+asn1crypto @ file:///tmp/build/80754af9/asn1crypto_1596577642040/work
+astor==0.8.1
+astroid==2.2.5
+astropy==4.0.2
+astunparse==1.6.3
+async-generator==1.10
+async-timeout==3.0.1
+atari-py==0.2.6
+atomicwrites==1.4.0
+attrs==19.1.0
+autopep8 @ file:///tmp/build/80754af9/autopep8_1596578164842/work
+Babel @ file:///tmp/build/80754af9/babel_1607110387436/work
+backcall==0.1.0
+backports.shutil-get-terminal-size==1.0.0
+beautifulsoup4==4.9.1
+bitarray @ file:///tmp/build/80754af9/bitarray_1605065125631/work
+bkcharts==0.2
+blaze==0.11.3
+bleach==1.5.0
+bokeh @ file:///tmp/build/80754af9/bokeh_1603297819746/work
+boto==2.49.0
+Bottleneck==1.3.2
+brotlipy==0.7.0
+cachetools==4.2.0
+certifi==2019.11.28
+cffi==1.14.0
+chardet==3.0.4
+click==7.1.2
+cloudpickle==1.3.0
+clyent==1.2.2
+colorama @ file:///tmp/build/80754af9/colorama_1603211150991/work
+colorlog==4.1.0
+contextlib2==0.6.0.post1
+contextvars==2.4
+cryptography @ file:///tmp/build/80754af9/cryptography_1607635303852/work
+cssselect==1.1.0
+cvxopt==1.2.3
+cycler==0.10.0
+Cython @ file:///tmp/build/80754af9/cython_1605457593821/work
+cytoolz==0.11.0
+dask @ file:///tmp/build/80754af9/dask-core_1602083700509/work
+dataclasses==0.8
+dataframe-image==0.1.1
+datashape==0.5.4
+decorator==4.4.2
+defusedxml==0.6.0
+diff-match-patch @ file:///tmp/build/80754af9/diff-match-patch_1594828741838/work
+dill==0.3.3
+distlib==0.3.0
+distributed @ file:///tmp/build/80754af9/distributed_1605066509088/work
+dm-sonnet==1.32
+docutils==0.16
+efficientnet==1.0.0
+entrypoints==0.3
+et-xmlfile==1.0.1
+fastcache==1.1.0
+filelock==3.0.12
+flake8 @ file:///tmp/build/80754af9/flake8_1601911421857/work
+Flask==1.1.2
+Flask-Cors==3.0.10
+flatbuffers==1.12
+fsspec @ file:///tmp/build/80754af9/fsspec_1602684995936/work
+future==0.18.2
+gast==0.2.2
+GetOldTweets3==0.0.11
+gevent==1.4.0
+glob2==0.7
+gmpy2==2.0.8
+google-auth==1.24.0
+google-auth-oauthlib==0.4.2
+google-pasta==0.2.0
+googleapis-common-protos==1.52.0
+greenlet @ file:///tmp/build/80754af9/greenlet_1600873993428/work
+grpcio==1.32.0
+gym==0.17.1
+h5py==2.10.0
+HeapDict==1.0.1
+html5lib==0.9999999
+hypothesis @ file:///tmp/build/80754af9/hypothesis_1607707612707/work
+idna==2.8
+idna-ssl==1.1.0
+image-classifiers==1.0.0
+imagecodecs @ file:///tmp/build/80754af9/imagecodecs_1603270457422/work
+imageio==2.8.0
+imagesize==1.2.0
+immutables @ file:///tmp/build/80754af9/immutables_1592425991038/work
+importlib-metadata==1.5.0
+importlib-resources==1.4.0
+iniconfig @ file:///tmp/build/80754af9/iniconfig_1602780191262/work
+intervaltree @ file:///tmp/build/80754af9/intervaltree_1598376443606/work
+ipykernel==5.1.4
+ipython==5.8.0
+ipython-genutils==0.2.0
+ipywidgets==7.5.1
+isort==4.3.20
+itsdangerous==1.1.0
+jax==0.1.61
+jaxlib==0.1.42
+jdcal==1.4.1
+jedi==0.16.0
+jeepney @ file:///tmp/build/80754af9/jeepney_1606148855031/work
+Jinja2==2.11.1
+joblib==0.14.1
+json5==0.9.5
+jsonschema==3.2.0
+jupyter==1.0.0
+jupyter-client==6.0.0
+jupyter-console @ file:///tmp/build/80754af9/jupyter_console_1598884538475/work
+jupyter-core==4.6.3
+jupyterlab==2.2.9
+jupyterlab-launcher @ file:///tmp/build/80754af9/jupyterlab_launcher_1601490670252/work
+jupyterlab-pygments @ file:///tmp/build/80754af9/jupyterlab_pygments_1601490720602/work
+jupyterlab-server==1.2.0
+Keras==2.1.5
+Keras-Applications==1.0.8
+Keras-Preprocessing==1.1.2
+kiwisolver==1.1.0
+lazy-object-proxy==1.4.1
+libarchive-c==2.9
+llvmlite==0.34.0
+locket==0.2.0
+lxml==4.5.0
+Markdown==3.2.1
+MarkupSafe==1.1.1
+matplotlib==3.1.1
+mccabe==0.6.1
+mistune==0.8.4
+mkl-fft==1.2.0
+mkl-random==1.1.1
+mkl-service==2.3.0
+mock==3.0.5
+more-itertools @ file:///tmp/build/80754af9/more-itertools_1605111547926/work
+mpmath==1.1.0
+msgpack @ file:///tmp/build/80754af9/msgpack-python_1607965556519/work
+multidict==4.7.6
+multipledispatch==0.6.0
+multitasking==0.0.9
+nashpy==0.0.19
+nbclient @ file:///tmp/build/80754af9/nbclient_1602783176460/work
+nbconvert==5.6.1
+nbformat==5.0.4
+nest-asyncio @ file:///tmp/build/80754af9/nest-asyncio_1606153767164/work
+networkx==2.2
+nibabel==3.2.1
+nltk==3.5
+nose @ file:///tmp/build/80754af9/nose_1606773131901/work
+notebook==6.0.3
+nox==2019.11.9
+numba @ file:///tmp/build/80754af9/numba_1600094284200/work
+numexpr @ file:///tmp/build/80754af9/numexpr_1607693743979/work
+numpy==1.18.1
+numpydoc @ file:///tmp/build/80754af9/numpydoc_1605117425582/work
+oauthlib==3.1.0
+odo==0.5.1
+olefile==0.46
+opencv-python==4.2.0.32
+openpyxl @ file:///tmp/build/80754af9/openpyxl_1598113097404/work
+opt-einsum==3.3.0
+packaging @ file:///tmp/build/80754af9/packaging_1606930849755/work
+pandas==0.25.3
+pandoc==1.0.2
+pandocfilters==1.4.2
+parso==0.6.2
+partd==1.1.0
+path @ file:///tmp/build/80754af9/path_1607537228413/work
+pathlib2 @ file:///tmp/build/80754af9/pathlib2_1607024958299/work
+pathtools==0.1.2
+patsy==0.5.1
+pbr==5.4.4
+pep8==1.7.1
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow @ file:///tmp/build/80754af9/pillow_1603822238230/work
+pip-autoremove==0.9.1
+pkginfo==1.6.1
+pluggy==0.13.1
+ply==3.11
+progressbar==2.5
+prometheus-client==0.7.1
+promise==2.3
+prompt-toolkit==1.0.18
+protobuf==3.14.0
+psutil @ file:///tmp/build/80754af9/psutil_1598370257337/work
+ptyprocess==0.6.0
+py==1.8.1
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycodestyle==2.6.0
+pycosat==0.6.3
+pycparser @ file:///tmp/build/80754af9/pycparser_1594388511720/work
+pycrypto==2.6.1
+pycurl==7.43.0.6
+pydocstyle @ file:///tmp/build/80754af9/pydocstyle_1598885001695/work
+pyflakes==2.2.0
+pyftpdlib==1.5.5
+pyglet==1.5.0
+Pygments==2.6.1
+pylint==2.3.1
+pymeanshift @ git+git://github.com/fjean/pymeanshift.git@1ba90da647342184ea7df378d7f21eba257a51d9
+pyodbc===4.0.0-unsupported
+pyOpenSSL @ file:///tmp/build/80754af9/pyopenssl_1606517880428/work
+pyparsing==2.4.6
+PyQt5-sip==12.8.1
+pyquery==1.4.1
+pyrsistent==0.15.7
+PySide2==5.15.2
+PySimpleGUI==4.33.0
+PySocks @ file:///tmp/build/80754af9/pysocks_1605305763431/work
+pytest==0.0.0
+pytest-arraydiff==0.3
+pytest-astropy==0.8.0
+pytest-astropy-header==0.1.2
+pytest-doctestplus @ file:///tmp/build/80754af9/pytest-doctestplus_1598624048543/work
+pytest-openfiles==0.5.0
+pytest-remotedata==0.3.2
+python-dateutil==2.8.1
+python-jsonrpc-server @ file:///tmp/build/80754af9/python-jsonrpc-server_1600278539111/work
+python-language-server @ file:///tmp/build/80754af9/python-language-server_1600454544709/work
+python-unibit==1.0.2
+pytz==2019.3
+PyWavelets @ file:///tmp/build/80754af9/pywavelets_1601658315997/work
+pyxdg @ file:///tmp/build/80754af9/pyxdg_1603822279816/work
+PyYAML==5.3.1
+pyzmq==19.0.0
+QDarkStyle==2.8.1
+QtAwesome @ file:///tmp/build/80754af9/qtawesome_1602272867890/work
+qtconsole==4.7.1
+QtPy==1.9.0
+redis==3.5.3
+regex==2020.9.27
+requests==2.22.0
+requests-oauthlib==1.3.0
+rope @ file:///tmp/build/80754af9/rope_1602264064449/work
+rq==1.7.0
+rsa==4.6
+ruamel-yaml==0.15.87
+scikit-image==0.17.2
+scikit-learn==0.22.2.post1
+scipy==1.4.1
+seaborn @ file:///tmp/build/80754af9/seaborn_1600553570093/work
+segmentation-models==1.0.1
+semantic-version==2.8.4
+Send2Trash==1.5.0
+shiboken2==5.15.2
+simplegeneric==0.8.1
+singledispatch @ file:///tmp/build/80754af9/singledispatch_1602523705405/work
+six==1.13.0
+snowballstemmer==2.0.0
+sortedcollections==1.2.1
+sortedcontainers @ file:///tmp/build/80754af9/sortedcontainers_1606865132123/work
+soupsieve==2.0.1
+Sphinx @ file:///tmp/build/80754af9/sphinx_1597428793432/work
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4
+sphinxcontrib-websupport @ file:///tmp/build/80754af9/sphinxcontrib-websupport_1597081412696/work
+spyder==3.2.8
+SQLAlchemy @ file:///tmp/build/80754af9/sqlalchemy_1603398008024/work
+statsmodels @ file:///tmp/build/80754af9/statsmodels_1606925348837/work
+sympy @ file:///tmp/build/80754af9/sympy_1605119506149/work
+tables==3.6.1
+tblib @ file:///tmp/build/80754af9/tblib_1597928476713/work
+teafiles==0.71.dev0
+tensorboard==2.0.2
+tensorboard-plugin-wit==1.7.0
+tensorflow-datasets==2.0.0
+tensorflow-estimator==2.0.1
+tensorflow-gpu==2.0.0
+tensorflow-metadata==0.26.0
+tensorflow-probability==0.7.0
+tensorflow-tensorboard==1.5.1
+termcolor==1.1.0
+terminado==0.8.3
+testpath==0.4.4
+testresources==2.0.1
+threadpoolctl @ file:///tmp/tmp9twdgx9k/threadpoolctl-2.1.0-py3-none-any.whl
+tifffile @ file:///tmp/build/80754af9/tifffile_1607369857969/work
+toml @ file:///tmp/build/80754af9/toml_1592853716807/work
+toolz @ file:///tmp/build/80754af9/toolz_1601054250827/work
+torch==1.6.0+cu101
+torchvision==0.7.0+cu101
+tornado==6.0.4
+tqdm==4.43.0
+traitlets==4.3.3
+typed-ast==1.4.0
+typing-extensions==3.7.4.3
+ujson @ file:///tmp/build/80754af9/ujson_1602523318190/work
+unicodecsv==0.14.1
+urllib3==1.25.7
+virtualenv==20.0.13
+wcwidth==0.1.8
+webencodings==0.5.1
+Werkzeug==1.0.0
+widgetsnbextension==3.5.1
+wrapt==1.12.1
+xlrd==1.2.0
+XlsxWriter @ file:///tmp/build/80754af9/xlsxwriter_1602692860603/work
+xlwt==1.3.0
+yapf @ file:///tmp/build/80754af9/yapf_1593528177422/work
+yarl==1.6.0
+yfinance==0.1.52
+zict==2.0.0
+zipp==3.1.0
--- a/tasks.py
+++ b/tasks.py
+import time
+import shutil
+
+from ai4hdr_backend.Scripts import ai4hdr_utils 
+
+from pathlib import Path
+from rq import get_current_job
+
+# SOURCE - https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-xxii-background-jobs
+
+def example(seconds):
+    job = get_current_job()
+    print('Starting task')
+    for i in range(seconds):
+        job.meta['progress'] = 100.0 * i / seconds
+        job.save_meta()
+        print(i)
+        time.sleep(1)
+    job.meta['progress'] = 100
+    job.save_meta()
+    print('Task completed')
+
+
+
+def importAndProcessImageTask( imagePath, dbPath, newName ):
+    print( imagePath )
+    print( dbPath )
+    print( newName )
+
+    ai4hdr_utils.importAndProcess( imagePath, dbPath, newName )
+
+    #example( 5 )
+
+    shutil.rmtree( Path( imagePath ).parent )
--- a/test.jpg
+++ b/test.jpg