modularizing the ml pipeline

e68b397c · josh · 77a69531 · e68b397c · e68b397c · e68b397c
Commit e68b397c authored Feb 04, 2021 by josh
--- a/.ipynb_checkpoints/Untitled-checkpoint.ipynb
+++ b/.ipynb_checkpoints/Untitled-checkpoint.ipynb
 {
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from modules.ai4hdrModel import *\n",
-    "\n",
-    "import matplotlib.pyplot as plt"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "PosixPath('/home/josh/projects/suli_fall2021/ai4hdr_backend/roadTest')"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "sampleDir = Path.cwd().joinpath(\"roadTest\")\n",
-    "sampleDir"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "(10, 128, 128, 3)\n",
-      "(10, 128, 128)\n"
-     ]
-    }
-   ],
-   "source": [
-    "(xSamples, ySamples) = getSamples(sampleDir)\n",
-    "print(xSamples.shape)\n",
-    "print(ySamples.shape)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "X TRAIN SHAPE: (5, 128, 128, 3)\n",
-      "Y TRAIN SHAPE: (5, 128, 128)\n",
-      "X TEST SHAPE: (5, 128, 128, 3)\n",
-      "Y TEST SHAPE: (5, 128, 128)\n"
-     ]
-    }
-   ],
-   "source": [
-    "xTrain = xSamples[0:5,:,:,:]\n",
-    "yTrain = ySamples[0:5,:,:]\n",
-    "print(\"X TRAIN SHAPE:\", xTrain.shape)\n",
-    "print(\"Y TRAIN SHAPE:\", yTrain.shape)\n",
-    "\n",
-    "xTest = xSamples[5:10,:,:,:]\n",
-    "yTest = ySamples[5:10,:,:]\n",
-    "print(\"X TEST SHAPE:\", xTest.shape)\n",
-    "print(\"Y TEST SHAPE:\", yTest.shape)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "gpu_tf",
-   "language": "python",
-   "name": "gpu_tf"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.6.9"
-  }
- },
+ "cells": [],
+ "metadata": {},
 "nbformat": 4,
 "nbformat_minor": 4
 }
-%% Cell type:code id: tags:
-
-``` python
-from modules.ai4hdrModel import *
-
-import matplotlib.pyplot as plt
-```
-
-%% Cell type:code id: tags:
-
-``` python
-sampleDir = Path.cwd().joinpath("roadTest")
-sampleDir
-```
-
-%% Output
-
-    PosixPath('/home/josh/projects/suli_fall2021/ai4hdr_backend/roadTest')
-
-%% Cell type:code id: tags:
-
-``` python
-(xSamples, ySamples) = getSamples(sampleDir)
-print(xSamples.shape)
-print(ySamples.shape)
-```
-
-%% Output
-
-    (10, 128, 128, 3)
-    (10, 128, 128)
-
-%% Cell type:markdown id: tags:
-
-#
-
-%% Cell type:code id: tags:
-
-``` python
-xTrain = xSamples[0:5,:,:,:]
-yTrain = ySamples[0:5,:,:]
-print("X TRAIN SHAPE:", xTrain.shape)
-print("Y TRAIN SHAPE:", yTrain.shape)
-
-xTest = xSamples[5:10,:,:,:]
-yTest = ySamples[5:10,:,:]
-print("X TEST SHAPE:", xTest.shape)
-print("Y TEST SHAPE:", yTest.shape)
-```
-
-%% Output
-
-    X TRAIN SHAPE: (5, 128, 128, 3)
-    Y TRAIN SHAPE: (5, 128, 128)
-    X TEST SHAPE: (5, 128, 128, 3)
-    Y TEST SHAPE: (5, 128, 128)
-
-%% Cell type:code id: tags:
-
-``` python
-```
-
-%% Cell type:code id: tags:
-
-``` python
-```
-
-%% Cell type:code id: tags:
-
-``` python
-```

--- a/HDRModelTest.ipynb
+++ b/HDRModelTest.ipynb
--- a/Untitled.ipynb
+++ b/Untitled.ipynb
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pickle\n",
+    "from pathlib import Path\n",
+    "\n",
+    "\n",
+    "from tensorflow import keras"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "OSError",
+     "evalue": "SavedModel file does not exist at: VGG16/{saved_model.pbtxt|saved_model.pb}",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-13-164cb73ad621>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mVGG_DIR\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"VGG16\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mVGG_DIR\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m/home/josh/anaconda3/envs/gpu_tf/lib/python3.6/site-packages/tensorflow_core/python/keras/saving/save.py\u001b[0m in \u001b[0;36mload_model\u001b[0;34m(filepath, custom_objects, compile)\u001b[0m\n\u001b[1;32m    147\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    148\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstring_types\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 149\u001b[0;31m     \u001b[0mloader_impl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse_saved_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    150\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0msaved_model_load\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/home/josh/anaconda3/envs/gpu_tf/lib/python3.6/site-packages/tensorflow_core/python/saved_model/loader_impl.py\u001b[0m in \u001b[0;36mparse_saved_model\u001b[0;34m(export_dir)\u001b[0m\n\u001b[1;32m     81\u001b[0m                   (export_dir,\n\u001b[1;32m     82\u001b[0m                    \u001b[0mconstants\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSAVED_MODEL_FILENAME_PBTXT\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m                    constants.SAVED_MODEL_FILENAME_PB))\n\u001b[0m\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mOSError\u001b[0m: SavedModel file does not exist at: VGG16/{saved_model.pbtxt|saved_model.pb}"
+     ]
+    }
+   ],
+   "source": [
+    "VGG_DIR = \"VGG16\"\n",
+    "model = keras.models.load_model(VGG_DIR)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "UnpicklingError",
+     "evalue": "invalid load key, '\\x0a'.",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mUnpicklingError\u001b[0m                           Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-7-9fbfa10e9190>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minFile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;31mUnpicklingError\u001b[0m: invalid load key, '\\x0a'."
+     ]
+    }
+   ],
+   "source": [
+    "obj = pickle.load(inFile)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "gpu_tf",
+   "language": "python",
+   "name": "gpu_tf"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
+%% Cell type:code id: tags:
+
+``` python
+import pickle
+from pathlib import Path
+
+
+from tensorflow import keras
+```
+
+%% Cell type:code id: tags:
+
+``` python
+VGG_DIR = "VGG16"
+model = keras.models.load_model(VGG_DIR)
+```
+
+%% Output
+
+    ---------------------------------------------------------------------------
+    OSError                                   Traceback (most recent call last)
+    <ipython-input-13-164cb73ad621> in <module>()
+          1 VGG_DIR = "VGG16"
+    ----> 2 model = keras.models.load_model(VGG_DIR)
+
+    /home/josh/anaconda3/envs/gpu_tf/lib/python3.6/site-packages/tensorflow_core/python/keras/saving/save.py in load_model(filepath, custom_objects, compile)
+        147
+        148   if isinstance(filepath, six.string_types):
+    --> 149     loader_impl.parse_saved_model(filepath)
+        150     return saved_model_load.load(filepath, compile)
+        151
+    /home/josh/anaconda3/envs/gpu_tf/lib/python3.6/site-packages/tensorflow_core/python/saved_model/loader_impl.py in parse_saved_model(export_dir)
+         81                   (export_dir,
+         82                    constants.SAVED_MODEL_FILENAME_PBTXT,
+    ---> 83                    constants.SAVED_MODEL_FILENAME_PB))
+         84
+         85
+    OSError: SavedModel file does not exist at: VGG16/{saved_model.pbtxt|saved_model.pb}
+
+%% Cell type:code id: tags:
+
+``` python
+obj = pickle.load(inFile)
+```
+
+%% Output
+
+    ---------------------------------------------------------------------------
+    UnpicklingError                           Traceback (most recent call last)
+    <ipython-input-7-9fbfa10e9190> in <module>()
+    ----> 1 obj = pickle.load(inFile)
+
+    UnpicklingError: invalid load key, '\x0a'.
+
+%% Cell type:code id: tags:
+
+``` python
+```
--- a/trainModel.py
+++ b/trainModel.py
@@ -18,10 +18,6 @@ def getSamples(dataDir: Path) -> [(np.array, np.array)]:
        inputImage = Image.open(inputPath)
        maskImage  = Image.open(maskPath)
        
-        #newSize = (128, 128)
-        #resizeInput = inputImage.resize(newSize)
-        #resizeMask  = maskImage.resize(newSize)
-        
        inputArr = np.array(inputImage)
        maskArr  = np.array(maskImage.convert("L"))