update training process

fc43603f · Yifan Wang · b059bb8b · fc43603f · fc43603f
Commit fc43603f authored Aug 16, 2021 by Yifan Wang
--- a/workspace/2-data_csv.ipynb
+++ b/workspace/2-data_csv.ipynb
@@ -2,7 +2,7 @@
 "cells": [
  {
   "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -10,41 +10,43 @@
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
-    "import utils.samplefiles"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "train_wnum = 50\n",
-    "train_nnum = 50\n",
-    "test_wnum = 50\n",
-    "test_nnum = 50"
+    "import utils.samplefiles\n",
+    "import h5py"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = utils.samplefiles.SampleFile()\n",
-    "data.read_hdf('./output/train.hdf')"
+    "data.read_hdf('./output/train.hdf')\n",
+    "\n",
+    "wave, noise = data.as_dataframe(injection_parameters=True, \n",
+    "                  static_arguments=False, \n",
+    "                  command_line_arguments=False, \n",
+    "                  split_injections_noise=True)"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 23,
   "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "4096"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
   "source": [
-    "wave, noise = data.as_dataframe(injection_parameters=True, \n",
-    "                  static_arguments=False, \n",
-    "                  command_line_arguments=False, \n",
-    "                  split_injections_noise=True)"
+    "wave['h1_strain'][0].size"
   ]
  },
  {
@@ -56,7 +58,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -66,6 +68,26 @@
    "nary = np.array(h1n)"
   ]
  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "4096"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "h1w[0].size"
+   ]
+  },
  {
   "cell_type": "markdown",
   "metadata": {},
@@ -75,16 +97,41 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
+    "train_wnum = 50\n",
+    "train_nnum = 50\n",
+    "test_wnum = 50\n",
+    "test_nnum = 50\n",
+    "\n",
    "wtrain = wary[:train_wnum,:]\n",
    "ntrain = nary[:train_nnum,:]\n",
    "wtest = wary[train_wnum:,:]\n",
    "ntest = nary[train_nnum:,:]"
   ]
  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(50, 4096)"
+      ]
+     },
+     "execution_count": 27,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "wtrain.shape"
+   ]
+  },
  {
   "cell_type": "markdown",
   "metadata": {},
@@ -94,7 +141,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -108,25 +155,25 @@
   "cell_type": "markdown",
   "metadata": {},
   "source": [
-    "Training set name"
+    "# Training set"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
-      "100%|██████████| 50/50 [00:00<00:00, 299593.14it/s]\n"
+      "100%|██████████| 4096/4096 [00:00<00:00, 774600.71it/s]\n"
     ]
    }
   ],
   "source": [
    "train_name = []\n",
-    "num = 50\n",
+    "num = wtrain.shape[1]-1 # 4096\n",
    "train_name.append('label')\n",
    "for i in tqdm(range(0,num)):\n",
    "    train_name.append('point{s1}'.format(s1=i))"
@@ -134,14 +181,14 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
-      "100%|██████████| 50/50 [00:00<00:00, 63.12it/s]\n"
+      "100%|██████████| 50/50 [00:00<00:00, 120.83it/s]\n"
     ]
    }
   ],
@@ -156,29 +203,58 @@
    "        writer.writerow(ntrain[i])"
   ]
  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_set = pd.read_csv(\"./output/train.csv\", dtype=np.float32)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(100, 4097)"
+      ]
+     },
+     "execution_count": 33,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_set.shape"
+   ]
+  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
-    "testing set name"
+    "# Testing set"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
-      "100%|██████████| 50/50 [00:00<00:00, 394201.50it/s]\n"
+      "100%|██████████| 4096/4096 [00:00<00:00, 457568.56it/s]\n"
     ]
    }
   ],
   "source": [
    "test_name = []\n",
-    "num = 50\n",
+    "num = wtrain.shape[1]-1 # 4096\n",
    "test_name.append('label')\n",
    "for i in tqdm(range(0,num)):\n",
    "    test_name.append('point{s1}'.format(s1=i))"
@@ -186,14 +262,14 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
-      "100%|██████████| 50/50 [00:00<00:00, 60.24it/s]\n"
+      "100%|██████████| 50/50 [00:00<00:00, 120.86it/s]\n"
     ]
    }
   ],
@@ -208,6 +284,35 @@
    "        writer.writerow(ntest[i])"
   ]
  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_set = pd.read_csv(\"./output/test.csv\", dtype=np.float32)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(100, 4097)"
+      ]
+     },
+     "execution_count": 38,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "test_set.shape"
+   ]
+  },
  {
   "cell_type": "code",
   "execution_count": null,
@@ -232,7 +337,7 @@
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
-   "version": "3.9.1"
+   "version": "3.8.10"
  }
 },
 "nbformat": 4,

 %% Cell type:code id: tags:

 ``` python
 import csv
 import numpy as np
 import pandas as pd
 from tqdm import tqdm
 import utils.samplefiles
-```
-
-%% Cell type:code id: tags:
-
-``` python
-train_wnum = 50
-train_nnum = 50
-test_wnum = 50
-test_nnum = 50
+import h5py
 ```

 %% Cell type:code id: tags:

 ``` python
 data = utils.samplefiles.SampleFile()
 data.read_hdf('./output/train.hdf')
-```
-
-%% Cell type:code id: tags:

-``` python
 wave, noise = data.as_dataframe(injection_parameters=True,
                  static_arguments=False,
                  command_line_arguments=False,
                  split_injections_noise=True)
 ```

+%% Cell type:code id: tags:
+
+``` python
+wave['h1_strain'][0].size
+```
+
+%% Output
+
+    4096
+
 %% Cell type:markdown id: tags:

 Turn strain into multi-dimension array

 %% Cell type:code id: tags:

 ``` python
 h1w = wave['h1_strain'].tolist()
 h1n = noise['h1_strain'].tolist()
 wary = np.array(h1w)
 nary = np.array(h1n)
 ```

+%% Cell type:code id: tags:
+
+``` python
+h1w[0].size
+```
+
+%% Output
+
+    4096
+
 %% Cell type:markdown id: tags:

 Split train and test set

 %% Cell type:code id: tags:

 ``` python
+train_wnum = 50
+train_nnum = 50
+test_wnum = 50
+test_nnum = 50
+
 wtrain = wary[:train_wnum,:]
 ntrain = nary[:train_nnum,:]
 wtest = wary[train_wnum:,:]
 ntest = nary[train_nnum:,:]
 ```

+%% Cell type:code id: tags:
+
+``` python
+wtrain.shape
+```
+
+%% Output
+
+    (50, 4096)
+
 %% Cell type:markdown id: tags:

 Insert label

 %% Cell type:code id: tags:

 ``` python
 wtrain = np.insert(wtrain, 0, values=1, axis=1)
 ntrain = np.insert(ntrain, 0, values=0, axis=1)
 wtest = np.insert(wtest, 0, values=1, axis=1)
 ntest = np.insert(ntest, 0, values=0, axis=1)
 ```

 %% Cell type:markdown id: tags:

-Training set name
+# Training set

 %% Cell type:code id: tags:

 ``` python
 train_name = []
-num = 50
+num = wtrain.shape[1]-1 # 4096
 train_name.append('label')
 for i in tqdm(range(0,num)):
    train_name.append('point{s1}'.format(s1=i))
 ```

 %% Output

-    100%|██████████| 50/50 [00:00<00:00, 299593.14it/s]
+    100%|██████████| 4096/4096 [00:00<00:00, 774600.71it/s]

 %% Cell type:code id: tags:

 ``` python
 with open("output/train.csv","w") as csvfile:
    writer = csv.writer(csvfile)
    #columns_name
    writer.writerow(train_name)
    #use writerows to write lines
    for i in tqdm(range(0,train_wnum)):
        writer.writerow(wtrain[i])
        writer.writerow(ntrain[i])
 ```

 %% Output

-    100%|██████████| 50/50 [00:00<00:00, 63.12it/s]
+    100%|██████████| 50/50 [00:00<00:00, 120.83it/s]
+
+%% Cell type:code id: tags:
+
+``` python
+train_set = pd.read_csv("./output/train.csv", dtype=np.float32)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+train_set.shape
+```
+
+%% Output
+
+    (100, 4097)

 %% Cell type:markdown id: tags:

-testing set name
+# Testing set

 %% Cell type:code id: tags:

 ``` python
 test_name = []
-num = 50
+num = wtrain.shape[1]-1 # 4096
 test_name.append('label')
 for i in tqdm(range(0,num)):
    test_name.append('point{s1}'.format(s1=i))
 ```

 %% Output

-    100%|██████████| 50/50 [00:00<00:00, 394201.50it/s]
+    100%|██████████| 4096/4096 [00:00<00:00, 457568.56it/s]

 %% Cell type:code id: tags:

 ``` python
 with open("output/test.csv","w") as csvfile:
    writer = csv.writer(csvfile)
    #columns_name
    writer.writerow(test_name)
    #use writerows to write lines
    for i in tqdm(range(0,test_wnum)):
        writer.writerow(wtest[i])
        writer.writerow(ntest[i])
 ```

 %% Output

-    100%|██████████| 50/50 [00:00<00:00, 60.24it/s]
+    100%|██████████| 50/50 [00:00<00:00, 120.86it/s]
+
+%% Cell type:code id: tags:
+
+``` python
+test_set = pd.read_csv("./output/test.csv", dtype=np.float32)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+test_set.shape
+```
+
+%% Output
+
+    (100, 4097)

 %% Cell type:code id: tags:

 ``` python
 ```

--- a/workspace/3-training.ipynb
+++ b/workspace/3-training.ipynb
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import torch.nn as nn\n",
+    "import torch.optim as optim\n",
+    "import matplotlib.pyplot as plt\n",
+    "import torch.nn.functional as F\n",
+    "from torchsummary import summary\n",
+    "from torch.optim import lr_scheduler\n",
+    "from torch.utils.data import DataLoader\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from models import *\n",
+    "%matplotlib inline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# hyper-parameters\n",
+    "# how many samples per batch to load\n",
+    "batch_size = 10\n",
+    "# percentage of training set to use as validation\n",
+    "valid_size = 0.1\n",
+    "# number of epochs to train the model\n",
+    "n_epochs = 30\n",
+    "# track change in validation loss\n",
+    "valid_loss_min = np.Inf\n",
+    "# specify the image classes\n",
+    "classes = ['noise', 'wave']\n",
+    "# gpu\n",
+    "DEVICE = torch.device('cuda: 3' if torch.cuda.is_available() else 'cpu')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# choose the training and test datasets\n",
+    "train_set = pd.read_csv(\"./output/train.csv\", dtype=np.float32)\n",
+    "\n",
+    "# Seperate the features and labels\n",
+    "total_train_label = train_set.label.values\n",
+    "total_train_data = train_set.loc[:, train_set.columns != 'label'].values\n",
+    "total_train_data  = total_train_data.reshape(-1, 1, 4096)\n",
+    "\n",
+    "# Split into training and test set\n",
+    "data_train, data_valid, label_train, label_valid = train_test_split(total_train_data, total_train_label, test_size=0.1, random_state=2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# create feature and targets tensor for train set. As you remember we need variable to accumulate gradients. Therefore first we create tensor, then we will create variable\n",
+    "dataTrain = torch.from_numpy(data_train)\n",
+    "labelTrain = torch.from_numpy(label_train).type(torch.LongTensor) # data type is long\n",
+    "\n",
+    "# create feature and targets tensor for valid set.\n",
+    "dataValid = torch.from_numpy(data_valid)\n",
+    "labelValid = torch.from_numpy(label_valid).type(torch.LongTensor) # data type is long"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Pytorch train and valid sets\n",
+    "train = torch.utils.data.TensorDataset(dataTrain,labelTrain)\n",
+    "valid = torch.utils.data.TensorDataset(dataValid,labelValid)\n",
+    "\n",
+    "# data loader\n",
+    "train_loader = torch.utils.data.DataLoader(train, batch_size = batch_size, shuffle = True)\n",
+    "valid_loader = torch.utils.data.DataLoader(valid, batch_size = batch_size, shuffle = True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# instantiate model\n",
+    "model =  ConvNet4().to(DEVICE)\n",
+    "# specify optimizer\n",
+    "optimizer = optim.Adam(model.parameters(), lr=5e-5)\n",
+    "#learning rate\n",
+    "lr_sched = lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.1)\n",
+    "# summary\n",
+    "#summary(model, input_size=(1,4096))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#curve list\n",
+    "train_Loss_list = []\n",
+    "valid_Loss_list = []\n",
+    "accracy_list = []\n",
+    "valid_len = 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Validation loss decreased (0.638117 --> 0.638116).  Saving model ...\n",
+      "Validation loss decreased (0.638116 --> 0.638115).  Saving model ...\n",
+      "Validation loss decreased (0.638115 --> 0.638113).  Saving model ...\n",
+      "Validation loss decreased (0.638113 --> 0.638113).  Saving model ...\n",
+      "iteration: 5.0 \tTraining Loss: 0.241428 \tValidation Loss: 0.638112\n",
+      "Validation loss decreased (0.638113 --> 0.638112).  Saving model ...\n",
+      "Validation loss decreased (0.638112 --> 0.638110).  Saving model ...\n",
+      "Validation loss decreased (0.638110 --> 0.638110).  Saving model ...\n",
+      "iteration: 15.0 \tTraining Loss: 0.278747 \tValidation Loss: 0.638117\n",
+      "iteration: 25.0 \tTraining Loss: 0.217533 \tValidation Loss: 0.638129\n",
+      "iteration: 35.0 \tTraining Loss: 0.268120 \tValidation Loss: 0.638133\n",
+      "iteration: 45.0 \tTraining Loss: 0.242722 \tValidation Loss: 0.638136\n",
+      "iteration: 55.0 \tTraining Loss: 0.210212 \tValidation Loss: 0.638121\n",
+      "iteration: 65.0 \tTraining Loss: 0.222743 \tValidation Loss: 0.638116\n",
+      "iteration: 75.0 \tTraining Loss: 0.180599 \tValidation Loss: 0.638120\n",
+      "iteration: 85.0 \tTraining Loss: 0.219321 \tValidation Loss: 0.638117\n",
+      "iteration: 95.0 \tTraining Loss: 0.218023 \tValidation Loss: 0.638128\n",
+      "iteration: 105.0 \tTraining Loss: 0.212116 \tValidation Loss: 0.638129\n",
+      "iteration: 115.0 \tTraining Loss: 0.226289 \tValidation Loss: 0.638129\n",
+      "iteration: 125.0 \tTraining Loss: 0.266237 \tValidation Loss: 0.638129\n",
+      "iteration: 135.0 \tTraining Loss: 0.228767 \tValidation Loss: 0.638129\n",
+      "iteration: 145.0 \tTraining Loss: 0.249504 \tValidation Loss: 0.638129\n",
+      "iteration: 155.0 \tTraining Loss: 0.242986 \tValidation Loss: 0.638129\n",
+      "iteration: 165.0 \tTraining Loss: 0.222421 \tValidation Loss: 0.638129\n",
+      "iteration: 175.0 \tTraining Loss: 0.195560 \tValidation Loss: 0.638129\n",
+      "iteration: 185.0 \tTraining Loss: 0.228540 \tValidation Loss: 0.638129\n",
+      "iteration: 195.0 \tTraining Loss: 0.217326 \tValidation Loss: 0.638129\n",
+      "iteration: 205.0 \tTraining Loss: 0.258029 \tValidation Loss: 0.638129\n",
+      "iteration: 215.0 \tTraining Loss: 0.254950 \tValidation Loss: 0.638129\n",
+      "iteration: 225.0 \tTraining Loss: 0.185815 \tValidation Loss: 0.638129\n",
+      "iteration: 235.0 \tTraining Loss: 0.224218 \tValidation Loss: 0.638129\n",
+      "iteration: 245.0 \tTraining Loss: 0.215922 \tValidation Loss: 0.638129\n",
+      "iteration: 255.0 \tTraining Loss: 0.245718 \tValidation Loss: 0.638129\n",
+      "iteration: 265.0 \tTraining Loss: 0.268707 \tValidation Loss: 0.638129\n",
+      "iteration: 275.0 \tTraining Loss: 0.195637 \tValidation Loss: 0.638129\n",
+      "iteration: 285.0 \tTraining Loss: 0.205642 \tValidation Loss: 0.638129\n",
+      "iteration: 295.0 \tTraining Loss: 0.205471 \tValidation Loss: 0.638129\n"
+     ]
+    }
+   ],
+   "source": [
+    "# train\n",
+    "for epoch in range(0, n_epochs):\n",
+    "\n",
+    "    # keep track of training, validation loss and correct\n",
+    "    train_loss = 0.0\n",
+    "    valid_loss = 0.0\n",
+    "    correct = 0.0\n",
+    "    # train the model \n",
+    "    model.train()\n",
+    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
+    "        data, target = data.to(DEVICE), target.to(DEVICE).float().reshape(batch_size, 1)\n",
+    "        optimizer.zero_grad(