adding hdf5_to_csv and try to train the data

0ee28ff1 · Yifan Wang · f9952731 · f9952731 · f9952731 · 0ee28ff1
Commit 0ee28ff1 authored Jul 19, 2021 by Yifan Wang
--- a/gen_samples_py3/.ipynb_checkpoints/99-whiten_wavepic_draw-checkpoint.ipynb
+++ b/gen_samples_py3/.ipynb_checkpoints/99-whiten_wavepic_draw-checkpoint.ipynb
--- a/gen_samples_py3/output/default.hdf
+++ b/gen_samples_py3/output/default.hdf
--- a/gen_samples_py3/1-generate_sample.py
+++ b/gen_samples_py3/1-generate_sample.py
--- a/workspace/2-data_csv.ipynb
+++ b/workspace/2-data_csv.ipynb
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import csv\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from tqdm import tqdm\n",
+    "import utils.samplefiles"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_wnum = 50\n",
+    "train_nnum = 50\n",
+    "test_wnum = 50\n",
+    "test_nnum = 50"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = utils.samplefiles.SampleFile()\n",
+    "data.read_hdf('./output/train.hdf')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wave, noise = data.as_dataframe(injection_parameters=True, \n",
+    "                  static_arguments=False, \n",
+    "                  command_line_arguments=False, \n",
+    "                  split_injections_noise=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Turn strain into multi-dimension array"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "h1w = wave['h1_strain'].tolist()\n",
+    "h1n = noise['h1_strain'].tolist()\n",
+    "wary = np.array(h1w)\n",
+    "nary = np.array(h1n)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Split train and test set"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wtrain = wary[:train_wnum,:]\n",
+    "ntrain = nary[:train_nnum,:]\n",
+    "wtest = wary[train_wnum:,:]\n",
+    "ntest = nary[train_nnum:,:]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Insert label"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wtrain = np.insert(wtrain, 0, values=1, axis=1)\n",
+    "ntrain = np.insert(ntrain, 0, values=0, axis=1)\n",
+    "wtest = np.insert(wtest, 0, values=1, axis=1)\n",
+    "ntest = np.insert(ntest, 0, values=0, axis=1)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Training set name"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 50/50 [00:00<00:00, 299593.14it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "train_name = []\n",
+    "num = 50\n",
+    "train_name.append('label')\n",
+    "for i in tqdm(range(0,num)):\n",
+    "    train_name.append('point{s1}'.format(s1=i))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 50/50 [00:00<00:00, 63.12it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "with open(\"output/train.csv\",\"w\") as csvfile: \n",
+    "    writer = csv.writer(csvfile)\n",
+    "    #columns_name\n",
+    "    writer.writerow(train_name)\n",
+    "    #use writerows to write lines\n",
+    "    for i in tqdm(range(0,train_wnum)):\n",
+    "        writer.writerow(wtrain[i])\n",
+    "        writer.writerow(ntrain[i])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "testing set name"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 50/50 [00:00<00:00, 394201.50it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "test_name = []\n",
+    "num = 50\n",
+    "test_name.append('label')\n",
+    "for i in tqdm(range(0,num)):\n",
+    "    test_name.append('point{s1}'.format(s1=i))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 50/50 [00:00<00:00, 60.24it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "with open(\"output/test.csv\",\"w\") as csvfile: \n",
+    "    writer = csv.writer(csvfile)\n",
+    "    #columns_name\n",
+    "    writer.writerow(test_name)\n",
+    "    #use writerows to write lines\n",
+    "    for i in tqdm(range(0,test_wnum)):\n",
+    "        writer.writerow(wtest[i])\n",
+    "        writer.writerow(ntest[i])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
+%% Cell type:code id: tags:
+
+``` python
+import csv
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+import utils.samplefiles
+```
+
+%% Cell type:code id: tags:
+
+``` python
+train_wnum = 50
+train_nnum = 50
+test_wnum = 50
+test_nnum = 50
+```
+
+%% Cell type:code id: tags:
+
+``` python
+data = utils.samplefiles.SampleFile()
+data.read_hdf('./output/train.hdf')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+wave, noise = data.as_dataframe(injection_parameters=True,
+                  static_arguments=False,
+                  command_line_arguments=False,
+                  split_injections_noise=True)
+```
+
+%% Cell type:markdown id: tags:
+
+Turn strain into multi-dimension array
+
+%% Cell type:code id: tags:
+
+``` python
+h1w = wave['h1_strain'].tolist()
+h1n = noise['h1_strain'].tolist()
+wary = np.array(h1w)
+nary = np.array(h1n)
+```
+
+%% Cell type:markdown id: tags:
+
+Split train and test set
+
+%% Cell type:code id: tags:
+
+``` python
+wtrain = wary[:train_wnum,:]
+ntrain = nary[:train_nnum,:]
+wtest = wary[train_wnum:,:]
+ntest = nary[train_nnum:,:]
+```
+
+%% Cell type:markdown id: tags:
+
+Insert label
+
+%% Cell type:code id: tags:
+
+``` python
+wtrain = np.insert(wtrain, 0, values=1, axis=1)
+ntrain = np.insert(ntrain, 0, values=0, axis=1)
+wtest = np.insert(wtest, 0, values=1, axis=1)
+ntest = np.insert(ntest, 0, values=0, axis=1)
+```
+
+%% Cell type:markdown id: tags:
+
+Training set name
+
+%% Cell type:code id: tags:
+
+``` python
+train_name = []
+num = 50
+train_name.append('label')
+for i in tqdm(range(0,num)):
+    train_name.append('point{s1}'.format(s1=i))
+```
+
+%% Output
+
+    100%|██████████| 50/50 [00:00<00:00, 299593.14it/s]
+
+%% Cell type:code id: tags:
+
+``` python
+with open("output/train.csv","w") as csvfile:
+    writer = csv.writer(csvfile)
+    #columns_name
+    writer.writerow(train_name)
+    #use writerows to write lines
+    for i in tqdm(range(0,train_wnum)):
+        writer.writerow(wtrain[i])
+        writer.writerow(ntrain[i])
+```
+
+%% Output
+
+    100%|██████████| 50/50 [00:00<00:00, 63.12it/s]
+
+%% Cell type:markdown id: tags:
+
+testing set name
+
+%% Cell type:code id: tags:
+
+``` python
+test_name = []
+num = 50
+test_name.append('label')
+for i in tqdm(range(0,num)):
+    test_name.append('point{s1}'.format(s1=i))
+```
+
+%% Output
+
+    100%|██████████| 50/50 [00:00<00:00, 394201.50it/s]
+
+%% Cell type:code id: tags:
+
+``` python
+with open("output/test.csv","w") as csvfile:
+    writer = csv.writer(csvfile)
+    #columns_name
+    writer.writerow(test_name)
+    #use writerows to write lines
+    for i in tqdm(range(0,test_wnum)):
+        writer.writerow(wtest[i])
+        writer.writerow(ntest[i])
+```
+
+%% Output
+
+    100%|██████████| 50/50 [00:00<00:00, 60.24it/s]
+
+%% Cell type:code id: tags:
+
+``` python
+```
--- a/workspace/3-machine_learning.ipynb
+++ b/workspace/3-machine_learning.ipynb
--- a/gen_samples_py3/99-loopplot.py
+++ b/gen_samples_py3/99-loopplot.py
--- a/gen_samples_py3/99-whiten_wavepic_draw.ipynb
+++ b/gen_samples_py3/99-whiten_wavepic_draw.ipynb
--- a/gen_samples_py3/config_files/default.json
+++ b/gen_samples_py3/config_files/default.json
--- a/gen_samples_py3/config_files/waveform_params.ini
+++ b/gen_samples_py3/config_files/waveform_params.ini
--- a/workspace/models/__init__.py
+++ b/workspace/models/__init__.py
+from .cnn import *
+from .cnn_1x1 import *
+from .cnn_bn import *
+from .cnn_dp import *
+from .cnn_dp_bn import *
+from .cnn_dp_bn_1x1 import *
--- a/workspace/models/cnn.py
+++ b/workspace/models/cnn.py
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class ConvNet1(nn.Module):
+    def __init__(self):
+        super(ConvNet1, self).__init__()
+        self.conv1 = nn.Conv1d(1, 16, 16)
+        self.max_pool1 = nn.MaxPool1d(4,4)
+        self.conv2 = nn.Conv1d(16, 32, 8)
+        self.max_pool2 = nn.MaxPool1d(4,4)
+        self.conv3 = nn.Conv1d(32, 64, 8)
+        self.max_pool3 = nn.MaxPool1d(4,4)
+        self.fc1 = nn.Linear(3904, 64)
+        self.fc2 = nn.Linear(64, 1)
+ 
+    def forward(self, x):
+        x = self.conv1(x)
+        x = F.relu(x)
+        x = self.max_pool1(x)
+        x = self.conv2(x)
+        x = F.relu(x)
+        x = self.max_pool2(x)
+        x = self.conv3(x)
+        x = F.relu(x)
+        x = self.max_pool3(x)
+        # resize
+        x = x.view(x.size(0), -1)
+        x = F.relu(self.fc1(x))
+        x = self.fc2(x)
+        x = torch.sigmoid(x)
+        return x
--- a/workspace/models/cnn_1x1.py
+++ b/workspace/models/cnn_1x1.py
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class ConvNet4(nn.Module):
+    def __init__(self):
+        super(ConvNet4, self).__init__()
+        self.conv1 = nn.Conv1d(1, 8, 1)
+        self.conv2 = nn.Conv1d(8, 16, 16)
+        self.max_pool1 = nn.MaxPool1d(4,4)
+        self.conv3 = nn.Conv1d(16, 32, 8)
+        self.max_pool2 = nn.MaxPool1d(4,4)
+        self.conv4 = nn.Conv1d(32, 64, 8)
+        self.max_pool3 = nn.MaxPool1d(4,4)
+        self.fc1 = nn.Linear(3904, 64)
+        self.fc2 = nn.Linear(64, 1)
+ 
+    def forward(self, x):
+        x = self.conv1(x)
+        x = F.relu(x)
+        x = self.conv2(x)
+        x = F.relu(x)
+        x = self.max_pool1(x)
+        x = self.conv3(x)
+        x = F.relu(x)
+        x = self.max_pool2(x)
+        x = self.conv4(x)
+        x = F.relu(x)
+        x = self.max_pool3(x)
+        # resize
+        x = x.view(x.size(0), -1)
+        x = F.relu(self.fc1(x))
+        x = self.fc2(x)
+        x = torch.sigmoid(x)
+        return x
--- a/workspace/models/cnn_bn.py
+++ b/workspace/models/cnn_bn.py
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+# 定义网络
+class ConvNet3(nn.Module):
+    def __init__(self):
+        super(ConvNet3, self).__init__()
+        self.conv1 = nn.Conv1d(1, 16, 16)
+        self.bn1 = nn.BatchNorm1d(16)
+        self.max_pool1 = nn.MaxPool1d(4,4)
+        self.conv2 = nn.Conv1d(16, 32, 8)
+        self.bn2 = nn.BatchNorm1d(32)
+        self.max_pool2 = nn.MaxPool1d(4,4)
+        self.conv3 = nn.Conv1d(32, 64, 8)
+        self.bn3 = nn.BatchNorm1d(64)
+        self.max_pool3 = nn.MaxPool1d(4,4)
+        self.fc1 = nn.Linear(3904, 64)
+        self.fc2 = nn.Linear(64, 1)
+ 
+    def forward(self, x):
+        x = self.conv1(x)
+        x = F.relu(self.bn1(x))
+        x = self.max_pool1(x)
+        x = self.conv2(x)
+        x = F.relu(self.bn2(x))
+        x = self.max_pool2(x)
+        x = self.conv3(x)
+        x = F.relu(self.bn3(x))
+        x = self.max_pool3(x)
+        # resize
+        x = x.view(x.size(0), -1)
+        x = F.relu(self.fc1(x))
+        x = self.fc2(x)
+        x = torch.sigmoid(x)
+        return x
--- a/workspace/models/cnn_dp.py
+++ b/workspace/models/cnn_dp.py
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class ConvNet2(nn.Module):
+    def __init__(self):
+        super(ConvNet2, self).__init__()
+        self.conv1 = nn.Conv1d(1, 16, 16)
+        self.max_pool1 = nn.MaxPool1d(4,4)
+        self.conv2 = nn.Conv1d(16, 32, 8)
+        self.max_pool2 = nn.MaxPool1d(4,4)
+        self.conv3 = nn.Conv1d(32, 64, 8)
+        self.max_pool3 = nn.MaxPool1d(4,4)
+        self.fc1 = nn.Linear(3904, 64)
+        self.fc2 = nn.Linear(64, 1)
+        # dropout
+        self.dropout = nn.Dropout(p=.5)
+ 
+    def forward(self, x):
+        x = self.conv1(x)
+        x = F.relu(x)
+        x = self.max_pool1(x)
+        x = self.conv2(x)
+        x = F.relu(x)
+        x = self.max_pool2(x)
+        x = self.conv3(x)
+        x = F.relu(x)
+        x = self.max_pool3(x)
+        # resize
+        x = x.view(x.size(0), -1)
+        x = self.dropout(F.relu(self.fc1(x)))
+        x = self.fc2(x)
+        x = torch.sigmoid(x)
+        return x
--- a/workspace/models/cnn_dp_bn.py
+++ b/workspace/models/cnn_dp_bn.py
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+# 定义网络
+class ConvNet5(nn.Module):
+    def __init__(self):
+        super(ConvNet5, self).__init__()
+        self.conv1 = nn.Conv1d(1, 16, 16)
+        self.bn1 = nn.BatchNorm1d(16)
+        self.max_pool1 = nn.MaxPool1d(4,4)
+        self.conv2 = nn.Conv1d(16, 32, 8)
+        self.bn2 = nn.BatchNorm1d(32)
+        self.max_pool2 = nn.MaxPool1d(4,4)
+        self.conv3 = nn.Conv1d(32, 64, 8)
+        self.bn3 = nn.BatchNorm1d(64)
+        self.max_pool3 = nn.MaxPool1d(4,4)
+        self.fc1 = nn.Linear(3904, 64)
+        self.fc2 = nn.Linear(64, 1)
+        # dropout
+        self.dropout = nn.Dropout(p=.5)
+ 
+    def forward(self, x):
+        x = self.conv1(x)
+        x = F.relu(self.bn1(x))
+        x = self.max_pool1(x)
+        x = self.conv2(x)
+        x = F.relu(self.bn2(x))
+        x = self.max_pool2(x)
+        x = self.conv3(x)
+        x = F.relu(self.bn3(x))
+        x = self.max_pool3(x)
+        # resize
+        x = x.view(x.size(0), -1)
+        x = self.dropout(F.relu(self.fc1(x)))
+        x = self.fc2(x)
+        x = torch.sigmoid(x)
+        return x
--- a/workspace/models/cnn_dp_bn_1x1.py
+++ b/workspace/models/cnn_dp_bn_1x1.py
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+# 定义网络
+class ConvNet6(nn.Module):
+    def __init__(self):
+        super(ConvNet6, self).__init__()
+        self.conv1 = nn.Conv1d(1, 8, 1)
+        self.bn1 = nn.BatchNorm1d(8)
+        self.conv2 = nn.Conv1d(8, 16, 16)
+        self.bn2 = nn.BatchNorm1d(16)
+        self.max_pool1 = nn.MaxPool1d(4,4)
+        self.conv3 = nn.Conv1d(16, 32, 8)
+        self.bn3 = nn.BatchNorm1d(32)
+        self.max_pool2 = nn.MaxPool1d(4,4)
+        self.conv4 = nn.Conv1d(32, 64, 8)
+        self.bn4 = nn.BatchNorm1d(64)
+        self.max_pool3 = nn.MaxPool1d(4,4)
+        self.fc1 = nn.Linear(3904, 64)
+        self.fc2 = nn.Linear(64, 1)
+        # dropout
+        self.dropout = nn.Dropout(p=.5)
+ 
+    def forward(self, x):
+        x = self.conv1(x)
+        x = F.relu(self.bn1(x))
+        x = self.conv2(x)
+        x = F.relu(self.bn2(x))
+        x = self.max_pool1(x)
+        x = self.conv3(x)
+        x = F.relu(self.bn3(x))
+        x = self.max_pool2(x)
+        x = self.conv4(x)
+        x = F.relu(self.bn4(x))
+        x = self.max_pool3(x)
+        # resize
+        x = x.view(x.size(0), -1)
+        x = self.dropout(F.relu(self.fc1(x)))
+        x = self.fc2(x)
+        x = torch.sigmoid(x)
+        return x
--- a/gen_samples_py3/plot_sample.py
+++ b/gen_samples_py3/plot_sample.py
--- a/gen_samples_py3/plots/0.png
+++ b/gen_samples_py3/plots/0.png
--- a/gen_samples_py3/plots/1.png
+++ b/gen_samples_py3/plots/1.png
--- a/gen_samples_py3/plots/100.png
+++ b/gen_samples_py3/plots/100.png