Skip to content
Snippets Groups Projects
Commit 0ee28ff1 authored by Yifan Wang's avatar Yifan Wang
Browse files

adding hdf5_to_csv and try to train the data

parent f9952731
Branches
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 463 additions and 347 deletions
gen_samples_py3/.ipynb_checkpoints/99-whiten_wavepic_draw-checkpoint.ipynb deleted 100644 → 0
+ 0
− 347
View file @ f9952731
%% Cell type:code id: tags:
``` python
import sys
import time
import numpy as np
import pandas as pd
import utils.samplefiles
import matplotlib
import matplotlib.pyplot as plt
```
%% Cell type:code id: tags:
``` python
# Start the stopwatch
script_start_time = time.time()
```
%% Cell type:code id: tags:
``` python
hdf_file_path = './output/train.hdf'
plot_path = './plots/signalnoise_id0.png'
# which sample is used to shown
sample_id = 0
```
%% Cell type:code id: tags:
``` python
data = utils.samplefiles.SampleFile()
data.read_hdf(hdf_file_path)
```
%% Output
key: config_file values: default.json
%% Cell type:code id: tags:
``` python
# when split_injections_noise is set to be True, GWs contain signals and pure waves are splited
df, noise = data.as_dataframe(injection_parameters=True,
static_arguments=True,
command_line_arguments=False,
split_injections_noise=True)
```
%% Cell type:code id: tags:
``` python
print(df.columns)
```
%% Output
Index(['approximant', 'bandpass_lower', 'bandpass_upper', 'coa_phase', 'dec',
'delta_f', 'delta_t', 'distance', 'domain', 'event_time', 'f_lower',
'fd_length', 'h1_output_signal', 'h1_signal', 'h1_snr', 'h1_strain',
'inclination', 'injection_snr', 'l1_output_signal', 'l1_signal',
'l1_snr', 'l1_strain', 'mass1', 'mass2', 'noise_interval_width',
'original_sampling_rate', 'polarization', 'ra', 'sample_length',
'scale_factor', 'seconds_after_event', 'seconds_before_event', 'spin1z',
'spin2z', 'target_sampling_rate', 'td_length', 'tukey_alpha',
'waveform_length', 'whitening_max_filter_duration',
'whitening_segment_duration'],
dtype='object')
%% Cell type:code id: tags:
``` python
print(df.injection_snr[sample_id])
```
%% Output
7.340279606636548
%% Cell type:code id: tags:
``` python
print(df.mass1[sample_id])
print(df.mass2[sample_id])
print(df.spin1z[sample_id])
print(df.spin2z[sample_id])
print(df.scale_factor[sample_id])
```
%% Output
36.217808319315374
76.55000144869413
0.7305299539277823
0.5974611672286425
0.017164934231118905
%% Cell type:code id: tags:
``` python
print(df.coa_phase[sample_id])
print(df.inclination[sample_id])
print(df.ra[sample_id])
print(df.dec[sample_id])
print(df.polarization[sample_id])
```
%% Output
0.9801424781769557
0.48680334688549004
3.776917009710014
0.821770111935331
4.448951217224888
%% Cell type:code id: tags:
``` python
sample = df.loc[sample_id]
```
%% Cell type:code id: tags:
``` python
# Read out and construct some necessary values for plotting
seconds_before_event = float(sample['seconds_before_event'])
seconds_after_event = float(sample['seconds_after_event'])
target_sampling_rate = float(sample['target_sampling_rate'])
sample_length = float(sample['sample_length'])
print(seconds_before_event)
print(target_sampling_rate)
print(sample_length)
```
%% Output
5.5
2048.0
8.0
%% Cell type:code id: tags:
``` python
# Create a grid on which the sample can be plotted so that the
# event_time is at position 0
grid = np.linspace(0 - seconds_before_event, 0 + seconds_after_event, int(target_sampling_rate * sample_length))
# for time from -0.15s to 0.05s
#grid = np.linspace(0 - seconds_before_event, 0 + seconds_after_event, int(target_sampling_rate * sample_length)+1)
```
%% Cell type:code id: tags:
``` python
det_name = 'H1'
det_string = 'h1_strain'
```
%% Cell type:code id: tags:
``` python
maximum = np.max(sample[det_string])
print(maximum)
```
%% Output
145.16235
%% Cell type:code id: tags:
``` python
maximum = max(np.max(sample['h1_signal']), np.max(sample['l1_signal']))
print(maximum)
```
%% Output
5.173865814288047e-21
%% Cell type:code id: tags:
``` python
maximum = max(np.max(sample['h1_output_signal']), np.max(sample['l1_output_signal']))
print(maximum)
```
%% Output
23.74179629063074
%% Cell type:code id: tags:
``` python
plt.plot(grid, sample[det_string], color='C0',label = 'strain')
plt.plot(grid, sample['h1_output_signal'], color='C1',label = 'signal')
#plt.xlim(-1.5, 0.5)
#plt.ylim(-150, 150)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.ylabel('Whitened Strain and Signal ({})'
.format(det_name), fontsize=15)
plt.xlabel('Time (s)', fontsize=15)
plt.legend(fontsize=15)
# Adjust the size and spacing of the subplots
plt.gcf().set_size_inches(9, 6, forward=True)
plt.tight_layout(rect=[0, 0, 1, 0.9])
plt.subplots_adjust(wspace=0, hspace=0)
# plt.show()
plt.savefig(plot_path)
```
%% Output
%% Cell type:code id: tags:
``` python
```
gen_samples_py3/output/default.hdf deleted 100644 → 0
+ 0
− 0
View file @ f9952731
File deleted
workspace/2-data_csv.ipynb 0 → 100644
+ 240
− 0
View file @ 0ee28ff1
%% Cell type:code id: tags:
``` python
import csv
import numpy as np
import pandas as pd
from tqdm import tqdm
import utils.samplefiles
```
%% Cell type:code id: tags:
``` python
train_wnum = 50
train_nnum = 50
test_wnum = 50
test_nnum = 50
```
%% Cell type:code id: tags:
``` python
data = utils.samplefiles.SampleFile()
data.read_hdf('./output/train.hdf')
```
%% Cell type:code id: tags:
``` python
wave, noise = data.as_dataframe(injection_parameters=True,
static_arguments=False,
command_line_arguments=False,
split_injections_noise=True)
```
%% Cell type:markdown id: tags:
Turn strain into multi-dimension array
%% Cell type:code id: tags:
``` python
h1w = wave['h1_strain'].tolist()
h1n = noise['h1_strain'].tolist()
wary = np.array(h1w)
nary = np.array(h1n)
```
%% Cell type:markdown id: tags:
Split train and test set
%% Cell type:code id: tags:
``` python
wtrain = wary[:train_wnum,:]
ntrain = nary[:train_nnum,:]
wtest = wary[train_wnum:,:]
ntest = nary[train_nnum:,:]
```
%% Cell type:markdown id: tags:
Insert label
%% Cell type:code id: tags:
``` python
wtrain = np.insert(wtrain, 0, values=1, axis=1)
ntrain = np.insert(ntrain, 0, values=0, axis=1)
wtest = np.insert(wtest, 0, values=1, axis=1)
ntest = np.insert(ntest, 0, values=0, axis=1)
```
%% Cell type:markdown id: tags:
Training set name
%% Cell type:code id: tags:
``` python
train_name = []
num = 50
train_name.append('label')
for i in tqdm(range(0,num)):
train_name.append('point{s1}'.format(s1=i))
```
%% Output
100%|██████████| 50/50 [00:00<00:00, 299593.14it/s]
%% Cell type:code id: tags:
``` python
with open("output/train.csv","w") as csvfile:
writer = csv.writer(csvfile)
#columns_name
writer.writerow(train_name)
#use writerows to write lines
for i in tqdm(range(0,train_wnum)):
writer.writerow(wtrain[i])
writer.writerow(ntrain[i])
```
%% Output
100%|██████████| 50/50 [00:00<00:00, 63.12it/s]
%% Cell type:markdown id: tags:
testing set name
%% Cell type:code id: tags:
``` python
test_name = []
num = 50
test_name.append('label')
for i in tqdm(range(0,num)):
test_name.append('point{s1}'.format(s1=i))
```
%% Output
100%|██████████| 50/50 [00:00<00:00, 394201.50it/s]
%% Cell type:code id: tags:
``` python
with open("output/test.csv","w") as csvfile:
writer = csv.writer(csvfile)
#columns_name
writer.writerow(test_name)
#use writerows to write lines
for i in tqdm(range(0,test_wnum)):
writer.writerow(wtest[i])
writer.writerow(ntest[i])
```
%% Output
100%|██████████| 50/50 [00:00<00:00, 60.24it/s]
%% Cell type:code id: tags:
``` python
```
workspace/3-machine_learning.ipynb 0 → 100644
NaN GiB (NaN%)
View file @ 0ee28ff1
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
gen_samples_py3/99-loopplot.py → workspace/99-loopplot.py
+ 0
− 0
View file @ 0ee28ff1
File moved
workspace/models/__init__.py 0 → 100644
+ 6
− 0
View file @ 0ee28ff1
from .cnn import *
from .cnn_1x1 import *
from .cnn_bn import *
from .cnn_dp import *
from .cnn_dp_bn import *
from .cnn_dp_bn_1x1 import *
workspace/models/cnn.py 0 → 100644
+ 32
− 0
View file @ 0ee28ff1
import torch
import torch.nn as nn
import torch.nn.functional as F
class ConvNet1(nn.Module):
def __init__(self):
super(ConvNet1, self).__init__()
self.conv1 = nn.Conv1d(1, 16, 16)
self.max_pool1 = nn.MaxPool1d(4,4)
self.conv2 = nn.Conv1d(16, 32, 8)
self.max_pool2 = nn.MaxPool1d(4,4)
self.conv3 = nn.Conv1d(32, 64, 8)
self.max_pool3 = nn.MaxPool1d(4,4)
self.fc1 = nn.Linear(3904, 64)
self.fc2 = nn.Linear(64, 1)
def forward(self, x):
x = self.conv1(x)
x = F.relu(x)
x = self.max_pool1(x)
x = self.conv2(x)
x = F.relu(x)
x = self.max_pool2(x)
x = self.conv3(x)
x = F.relu(x)
x = self.max_pool3(x)
# resize
x = x.view(x.size(0), -1)
x = F.relu(self.fc1(x))
x = self.fc2(x)
x = torch.sigmoid(x)
return x
workspace/models/cnn_1x1.py 0 → 100644
+ 35
− 0
View file @ 0ee28ff1
import torch
import torch.nn as nn
import torch.nn.functional as F
class ConvNet4(nn.Module):
def __init__(self):
super(ConvNet4, self).__init__()
self.conv1 = nn.Conv1d(1, 8, 1)
self.conv2 = nn.Conv1d(8, 16, 16)
self.max_pool1 = nn.MaxPool1d(4,4)
self.conv3 = nn.Conv1d(16, 32, 8)
self.max_pool2 = nn.MaxPool1d(4,4)
self.conv4 = nn.Conv1d(32, 64, 8)
self.max_pool3 = nn.MaxPool1d(4,4)
self.fc1 = nn.Linear(3904, 64)
self.fc2 = nn.Linear(64, 1)
def forward(self, x):
x = self.conv1(x)
x = F.relu(x)
x = self.conv2(x)
x = F.relu(x)
x = self.max_pool1(x)
x = self.conv3(x)
x = F.relu(x)
x = self.max_pool2(x)
x = self.conv4(x)
x = F.relu(x)
x = self.max_pool3(x)
# resize
x = x.view(x.size(0), -1)
x = F.relu(self.fc1(x))
x = self.fc2(x)
x = torch.sigmoid(x)
return x
workspace/models/cnn_bn.py 0 → 100644
+ 36
− 0
View file @ 0ee28ff1
import torch
import torch.nn as nn
import torch.nn.functional as F
# 定义网络
class ConvNet3(nn.Module):
def __init__(self):
super(ConvNet3, self).__init__()
self.conv1 = nn.Conv1d(1, 16, 16)
self.bn1 = nn.BatchNorm1d(16)
self.max_pool1 = nn.MaxPool1d(4,4)
self.conv2 = nn.Conv1d(16, 32, 8)
self.bn2 = nn.BatchNorm1d(32)
self.max_pool2 = nn.MaxPool1d(4,4)
self.conv3 = nn.Conv1d(32, 64, 8)
self.bn3 = nn.BatchNorm1d(64)
self.max_pool3 = nn.MaxPool1d(4,4)
self.fc1 = nn.Linear(3904, 64)
self.fc2 = nn.Linear(64, 1)
def forward(self, x):
x = self.conv1(x)
x = F.relu(self.bn1(x))
x = self.max_pool1(x)
x = self.conv2(x)
x = F.relu(self.bn2(x))
x = self.max_pool2(x)
x = self.conv3(x)
x = F.relu(self.bn3(x))
x = self.max_pool3(x)
# resize
x = x.view(x.size(0), -1)
x = F.relu(self.fc1(x))
x = self.fc2(x)
x = torch.sigmoid(x)
return x
workspace/models/cnn_dp.py 0 → 100644
+ 34
− 0
View file @ 0ee28ff1
import torch
import torch.nn as nn
import torch.nn.functional as F
class ConvNet2(nn.Module):
def __init__(self):
super(ConvNet2, self).__init__()
self.conv1 = nn.Conv1d(1, 16, 16)
self.max_pool1 = nn.MaxPool1d(4,4)
self.conv2 = nn.Conv1d(16, 32, 8)
self.max_pool2 = nn.MaxPool1d(4,4)
self.conv3 = nn.Conv1d(32, 64, 8)
self.max_pool3 = nn.MaxPool1d(4,4)
self.fc1 = nn.Linear(3904, 64)
self.fc2 = nn.Linear(64, 1)
# dropout
self.dropout = nn.Dropout(p=.5)
def forward(self, x):
x = self.conv1(x)
x = F.relu(x)
x = self.max_pool1(x)
x = self.conv2(x)
x = F.relu(x)
x = self.max_pool2(x)
x = self.conv3(x)
x = F.relu(x)
x = self.max_pool3(x)
# resize
x = x.view(x.size(0), -1)
x = self.dropout(F.relu(self.fc1(x)))
x = self.fc2(x)
x = torch.sigmoid(x)
return x
workspace/models/cnn_dp_bn.py 0 → 100644
+ 38
− 0
View file @ 0ee28ff1
import torch
import torch.nn as nn
import torch.nn.functional as F
# 定义网络
class ConvNet5(nn.Module):
def __init__(self):
super(ConvNet5, self).__init__()
self.conv1 = nn.Conv1d(1, 16, 16)
self.bn1 = nn.BatchNorm1d(16)
self.max_pool1 = nn.MaxPool1d(4,4)
self.conv2 = nn.Conv1d(16, 32, 8)
self.bn2 = nn.BatchNorm1d(32)
self.max_pool2 = nn.MaxPool1d(4,4)
self.conv3 = nn.Conv1d(32, 64, 8)
self.bn3 = nn.BatchNorm1d(64)
self.max_pool3 = nn.MaxPool1d(4,4)
self.fc1 = nn.Linear(3904, 64)
self.fc2 = nn.Linear(64, 1)
# dropout
self.dropout = nn.Dropout(p=.5)
def forward(self, x):
x = self.conv1(x)
x = F.relu(self.bn1(x))
x = self.max_pool1(x)
x = self.conv2(x)
x = F.relu(self.bn2(x))
x = self.max_pool2(x)
x = self.conv3(x)
x = F.relu(self.bn3(x))
x = self.max_pool3(x)
# resize
x = x.view(x.size(0), -1)
x = self.dropout(F.relu(self.fc1(x)))
x = self.fc2(x)
x = torch.sigmoid(x)
return x
workspace/models/cnn_dp_bn_1x1.py 0 → 100644
+ 42
− 0
View file @ 0ee28ff1
import torch
import torch.nn as nn
import torch.nn.functional as F
# 定义网络
class ConvNet6(nn.Module):
def __init__(self):
super(ConvNet6, self).__init__()
self.conv1 = nn.Conv1d(1, 8, 1)
self.bn1 = nn.BatchNorm1d(8)
self.conv2 = nn.Conv1d(8, 16, 16)
self.bn2 = nn.BatchNorm1d(16)
self.max_pool1 = nn.MaxPool1d(4,4)
self.conv3 = nn.Conv1d(16, 32, 8)
self.bn3 = nn.BatchNorm1d(32)
self.max_pool2 = nn.MaxPool1d(4,4)
self.conv4 = nn.Conv1d(32, 64, 8)
self.bn4 = nn.BatchNorm1d(64)
self.max_pool3 = nn.MaxPool1d(4,4)
self.fc1 = nn.Linear(3904, 64)
self.fc2 = nn.Linear(64, 1)
# dropout
self.dropout = nn.Dropout(p=.5)
def forward(self, x):
x = self.conv1(x)
x = F.relu(self.bn1(x))
x = self.conv2(x)
x = F.relu(self.bn2(x))
x = self.max_pool1(x)
x = self.conv3(x)
x = F.relu(self.bn3(x))
x = self.max_pool2(x)
x = self.conv4(x)
x = F.relu(self.bn4(x))
x = self.max_pool3(x)
# resize
x = x.view(x.size(0), -1)
x = self.dropout(F.relu(self.fc1(x)))
x = self.fc2(x)
x = torch.sigmoid(x)
return x
gen_samples_py3/plot_sample.py → workspace/plot_sample.py
+ 0
− 0
View file @ 0ee28ff1
File moved
gen_samples_py3/plots/0.png → workspace/plots/0.png
+ 0
− 0
View file @ 0ee28ff1
File moved
gen_samples_py3/plots/1.png → workspace/plots/1.png
+ 0
− 0
View file @ 0ee28ff1
File moved
gen_samples_py3/plots/100.png → workspace/plots/100.png
+ 0
− 0
View file @ 0ee28ff1
File moved
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment

AEI Privacy Policy Imprint