diff --git a/__pycache__/config.cpython-38.pyc b/__pycache__/config.cpython-38.pyc
index da6e31c3da59664bc76d25f4e1570bf3e01dc3f9..5bb18e1613b8d7cef640aae6b04dcc3e7fb98408 100644
Binary files a/__pycache__/config.cpython-38.pyc and b/__pycache__/config.cpython-38.pyc differ
diff --git a/config.py b/config.py
index 7a405dfea3272435e98315e4ef77f572f714b814..0e64588ac0b50d5dea97d306d8061f0c9feba135 100644
--- a/config.py
+++ b/config.py
@@ -1,10 +1,8 @@
DEBUG = False
-NROWS = 1008
-MARKER_FS = 120
BR_FS = 18
ACC_FS = 100
IMU_FS = 120
-N_MARKERS = 7
+PPG_FS = 100
ACC_THOLD = 10
WIN_THOLD = 0.03
@@ -18,39 +16,24 @@ MAX_RESP_RATE = 45 # BPM
TIME_COLS = ['Timestamps', 'Event', 'Text', 'Color']
-MOCAP_ACCEL_SD = 0.00352
-
TRAIN_VAL_TEST_SPLIT = [0.6, 0.2, 0.2]
TRAIN_TEST_SPLIT = [0.8, 0.2]
import matplotlib as mpl
-mpl.rcParams['figure.titlesize'] = 6
+mpl.rcParams['figure.titlesize'] = 6
mpl.rcParams['axes.titlesize'] = 6
mpl.rcParams['axes.titleweight'] = 'bold'
mpl.rcParams['axes.labelsize'] = 6
mpl.rcParams['xtick.labelsize'] = 6
mpl.rcParams['ytick.labelsize'] = 6
-LOW_HACC_FS_ID = [1, 2, 9, 11, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 23, 24,
- 25, 26, 27]
-NO_HACC_ID = [3, 4, 5, 6]
-
-# issues with marker data on MR conditions:
-MARKER_ISSUES_MR = [12, 14, 17, 18, 26, 30]
-MARKER_ISSUES_R = [12, 14, 18]
-MARKER_ISSUES_M = [12, 14]
-# issues with imu data on MR and L0-3 conditions:
-IMU_ISSUES = [17, 21, 23, 26, 28, 30]
-IMU_ISSUES_L = [15, 17, 21, 23, 26, 28]
-
-# issues with imu data on MR:
-IMU_ISSUES_MR = [17, 26, 30]
-
DPI = 300
-FIG_FMT = 'pdf'
+FIG_FMT = 'png'
from sys import platform
if 'linux' in platform:
- DATA_DIR = '/projects/CIBCIGroup/00DataUploading/rqchia/aria_seated/Data'
+ DATA_DIR = "/projects/CIBCIGroup/00DataUploading/rqchia/"\
+ "aria-respiration-cal/"
elif 'win' in platform:
- DATA_DIR = 'D:/Raymond Chia/UTS/Howe Zhu - Data/1stExperiment_sitting'
+ DATA_DIR = 'D:/Raymond Chia/UTS/Howe Zhu - ' \
+ 'Data/1stExperiment_sitting/respiration-calibration'
diff --git a/models/__pycache__/ardregression.cpython-38.pyc b/models/__pycache__/ardregression.cpython-38.pyc
index 44a34a21a2c188b4f8fe15656377cd2e5f00972e..623dec49edcd93f7734e1624faade447c65721f8 100644
Binary files a/models/__pycache__/ardregression.cpython-38.pyc and b/models/__pycache__/ardregression.cpython-38.pyc differ
diff --git a/models/__pycache__/elasticnet.cpython-38.pyc b/models/__pycache__/elasticnet.cpython-38.pyc
index 5201bebcea6d268e52f185dd3958282a29cddda8..d0c83c0abefc1c8776068d987e4f095e7e2d26da 100644
Binary files a/models/__pycache__/elasticnet.cpython-38.pyc and b/models/__pycache__/elasticnet.cpython-38.pyc differ
diff --git a/models/__pycache__/knn.cpython-38.pyc b/models/__pycache__/knn.cpython-38.pyc
index abf30b8ab2d7016cee9424f1d9f6939f1b07a579..a274e6e23cb14cdf25a14d6b68a0b165a2ae3e51 100644
Binary files a/models/__pycache__/knn.cpython-38.pyc and b/models/__pycache__/knn.cpython-38.pyc differ
diff --git a/models/__pycache__/lda.cpython-38.pyc b/models/__pycache__/lda.cpython-38.pyc
index e3e967574042540a88a8df29240db0a4c2a0f825..5231a52ca76b1fdedac034d67c4e2bd4759ab679 100644
Binary files a/models/__pycache__/lda.cpython-38.pyc and b/models/__pycache__/lda.cpython-38.pyc differ
diff --git a/models/__pycache__/linearregression.cpython-38.pyc b/models/__pycache__/linearregression.cpython-38.pyc
index 3ae3c03e38afb0f891cd328d5b1330b4856c5f5b..5af69ede993292168663e1df6cdf342d2e918b5b 100644
Binary files a/models/__pycache__/linearregression.cpython-38.pyc and b/models/__pycache__/linearregression.cpython-38.pyc differ
diff --git a/models/__pycache__/logisticregression.cpython-38.pyc b/models/__pycache__/logisticregression.cpython-38.pyc
index 524dd77bd435b5a55c557bcce13715362dd509bc..1123200fa3a91bcd964aadab525b1a2ca168260c 100644
Binary files a/models/__pycache__/logisticregression.cpython-38.pyc and b/models/__pycache__/logisticregression.cpython-38.pyc differ
diff --git a/models/__pycache__/neuralnet.cpython-38.pyc b/models/__pycache__/neuralnet.cpython-38.pyc
index aae3d8b200118e5ce56a46201c442b275453ebb1..02fcc1fffdf3521fed55f79a6cb176a8e820878c 100644
Binary files a/models/__pycache__/neuralnet.cpython-38.pyc and b/models/__pycache__/neuralnet.cpython-38.pyc differ
diff --git a/models/__pycache__/resnet.cpython-38.pyc b/models/__pycache__/resnet.cpython-38.pyc
index 997d4faf545fc18d31cfa4a588a7f19135466105..ff17be87b4954de445b870d25f0cf8f6bd67ad48 100644
Binary files a/models/__pycache__/resnet.cpython-38.pyc and b/models/__pycache__/resnet.cpython-38.pyc differ
diff --git a/models/__pycache__/ridgeclass.cpython-38.pyc b/models/__pycache__/ridgeclass.cpython-38.pyc
index 87bc5cbb037961543d9bb233ae017c2a17dd5de7..6cb74a1220e288f5ef83b1e9b349edb87b6e0ff6 100644
Binary files a/models/__pycache__/ridgeclass.cpython-38.pyc and b/models/__pycache__/ridgeclass.cpython-38.pyc differ
diff --git a/models/__pycache__/svm.cpython-38.pyc b/models/__pycache__/svm.cpython-38.pyc
index b4557be071ba83d26bcee4d2972778b589c96442..c7a4837d50c610588b2c2fd7cb525e3d66719b45 100644
Binary files a/models/__pycache__/svm.cpython-38.pyc and b/models/__pycache__/svm.cpython-38.pyc differ
diff --git a/models/__pycache__/svr.cpython-38.pyc b/models/__pycache__/svr.cpython-38.pyc
index bf9763815f158510f638928fd6c517c0d683c37c..42f47b85e381881482dc4bdebc4ccaae6189656e 100644
Binary files a/models/__pycache__/svr.cpython-38.pyc and b/models/__pycache__/svr.cpython-38.pyc differ
diff --git a/models/__pycache__/xgboostclass.cpython-38.pyc b/models/__pycache__/xgboostclass.cpython-38.pyc
index 4e8ed4221b837e31ab20581b164e21975fde3637..64985e5a3d5d0000725338d7d6a4ebda1faa7123 100644
Binary files a/models/__pycache__/xgboostclass.cpython-38.pyc and b/models/__pycache__/xgboostclass.cpython-38.pyc differ
diff --git a/modules/__pycache__/datapipeline.cpython-38.pyc b/modules/__pycache__/datapipeline.cpython-38.pyc
index 0adbf5175d6feae368455db0e588886d4cea42ee..527236c49a3e1e135556397291a90d4d3a38c2f8 100644
Binary files a/modules/__pycache__/datapipeline.cpython-38.pyc and b/modules/__pycache__/datapipeline.cpython-38.pyc differ
diff --git a/modules/__pycache__/digitalsignalprocessing.cpython-38.pyc b/modules/__pycache__/digitalsignalprocessing.cpython-38.pyc
index 15c4e5aac44dbc2d66247bd0293dd784e3ef8947..fca6f3076758c0f03a274c04595a7f65e80cc225 100644
Binary files a/modules/__pycache__/digitalsignalprocessing.cpython-38.pyc and b/modules/__pycache__/digitalsignalprocessing.cpython-38.pyc differ
diff --git a/modules/__pycache__/evaluations.cpython-38.pyc b/modules/__pycache__/evaluations.cpython-38.pyc
index 150f75a863075a6551abc9303dc695a4fd2022aa..1e72d2c91b98da466c7b08f0bb2136901b06b1b8 100644
Binary files a/modules/__pycache__/evaluations.cpython-38.pyc and b/modules/__pycache__/evaluations.cpython-38.pyc differ
diff --git a/modules/__pycache__/utils.cpython-38.pyc b/modules/__pycache__/utils.cpython-38.pyc
index 3779f2328d7e4b76f6393ce35d676cd00db208ed..29f9b792e78cd77fbddf620dbd8b764782742cce 100644
Binary files a/modules/__pycache__/utils.cpython-38.pyc and b/modules/__pycache__/utils.cpython-38.pyc differ
diff --git a/modules/datapipeline.py b/modules/datapipeline.py
index d66fbc5a37b7c76edce2484080b88d6554fc81b5..819ebf0bca8b3ebccec2e4916b4325f92f9a0fa7 100644
--- a/modules/datapipeline.py
+++ b/modules/datapipeline.py
@@ -27,8 +27,7 @@ from sklearn.cluster import MiniBatchKMeans
import cv2
-from config import DEBUG, NROWS, N_MARKERS\
- ,TIME_COLS, NO_HACC_ID, DATA_DIR
+from config import DEBUG, TIME_COLS, DATA_DIR
def datetime_to_sec(time_in, is_iso=False):
dstr = datetime.today()
@@ -312,14 +311,10 @@ def shuffle_split(x):
class DataImporter():
def __init__(self):
self.imu_fname = ''
- self.marker_fname = ''
self.timeline_fname = ''
self.summary_fname = ''
self.video_fname = ''
- if DEBUG:
- self.nrows_to_import = NROWS
- else:
- self.nrows_to_import = None
+ self.nrows_to_import = None
if platform =='linux' or platform == 'linux2':
self.sep = "/"
@@ -331,110 +326,12 @@ class DataImporter():
else:
self.parent_dir = 'D:\\Raymond Chia\\UTS\\Howe Zhu - Data\\'
- def import_rigid_body_data(self):
- col_keys = ['frame', 'time (seconds)', 'mean marker error',
- 'marker quality', 'rigid body', 'position', 'rotation',
- 'x', 'y', 'z']
- filename = self.marker_fname
- header = pd.read_csv(self.marker_hdr_fname, nrows=1, usecols=list(range(0,22)),
- header=None)
- header = dict(header.values.reshape((11,2)).tolist())
- if self.nrows_to_import is None:
- df = pd.read_csv(
- filename, header=list(range(0,5))
- )
- else:
- df = pd.read_csv(
- filename, nrows=self.nrows_to_import, header=list(range(0,5))
- )
- shape = df.shape
- if shape[1] > 10:
- diff = shape[1] - 10
- df = df.drop(df.columns[-diff::], axis=1)
- cols = df.columns.values
- new_cols = []
- for i, lstr in enumerate(cols):
- col_val = []
- lstr_list = [ls for ls in lstr]
- if 'Rigid Body Marker' in lstr_list: continue
- for j, str_val in enumerate(lstr):
- if str_val.lower() in col_keys or 'glasses' in str_val.lower():
- if ' ' in str_val:
- str_val = str_val.replace(' ', '_')
- col_val.append(str_val)
- new_cols.append('_'.join(col_val))
- df.columns = new_cols
-
- return df, header
-
- def cleanup_marker_data(self, filename):
- chunksize = 10
- file_size_mb = stat(filename).st_size/(1024*1024)
- ff = filename.split(self.sep)[:-1]
- if file_size_mb > 0.5:
- print("processing: ", filename)
- header = pd.read_csv(
- filename, nrows=1, usecols=list(range(0,22)), header=None)
- hdr_name = path_join(self.sep.join(ff),
- filename[:-4] + '_header.csv')
- header.to_csv(hdr_name, index=False)
- df_hdr = pd.read_csv(filename, skiprows=2, header=list(range(0,5)),
- nrows=0)
- df = pd.read_csv(filename, skiprows=6, usecols=list(range(38)))
- df.columns = df_hdr.columns[:38]
- amended_df_name = path_join(
- self.sep.join(ff), filename[:-4] + '_amended.csv')
- df.to_csv(amended_df_name, index=False)
- print("saved: ", amended_df_name)
-
- def import_marker_file(self, filename):
- if self.nrows_to_import is None:
- df = pd.read_csv(
- filename, header=list(range(0,5))
- )
- else:
- df = pd.read_csv(
- filename, nrows=self.nrows_to_import,
- header=list(range(0,5))
- )
- return df
def import_header_file(self, filename):
df = pd.read_csv(filename, skiprows=1, nrows=1,
usecols=list(range(0,22)), header=None)
return df
-# Import .mat files from markers
- def import_marker_data(self):
- col_keys = ['frame', 'time (seconds)', 'mean marker error',
- 'marker quality', 'marker', 'position', 'rotation',
- 'x', 'y', 'z']
- filename = self.marker_fname
- header = self.import_header_file(self.marker_hdr_fname)
- df = self.import_marker_file(filename)
-
- shape = df.shape
- if shape[1] > 38:
- diff = shape[1] - 38
- df = df.drop(df.columns[-diff::], axis=1)
- cols = df.columns.values
- new_cols = []
- for i, lstr in enumerate(cols):
- col_val = []
- if type(lstr[0]) is str and "('" in lstr[0]:
- tmp = lstr[0][1:-1].split(',')
- lstr = [ll.replace(" '", '').replace("'", "") for ll in tmp]
- for j, str_val in enumerate(lstr):
- if str_val.lower() in col_keys or 'glasses' in str_val.lower():
- if ' ' in str_val:
- str_val = str_val.replace(' ', '_')
- col_val.append(str_val)
- new_cols.append('_'.join(col_val))
- df.columns = new_cols
-
- header = dict(header.values.reshape((11,2)).tolist())
- return df, header
-
# Import labels from csv
def import_labels(self, filename):
if self.nrows_to_import is None:
@@ -559,7 +456,7 @@ class DataSynchronizer():
self.end_ind = end_ind
class SubjectData(DataImporter):
- ''' Loads in data for the rigid body, marker, breathing rate, summary files
+ ''' Loads in data for the rigid body, breathing rate, summary files
and syncs them accordingly '''
def __init__(self, condition='M', subject='S01'):
super().__init__()
@@ -574,14 +471,11 @@ class SubjectData(DataImporter):
self.subject_dir = path_join(self.parent_dir, self.subject)
- self.marker_df = pd.DataFrame()
self.pressure_df = pd.DataFrame()
self.summary_df = pd.DataFrame()
self.accel_df = pd.DataFrame()
self.imu_df = pd.DataFrame()
- self.read_marker_data = False
-
def get_cond_file(self, files):
for f in files:
if self.condition in f.split(sep)[-1] and \
@@ -626,30 +520,6 @@ class SubjectData(DataImporter):
sel_dir = sub_dirs[i]
return sel_dir
- def set_marker_fname(self):
- subject_dir = self.subject_dir
- data_dir = path_join(subject_dir, 'Motive Logs')
- if not path_exists(data_dir):
- data_dir = subject_dir
- data_glob = path_join(data_dir, "*Take*_amended.csv")
- else:
- data_glob = path_join(data_dir, "*_amended.csv")
-
- data_files = sorted(glob.glob(data_glob))
- if self.subject_id > 16:
- condition_chk = self.condition in 'MR'
- if not condition_chk:
- data_files = [dg for dg in data_files if 'MR' not in dg]
- else:
- data_files = [dg for dg in data_files if 'MR' in dg]
- self.marker_fname = data_files[-1]
- if len(data_files)> 1:
- # Check directory times with timeline
- self.marker_fname = self.check_times(data_files)
- # set marker header name
- self.marker_hdr_fname = self.marker_fname.split('_amended')[0] + \
- '_header.csv'
-
def set_pressure_fname(self):
subject_dir = self.subject_dir
sub_dirs = self.list_sub_dirs(subject_dir)
@@ -735,56 +605,17 @@ class SubjectData(DataImporter):
self.study_end = mat_end
def set_fnames(self):
- if self.read_marker_data:
- self.set_marker_fname()
self.set_pressure_fname()
self.set_summary_fname()
- if self.subject_id > 11:
- self.set_imu_fname()
- if self.subject_id not in NO_HACC_ID:
- self.set_accel_fname()
+ self.set_imu_fname()
+ self.set_accel_fname()
def load_dataframes(self):
self.timeline_df = self.import_time_data()
self.pressure_df = self.import_labels(self.pressure_fname)
self.summary_df = self.import_labels(self.summary_fname)
- if self.read_marker_data:
- try:
- self.marker_df, self.mkr_hdr = self.import_marker_data()
- self.rb_df, self.rb_hdr = self.import_rigid_body_data()
- except:
- print("error reading marker data on {0} - {1}".format(
- self.subject_id, self.condition))
- if self.subject_id not in NO_HACC_ID:
- try:
- self.accel_df = self.import_labels(self.accel_fname)
- except:
- print("error reading accel data on {0} - {1}".format(
- self.subject_id, self.condition))
- if self.subject_id > 11:
- try:
- self.imu_df, self.imu_hdr = self.import_imu_data()
- except:
- print("error reading imu data on {0} - {1}".format(
- self.subject_id, self.condition))
-
- def sync_marker_df(self):
- data_sync = DataSynchronizer()
-
- cst = self.mkr_hdr['Capture Start Time']
-
- time_start = datetime_to_sec(cst)
- marker_time = self.marker_df['Time_(Seconds)'].values + time_start
- time_end = marker_time[-1]
-
- self.marker_df['Time_(Seconds)'] = marker_time
- self.marker_df['sec'] = marker_time
- self.rb_df['Time_(Seconds)'] = marker_time
- self.rb_df['sec'] = marker_time
- data_sync.set_bounds(marker_time, self.study_start, self.study_end)
-
- self.marker_df = data_sync.sync_df(self.marker_df).fillna(0)
- self.rb_df = data_sync.sync_df(self.rb_df)
+ self.accel_df = self.import_labels(self.accel_fname)
+ self.imu_df, self.imu_hdr = self.import_imu_data()
def sync_pressure_df(self):
data_sync = DataSynchronizer()
@@ -886,67 +717,8 @@ class SubjectData(DataImporter):
self.set_timeline()
self.sync_pressure_df()
self.sync_summary_df()
- if self.subject_id not in NO_HACC_ID:
- try:
- self.sync_accel_df()
- except:
- print("Error syncing accel data on {0} - {1}".format(
- self.subject_id, self.condition))
- self.marker_df = pd.DataFrame()
- if self.read_marker_data:
- try:
- self.sync_marker_df()
- except:
- print("Error syncing marker data on {0} - {1}".format(
- self.subject_id, self.condition))
- self.marker_df = pd.DataFrame()
- if self.subject_id > 11:
- try:
- self.sync_imu_df()
- except:
- print("Error syncing imu data on {0} - {1}".format(
- self.subject_id, self.condition))
- self.imu_df = pd.DataFrame()
-
- def get_rigid_body_data(self, col_str='Rigid_Body'):
- data_cols = [col for col in self.marker_df.columns.values \
- if col_str in col]
- marker_data = np.zeros((marker_df.shape[0], 3))
- for col in data_cols:
- if 'position_x' in col.lower():
- marker_data[:, 0] = marker_df[col].values
- elif 'position_y' in col.lower():
- marker_data[:, 1] = marker_df[col].values
- elif 'position_z' in col.lower():
- marker_data[:, 2] = marker_df[col].values
- return marker_data
-
- def get_marker_data(self, col_str='Marker'):
- data_cols = [col for col in self.marker_df.columns.values \
- if col_str in col]
- marker_data = np.zeros((self.marker_df.shape[0], N_MARKERS, 3))
- for col in data_cols:
- for i in range(N_MARKERS):
- if str(i+1) not in col:
- continue
- if 'position_x' in col.lower():
- marker_data[:, i, 0] = self.marker_df[col].values
- elif 'position_y' in col.lower():
- marker_data[:, i, 1] = self.marker_df[col].values
- elif 'position_z' in col.lower():
- marker_data[:, i, 2] = self.marker_df[col].values
- return marker_data
-
- def get_marker_quality(self, col_str='quality'):
- data_cols = [col for col in self.marker_df.columns.values \
- if col_str in col.lower()]
- marker_quality = np.zeros((self.marker_df.shape[0], N_MARKERS))
- for col in data_cols:
- for i in range(N_MARKERS):
- if str(i+1) not in col:
- continue
- marker_quality[:, i] = self.marker_df[col].values
- return marker_quality
+ self.sync_accel_df()
+ self.sync_imu_df()
def get_accel_data(self):
accel_cols = self.accel_df.columns
@@ -956,9 +728,6 @@ class SubjectData(DataImporter):
data_cols = ['X Data', 'Y Data', 'Z Data']
return self.accel_df[data_cols].values
- def get_marker_sec(self):
- return self.marker_df['Time_(Seconds)'].values
-
class TFDataPipeline():
def __init__(self, window_size=60, batch_size=32):
self.window_size = window_size
diff --git a/modules/digitalsignalprocessing.py b/modules/digitalsignalprocessing.py
index 7731f26f609e42b38183356d573740190f5b953e..c3ffa192f97d50ac65653b012e5e4a91c9e4e746 100644
--- a/modules/digitalsignalprocessing.py
+++ b/modules/digitalsignalprocessing.py
@@ -17,7 +17,7 @@ from skimage.feature import corner_harris, corner_shi_tomasi, peak_local_max
from ssqueezepy import cwt as sqz_cwt
from config import WIN_THOLD, WINDOW_SIZE, WINDOW_SHIFT, MQA_THOLD, ACC_THOLD
-from config import MARKER_FS, ACC_FS, IMU_FS, FS_RESAMPLE, BR_FS
+from config import ACC_FS, IMU_FS, FS_RESAMPLE, BR_FS
from config import MIN_RESP_RATE, MAX_RESP_RATE
def butter_lowpass(lowcut, fs, order=5):
@@ -200,16 +200,6 @@ def attenuate_edges(signal, nsamples):
return(signal*e_arr.T)
return(signal*edge_attenuator)
-def marker_quality_processing(marker_quality):
- new_arr = []
- for i in range(marker_quality.shape[1]):
- new_arr.append(
- linear_interp(
- marker_quality[:,i], MARKER_FS, FS_RESAMPLE
- )
- )
- return np.array(new_arr).T
-
def std_scaler(data, **kwargs):
mu = np.mean(data, **kwargs)
sd = np.std(data, **kwargs)
@@ -268,8 +258,8 @@ def imu_signal_processing(data, fs:int=IMU_FS):
ma = movingaverage(bp, 8, axis=0)
return ma
-def roddiger_sp(data=None, fs:int=MARKER_FS, is_marker:bool=True):
- ''' Run markers through the following steps:
+def roddiger_sp(data=None, fs:int=IMU_FS):
+ ''' Run data through the following steps:
* Second order taylor estimation
* 3 sample mov mean subtraction
* 2s mov mena
@@ -280,22 +270,11 @@ def roddiger_sp(data=None, fs:int=MARKER_FS, is_marker:bool=True):
triang_window = 2
thold = 2
- # get accelaration profile
- if is_marker:
- mkr_shape = data.shape
- accel = second_order_diff(data, fs)
- accel = data
- else:
- accel = data.astype(float)
+ accel = data.astype(float)
# movmean
ma = accel - movingaverage(accel, 3, axis=0)
- if is_marker and len(accel.shape) > 2:
- for i in range(accel.shape[1]):
- for j in range(3):
- ma[:,i,j] = movingaverage(ma[:,i,j], 2*fs, axis=0)
- else:
- ma = movingaverage(ma, 2*fs, axis=0)
+ ma = movingaverage(ma, 2*fs, axis=0)
# cubic interp
accel = cubic_interp(accel, fs, FS_RESAMPLE)
@@ -318,8 +297,8 @@ def roddiger_sp(data=None, fs:int=MARKER_FS, is_marker:bool=True):
return accel, smth
-def hernandez_sp(data=None, fs:int=MARKER_FS, is_marker:bool=True):
- ''' Run markers through the following steps:
+def hernandez_sp(data=None, fs:int=IMU_FS):
+ ''' Run data through the following steps:
* Second order taylor estimation
* Cubic interpolation (~256Hz)
* Max Resp Rate mov mean
@@ -329,10 +308,7 @@ def hernandez_sp(data=None, fs:int=MARKER_FS, is_marker:bool=True):
# max_br = MAX_RESP_RATE #bpm
max_br = 45 #bpm
- if is_marker:
- accel = second_order_diff(data, fs)
- else:
- accel = data.astype(float)
+ accel = data.astype(float)
# cubic interp
accel = cubic_interp(accel, fs, FS_RESAMPLE)
diff --git a/modules/evaluations.py b/modules/evaluations.py
index 6d6c833a430b2b8cdaca68563b0dcac104f0b9eb..c92c371ee4d16c5fee7eeefad4f3763733a5a3d7 100644
--- a/modules/evaluations.py
+++ b/modules/evaluations.py
@@ -8,8 +8,6 @@ import ipdb
from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error
from scipy.stats import pearsonr, f_oneway, mannwhitneyu
-from config import N_MARKERS
-
class Evaluation():
def __init__(self, y_true, y_pred):
self.y_true = y_true
diff --git a/regress_rr.py b/regress_rr.py
index c087390a58ba62850c3d8ef16e917fa6fa24b62b..282fe46e2e041e72941186544219efc0038c60ee 100644
--- a/regress_rr.py
+++ b/regress_rr.py
@@ -1,3 +1,4 @@
+import glob
from os import makedirs, mkdir
from os.path import join, exists
import pandas as pd
@@ -8,6 +9,7 @@ import re
import pickle
import sys
import time
+from zipfile import ZipFile
import argparse
from datetime import datetime, timedelta, timezone, timedelta
@@ -62,14 +64,18 @@ from sktime.transformations.panel.rocket import (
MiniRocketMultivariateVariable,
)
-from config import WINDOW_SIZE, WINDOW_SHIFT, IMU_FS, DATA_DIR, IMU_ISSUES_L,\
- IMU_ISSUES, BR_FS
+from config import WINDOW_SIZE, WINDOW_SHIFT, IMU_FS, DATA_DIR, BR_FS
IMU_COLS = ['acc_x', 'acc_y', 'acc_z', 'gyr_x', 'gyr_y', 'gyr_z']
def utc_to_local(utc_dt, tz=None):
return utc_dt.replace(tzinfo=timezone.utc).astimezone(tz=tz)
+def datetime_from_utc_to_local(utc_datetime):
+ now_timestamp = time.time()
+ offset = datetime.fromtimestamp(now_timestamp) - datetime.utcfromtimestamp(now_timestamp)
+ return utc_datetime + offset
+
# Load data
def load_bioharness_file(f:str, skiprows=0, skipfooter=0, **kwargs):
df_list = []
@@ -157,6 +163,31 @@ def load_imu_files(f_list:list):
data_df = pd.concat(data, axis=0)
return data_df, hdr
+def load_e4_file(e4_file:str):
+ ''' First row is the initial time of the session as unix time.
+ Second row is the sample rate in Hz'''
+ zip_file = ZipFile(e4_file)
+ dfs = {csv_file.filename: pd.read_csv(zip_file.open(csv_file.filename)
+ ,header=None)
+ for csv_file in zip_file.infolist()
+ if csv_file.filename.endswith('.csv')}
+ bvp = dfs["BVP.csv"]
+ t0 = bvp.iloc[0].values[0]
+ fs = bvp.iloc[1].values[0]
+ nsamples = len(bvp) - 2
+
+ t0_datetime = datetime.utcfromtimestamp(t0)
+ t0_local = datetime_from_utc_to_local(t0_datetime)
+ ipdb.set_trace()
+ time = [t0_local.timestamp() + ind*(1/fs) for ind in
+ range(nsamples)]
+ tmp = [np.nan, np.nan]
+ time = tmp + time
+ bvp['sec'] = np.array(time)
+
+ return bvp
+
+
# Synchronising data
def sync_to_ref(df0, df1):
dsync0 = DataSynchronizer()
@@ -395,6 +426,7 @@ def load_and_sync_xsens(subject):
return xsens_df
+
def load_tsfresh(subject, project_dir,
window_size=12, window_shift=0.2, fs=IMU_FS,
overwrite=False):
@@ -648,8 +680,8 @@ def arg_parser():
'elastic'],
)
parser.add_argument("-s", '--subject', type=int,
- default=12,
- choices=list(range(12,31))+[-1],
+ default=1,
+ choices=list(range(1,3))+[-1],
)
parser.add_argument("-f", '--feature_method', type=str,
default='minirocket',
@@ -677,30 +709,13 @@ def arg_parser():
if __name__ == '__main__':
# choose either intra or inter subject features to use for model training
# '[!M]*'
- '''
- Dataset time per subject (s):
- S12: 2658.65
- S13: 2937.76
- S14: 3009.97
- S15: 1525.07
- S16: 1576.35
- S17: Error
- S18: 2413.58
- S19: 3282.91
- S20: 2218.55
- S21: 824.58
- S22: 3325.08
- S23: 2751.38
- S24: 3153.87
- S25: 2974.67
- S26: Error
- S27: 2608.63
- S28: 1586.10
- S29: 3289.75
- S30: 1984.31
- '''
np.random.seed(100)
- imu_issues = [17, 26]
+ n_subject_max = 2
+
+ e4_glob = join(DATA_DIR, 'Pilot02', 'e4', '*')
+ e4_file = glob.glob(e4_glob)[0]
+ bvp_df = load_e4_file(e4_file)
+ ipdb.set_trace()
args = arg_parser()
@@ -716,16 +731,18 @@ if __name__ == '__main__':
print(args)
assert train_len>0,"--train_len must be an integer greater than 0"
+ subject_pre_string = 'Pilot'
+
if subject > 0:
- subject = 'S'+str(subject).zfill(2)
+ subject = subject_pre_string+str(subject).zfill(2)
imu_rr_model(subject, window_size=window_size, window_shift=window_shift,
lbl_str=lbl_str, mdl_str=mdl_str, overwrite=overwrite,
feature_method=feature_method, train_len=train_len
)
else:
- subjects = ['S'+str(i).zfill(2) for i in range(12,31) \
- if i not in imu_issues]
+ subjects = [subject_pre_string+str(i).zfill(2) for i in \
+ range(1, n_subject_max+1) if i not in imu_issues]
imu_rr_func = partial(imu_rr_model,
window_size=window_size,
window_shift=window_shift,