mvp

regress_rr.py

@@ -18,7 +18,7 @@ import pytz
 import matplotlib.pyplot as plt
 from functools import partial
 from collections import Counter
-from itertools import repeat, chain
+from itertools import repeat, chain, combinations
 from multiprocessing import Pool, cpu_count
 import tensorflow as tf
 
@@ -380,9 +380,9 @@ def load_and_sync_xsens(subject):
 
     xsens_list = []
     # skip the first and last x minute(s)
-    minutes_to_skip = 2
-    br_skiprows = br_skipfooter = minutes_to_skip*60
-    pss_skiprows = pss_skipfooter = minutes_to_skip*60*BR_FS
+    minutes_to_skip = .5
+    br_skiprows = br_skipfooter = int(minutes_to_skip*60)
+    pss_skiprows = pss_skipfooter = int(minutes_to_skip*60*BR_FS)
     # load each bioharness file and sync the imu to it
     for pss_file, br_file in zip(pss_list, br_list):
         pss_df = load_bioharness_file(pss_file, skiprows=pss_skiprows,
@@ -492,6 +492,70 @@ def get_respiration_log(subject):
     log_dfs = [pd.read_json(f) for f in log_list]
     return pd.concat(log_dfs, axis=0)
 
+def get_cal_data(event_df, xsens_df):
+    fmt ="%Y-%m-%d %H.%M.%S" 
+    cal_list = []
+    cpms = []
+    start_sec = 0
+    stop_sec = 0
+    for index, row in event_df.iterrows():
+        event = row['eventTag']
+        timestamp = row['timestamp']
+        inhalePeriod = row['inhalePeriod']
+        exhalePeriod = row['exhalePeriod']
+
+        cpm = np.round( 60/(inhalePeriod + exhalePeriod) )
+
+        sec = timestamp.to_pydatetime().timestamp()
+
+        if event == 'Start':
+            start_sec = sec
+            continue
+        elif event == 'Stop':
+            stop_sec = sec
+
+            dsync = DataSynchronizer()
+            dsync.set_bounds(xsens_df['sec'].values, start_sec, stop_sec)
+
+            sync_df = dsync.sync_df(xsens_df.copy())
+            cal_data = {'cpm': cpm, 'data': sync_df}
+            cal_list.append(cal_data)
+
+            assert np.round(sync_df.sec.iloc[0])==np.round(start_sec), \
+            "error with start sync"
+            assert np.round(sync_df.sec.iloc[-1])==np.round(stop_sec), \
+            "error with stop sync"
+
+    return pd.DataFrame(cal_list)
+
+def get_test_data(cal_df, activity_df, xsens_df, test_standing):
+    fmt = "%d/%m/%Y %H:%M:%S"
+    start_time = cal_df.iloc[-1]['data'].sec.values[-1]
+    data_df = xsens_df[xsens_df.sec > start_time]
+    activity_start = 0
+    activity_end = 0
+
+    activity_list = []
+
+    for index, row in activity_df.iterrows():
+        sec = datetime.strptime(row['Timestamps'], fmt).timestamp()
+        if not test_standing and row['Activity'] == 'standing':
+            continue
+        if row['Event'] == 'start':
+            activity_start = sec
+        elif row['Event'] == 'end':
+            activity_stop = sec
+
+            dsync = DataSynchronizer()
+            dsync.set_bounds(data_df['sec'].values, activity_start,
+                             activity_stop)
+
+            sync_df = dsync.sync_df(data_df.copy())
+            activity_data = {'activity': row['Activity'], 'data': sync_df}
+            activity_list.append(activity_data)
+
+    return pd.DataFrame(activity_list)
+
 # save evaluation metrics in single file that handles the models for the
 # subject and config
 class EvalHandler():
@@ -529,7 +593,8 @@ class EvalHandler():
             index_list = eval_hist[
                 (eval_hist['subject'] == self.entry['subject']) &\
                 (eval_hist['config_id'] == self.entry['config_id']) &\
-                (eval_hist['mdl_str'] == self.entry['mdl_str'])\
+                (eval_hist['mdl_str'] == self.entry['mdl_str']) &\
+                (eval_hist['cpm'] == self.entry['cpm'])\
             ].index.tolist()
             if len(index_list) == 0:
                 print("adding new entry")
@@ -549,11 +614,20 @@ def imu_rr_model(subject,
                  mdl_str='knn',
                  overwrite=False,
                  feature_method='tsfresh',
-                 train_len=1
+                 train_len:int=3,
+                 test_standing=False,
                 ):
     # window_size, window_shift, intra, inter
+    cal_str = 'cpm'
     fs = IMU_FS
     tmp = []
+    imu_cols = ['acc_x', 'acc_y', 'acc_z', 'gyro_x', 'gyro_y', 'gyro_z']
+    bvp_cols= ['bvp']
+
+    # TODO: 
+        # implement and input args config by data cols
+        # implement and input args config with test_standing
+    data_cols = imu_cols + bvp_cols
 
     do_minirocket = False
     use_tsfresh   = False
@@ -571,6 +645,8 @@ def imu_rr_model(subject,
               'do_minirocket' : do_minirocket,
               'use_tsfresh'   : use_tsfresh,
               'train_len'     : train_len,
+              'data_cols'     : data_cols,
+              'test_standing' : test_standing,
              }
 
     pfh = ProjectFileHandler(config)
@@ -586,8 +662,6 @@ def imu_rr_model(subject,
         print('Using pre-set data id: ', pfh.fset_id)
     project_dir = pfh.project_directory
 
-    marker = f'imu_rr_{subject}_id{pfh.fset_id}'
-
     if not use_tsfresh:
         xsens_df = load_and_sync_xsens(subject)
     else:
@@ -598,115 +672,115 @@ def imu_rr_model(subject,
                                 fs=IMU_FS,
                                 overwrite=overwrite_tsfresh)
 
-    large_win_size = 60*fs
-    xsens_inds = np.arange(len(xsens_df))
-    large_windows = vsw(xsens_inds, len(xsens_inds),
-                        sub_window_size=large_win_size,
-                        stride_size=large_win_size)
-    keep_inds = []
-    for i, win in enumerate(large_windows):
-        if win[-1] != 0:
-            keep_inds.append(i)
-
-    large_windows = large_windows[keep_inds]
+    activity_df = get_activity_log(subject)
+    event_df = get_respiration_log(subject)
+
+    cal_df = get_cal_data(event_df, xsens_df)
+
+    # include standing or not
+    test_df_tmp = get_test_data(cal_df, activity_df, xsens_df, test_standing)
+    test_df = pd.concat([df for df in test_df_tmp['data']], axis=0)
     
-    train_inds, test_inds = train_test_split(large_windows,
-                                             train_size=train_size,
-                                             shuffle=True,
-                                             random_state=123)
-
-    train_df = pd.concat([xsens_df.iloc[win] for win in train_inds if
-                          win[-1]!=0])
-    test_df = pd.concat([xsens_df.iloc[win] for win in test_inds if win[-1]!=0])
-
-    assert np.isin(train_df.index.values, test_df.index.values).any()==False,\
-            "overlapping test and train data"
-
-    print("train")
-    print(train_df.shape)
-    print("test")
-    print(test_df.shape)
-
-    if do_minirocket:
-        cols = ['acc_x', 'acc_y', 'acc_z', 'gyro_x', 'gyro_y', 'gyro_z']
-        x_train_df, y_train_df = get_df_windows(train_df,
-                                                imu_df_win_task,
-                                                window_size=window_size,
-                                                window_shift=window_shift,
-                                                fs=fs,
-                                               )
-        x_test_df, y_test_df = get_df_windows(test_df, 
-                                              imu_df_win_task,
-                                              window_size=window_size,
-                                              window_shift=window_shift,
-                                              fs=fs,
-                                             )
-
-        x_train = make_windows_from_id(x_train_df, cols)
-        x_test  = make_windows_from_id(x_test_df, cols)
-        y_train = y_train_df[lbl_str].values.reshape(-1, 1)
-        y_test  = y_test_df[lbl_str].values.reshape(-1, 1)
-
-        print("minirocket transforming...")
-        x_train = np.swapaxes(x_train, 1, 2)
-        x_test = np.swapaxes(x_test, 1, 2)
-        minirocket = MiniRocketMultivariate()
-        x_train    = minirocket.fit_transform(x_train)
-        x_test     = minirocket.transform(x_test)
-    elif use_tsfresh:
-        x_train = train_df.iloc[:, 3:].values
-        y_train = train_df[lbl_str].values.reshape(-1, 1)
-        x_test  = test_df.iloc[:, 3:].values
-        y_test  = test_df[lbl_str].values.reshape(-1, 1)
-    else:
-        cols = ['acc_x', 'acc_y', 'acc_z', 'gyro_x', 'gyro_y', 'gyro_z']
-        x_train_df, y_train_df = get_df_windows(train_df,
-                                                imu_df_win_task,
-                                                window_size=window_size,
-                                                window_shift=window_shift,
-                                                fs=fs,
-                                               )
-        x_test_df, y_test_df = get_df_windows(test_df, 
-                                              imu_df_win_task,
-                                              window_size=window_size,
-                                              window_shift=window_shift,
-                                              fs=fs,
-                                             )
-
-        x_train = make_windows_from_id(x_train_df, cols)
-        x_test  = make_windows_from_id(x_test_df, cols)
-        y_train = y_train_df[lbl_str].values.reshape(-1, 1)
-        y_test  = y_test_df[lbl_str].values.reshape(-1, 1)
-
-    transforms, model = model_training(mdl_str, x_train, y_train,
-                                       marker, validation_data=None,
-                                       overwrite=overwrite,
-                                       is_regression=True,
-                                       project_directory=project_dir,
-                                       window_size=int(window_size*fs),
-                                       extra_train=200,
-                                      )
-
-    if transforms is not None:
-        x_test = transforms.transform(x_test)
-
-    preds = model.predict(x_test)
-
-    eval_handle = EvalHandler(y_test.flatten(), preds.flatten(), subject,
-                              pfh, mdl_str, overwrite=overwrite)
-    eval_handle.update_eval_history()
-    eval_handle.save_eval_history()
-    pp = PrettyPrinter()
-    pp.pprint(eval_handle.load_eval_history())
-
-    fig, ax = plt.subplots()
-    ax.plot(y_test)
-    ax.plot(preds)
-    ax.set_title(' '.join([mdl_str, subject]))
-    ax.legend([lbl_str, 'pred'])
-    fig_dir = join(project_dir, 'figures',)
-    if not exists(fig_dir): mkdir(fig_dir)
-    fig.savefig(join(fig_dir, mdl_str))
+    for combi in combinations(cal_df[cal_str].values, train_len):
+        combi_str = "-".join([str(x) for x in combi])
+        pfh.config[cal_str] = combi_str
+        marker = f'imu_rr_{subject}_id{pfh.fset_id}_combi{combi_str}'
+        print(marker)
+
+        train_df = pd.concat(
+            [cal_df[cal_df[cal_str] == cpm]['data'].iloc[0] for cpm in combi],
+            axis=0
+        )
+
+        assert np.isin(train_df.index.values, test_df.index.values).any()==False,\
+                "overlapping test and train data"
+
+        print("train")
+        print(train_df.shape)
+        print("test")
+        print(test_df.shape)
+
+        if do_minirocket:
+            x_train_df, y_train_df = get_df_windows(train_df,
+                                                    imu_df_win_task,
+                                                    window_size=window_size,
+                                                    window_shift=window_shift,
+                                                    fs=fs,
+                                                   )
+            x_test_df, y_test_df = get_df_windows(test_df, 
+                                                  imu_df_win_task,
+                                                  window_size=window_size,
+                                                  window_shift=window_shift,
+                                                  fs=fs,
+                                                 )
+
+            x_train = make_windows_from_id(x_train_df, imu_cols)
+            x_test  = make_windows_from_id(x_test_df, imu_cols)
+            y_train = y_train_df[lbl_str].values.reshape(-1, 1)
+            y_test  = y_test_df[lbl_str].values.reshape(-1, 1)
+
+            print("minirocket transforming...")
+            x_train = np.swapaxes(x_train, 1, 2)
+            x_test = np.swapaxes(x_test, 1, 2)
+            minirocket = MiniRocketMultivariate()
+            x_train    = minirocket.fit_transform(x_train)
+            x_test     = minirocket.transform(x_test)
+        elif use_tsfresh:
+            x_train = train_df.iloc[:, 3:].values
+            y_train = train_df[lbl_str].values.reshape(-1, 1)
+            x_test  = test_df.iloc[:, 3:].values
+            y_test  = test_df[lbl_str].values.reshape(-1, 1)
+        else:
+            x_train_df, y_train_df = get_df_windows(train_df,
+                                                    imu_df_win_task,
+                                                    window_size=window_size,
+                                                    window_shift=window_shift,
+                                                    fs=fs,
+                                                   )
+            x_test_df, y_test_df = get_df_windows(test_df, 
+                                                  imu_df_win_task,
+                                                  window_size=window_size,
+                                                  window_shift=window_shift,
+                                                  fs=fs,
+                                                 )
+
+            x_train = make_windows_from_id(x_train_df, imu_cols)
+            x_test  = make_windows_from_id(x_test_df, imu_cols)
+            y_train = y_train_df[lbl_str].values.reshape(-1, 1)
+            y_test  = y_test_df[lbl_str].values.reshape(-1, 1)
+
+        transforms, model = model_training(mdl_str, x_train, y_train,
+                                           marker, validation_data=None,
+                                           overwrite=overwrite,
+                                           is_regression=True,
+                                           project_directory=project_dir,
+                                           window_size=int(window_size*fs),
+                                           extra_train=200,
+                                          )
+
+        if transforms is not None:
+            x_test = transforms.transform(x_test)
+
+        preds = model.predict(x_test)
+
+        eval_handle = EvalHandler(y_test.flatten(), preds.flatten(), subject,
+                                  pfh, mdl_str, overwrite=overwrite)
+        eval_handle.update_eval_history()
+        eval_handle.save_eval_history()
+
+        pp = PrettyPrinter()
+        pp.pprint(eval_handle.load_eval_history())
+
+        fig, ax = plt.subplots()
+        fig_title = '_'.join([mdl_str, subject]+[combi_str])
+        ax.plot(y_test)
+        ax.plot(preds)
+        ax.set_title(fig_title)
+        ax.legend([lbl_str, 'pred'])
+        fig_dir = join(project_dir, 'figures')
+        if not exists(fig_dir): mkdir(fig_dir)
+        fig.savefig(join(fig_dir, fig_title+".png"))
+        plt.close()
 
 def arg_parser():
     parser = argparse.ArgumentParser()
@@ -717,7 +791,7 @@ def arg_parser():
                                  'elastic'],
                        )
     parser.add_argument("-s", '--subject', type=int,
-                        default=1,
+                        default=2,
                         choices=list(range(1,3))+[-1],
                        )
     parser.add_argument("-f", '--feature_method', type=str,
@@ -734,10 +808,10 @@ def arg_parser():
                         default=0.2,
                        )
     parser.add_argument('-l', '--lbl_str', type=str,
-                        default='br',
+                        default='pss',
                        )
     parser.add_argument('-tl', '--train_len', type=int,
-                        default=1,
+                        default=3,
                         help='minutes of data to use for calibration'
                        )
     args = parser.parse_args()
@@ -748,13 +822,6 @@ if __name__ == '__main__':
     # '[!M]*'
     np.random.seed(100)
     n_subject_max = 2
-    xsens_df = load_and_sync_xsens('Pilot02')
-
-    activity_df = get_activity_log('Pilot02')
-    event_df = get_respiration_log('Pilot02')
-
-    ipdb.set_trace()
-    
     args = arg_parser()
 
     mdl_str        = args.model