regress_rr.py

    plt.plot(acc_y_dsp_df[lbl_str]); plt.plot(acc_dsp_df['pred'])
    plt.subplot(212)
    plt.plot(gyr_y_dsp_df[lbl_str]); plt.plot(gyr_dsp_df['pred'])
    plt.show()

    eval_handle = EvalHandler(y_test.flatten(), preds.flatten(), subject,
                              pfh, None, overwrite=overwrite)
    eval_handle.update_eval_history()
    eval_handle.save_eval_history()

    pp = PrettyPrinter()
    pp.pprint(eval_handle.load_eval_history())

    fig, ax = plt.subplots()
    fig_title = '_'.join([mdl_str, subject]+[combi_str])
    ax.plot(y_test)
    ax.plot(preds)
    ax.set_title(fig_title)
    ax.legend([lbl_str, 'pred'])
    fig_dir = join(project_dir, 'figures')
    if not exists(fig_dir): mkdir(fig_dir)
    fig.savefig(join(fig_dir, fig_title+".png"))
    plt.close()

def sens_rr_model(subject,
                  window_size=12,
                  window_shift=0.2,
                  lbl_str='pss',
                  mdl_str='knn',
                  overwrite=False,
                  feature_method='tsfresh',
                  train_len:int=3,
                  test_standing=False,
                  data_input:str='imu+bvp',
                 ):
    # TODO: 
        # implement tsfresh
    """Loads, preprocesses, and trains a select model using the configured
    settings.
    Attributes
    ----------
    subject: str
        specify the subject code (i.e. 'Pilot02', 'S02')
    window_size : float
        a numpy array of the respiration rate ground truth values from the
        bioharness
    window_shift : float
        a portion of the window size between 0 and 1
    mdl_str : str
        a string to infoa portion of the window size between 0 and 1rm what model was used
    overwrite : bool
        overwrites the evaluations, models, and graphs (default False)
    feature_method : str
        choose between 'minirocket', 'tsfresh', or 'None'
    train_len : int
        number of minutes to sample from, choose between 1 to 7
    test_standing : bool
        boolean to use standing data
    data_input : str
        sensors to use, choose from 'imu', 'bvp', 'imu+bvp'

    Returns
    ------
    None
    """
    cal_str = 'cpm'
    tmp = []
    imu_cols = IMU_COLS
    bvp_cols = ['bvp']
    if 'imu' in data_input and 'bvp' in data_input:
        data_cols = ['acc_x', 'acc_y', 'acc_z',
                     'gyro_x', 'gyro_y', 'gyro_z',
                     'bvp']
        parent_directory_string = "imu-bvp_rr"
        data_input = 'imu+bvp'
        sens_list = ['imu', 'bvp']
        fs = IMU_FS
    elif 'imu' in data_input and not 'bvp' in data_input:
        data_cols = ['acc_x', 'acc_y', 'acc_z',
                     'gyro_x', 'gyro_y', 'gyro_z',]
        parent_directory_string = "imu_rr"
        sens_list = ['imu']
        fs = IMU_FS
    elif not 'imu' in data_input and 'bvp' in data_input:
        data_cols = ['bvp']
        parent_directory_string = "bvp_rr"
        sens_list = ['bvp']
        fs = PPG_FS

    do_minirocket = False
    use_tsfresh   = False
    overwrite_tsfresh = True
    train_size = int(train_len)

    if feature_method == 'tsfresh':
        use_tsfresh = True
    elif feature_method == 'minirocket':
        do_minirocket = True

    config = {'window_size'   : window_size,
              'window_shift'  : window_shift,
              'lbl_str'       : lbl_str,
              'do_minirocket' : do_minirocket,
              'use_tsfresh'   : use_tsfresh,
              'train_len'     : train_len,
              'test_standing' : test_standing,
              'sens_list'     : data_input
             }

    pfh = ProjectFileHandler(config)
    pfh.set_home_directory(join(DATA_DIR, 'subject_specific', subject))
    pfh.set_parent_directory(parent_directory_string)
    id_check = pfh.get_id_from_config()
    if id_check is None:
        pfh.set_project_directory()
        pfh.save_metafile()
    else:
        pfh.set_id(int(id_check))
        pfh.set_project_directory()
        print('Using pre-set data id: ', pfh.fset_id)
    project_dir = pfh.project_directory

    xsens_df = load_and_sync_xsens(subject, sens_list=sens_list)
    activity_df = get_activity_log(subject)
    event_df = get_respiration_log(subject)

    cal_df = get_cal_data(event_df, xsens_df)

    if use_tsfresh:
        xsens_df = load_tsfresh(xsens_df,
                                project_dir,
                                sens_list=sens_list,
                                window_size=window_size,
                                window_shift=window_shift,
                                fs=fs,
                                overwrite=overwrite_tsfresh,
                                data_cols=data_cols,
                               )

    # include standing or not
    test_df_tmp = get_test_data(cal_df, activity_df, xsens_df, test_standing)
    test_df = pd.concat([df for df in test_df_tmp['data']], axis=0)

    x_test_df, y_test_df = get_df_windows(
        test_df, df_win_task, window_size=window_size,
        window_shift=window_shift, fs=fs, cols=data_cols)


    for combi in combinations(cal_df[cal_str].values, train_len):
        combi_str = "-".join([str(x) for x in combi])
        pfh.config[cal_str] = combi_str
        marker = f'{parent_directory_string}_{subject}_id{pfh.fset_id}'\
                f'_combi{combi_str}'
        print(marker)

        train_df_list = []
        for cpm in combi:
            df = cal_df[cal_df[cal_str] == cpm]
            data_df = df['data'].iloc[0]
            data_df['cpm'] = cpm
            train_df_list.append(data_df)
        train_df = pd.concat(train_df_list)

        assert np.isin(train_df.index.values, test_df.index.values).any()==False,\
                "overlapping test and train data"

        print("train")
        print(train_df.shape)
        print("test")
        print(test_df.shape)

        if do_minirocket:
            x_train_df, y_train_df = get_df_windows(train_df,
                                                    df_win_task,
                                                    window_size=window_size,
                                                    window_shift=window_shift,
                                                    fs=fs,
                                                    cols=data_cols
                                                   )

            x_train = make_windows_from_id(x_train_df, data_cols)
            y_train = y_train_df['cpm'].values.reshape(-1, 1)
            x_test  = make_windows_from_id(x_test_df, data_cols)
            y_test  = y_test_df[lbl_str].values.reshape(-1, 1)
    
            # x_train = y_train_df['bvp_est'].values.reshape(-1, 1)
            # x_test  = y_test_df['bvp_est'].values.reshape(-1, 1)
            print("minirocket transforming...")
            x_train = np.swapaxes(x_train, 1, 2)
            x_test = np.swapaxes(x_test, 1, 2)
            minirocket = MiniRocketMultivariate()
            x_train    = minirocket.fit_transform(x_train)
            x_test     = minirocket.transform(x_test)
        elif use_tsfresh:
            x_train = train_df.iloc[:, 3:].values
            y_train = train_df['cpm'].values.reshape(-1, 1)
            x_test  = test_df.iloc[:, 3:].values
            y_test  = test_df[lbl_str].values.reshape(-1, 1)
        else:
            x_train_df, y_train_df = get_df_windows(train_df,
                                                    df_win_task,
                                                    window_size=window_size,
                                                    window_shift=window_shift,
                                                    fs=fs,
                                                    cols=data_cols,
                                                   )
            x_train = make_windows_from_id(x_train_df, data_cols)
            x_test  = make_windows_from_id(x_test_df, data_cols)
            y_train = y_train_df['cpm'].values.reshape(-1, 1)
            y_test  = y_test_df[lbl_str].values.reshape(-1, 1)

        transforms, model = model_training(mdl_str, x_train, y_train,
                                           marker, validation_data=None,
                                           overwrite=overwrite,
                                           is_regression=True,
                                           project_directory=project_dir,
                                           window_size=int(window_size*fs),
                                           extra_train=200,
                                           poly_deg=1
                                          )

        if transforms is not None:
            x_test = transforms.transform(x_test)

        preds = model.predict(x_test)

        eval_handle = EvalHandler(y_test.flatten(), preds.flatten(), subject,
                                  pfh, mdl_str, overwrite=overwrite)
        eval_handle.update_eval_history()
        eval_handle.save_eval_history()

        pp = PrettyPrinter()
        pp.pprint(eval_handle.load_eval_history())

        fig, ax = plt.subplots(2, 1, figsize=(7.3, 4.5))
        fig_title = '_'.join([mdl_str, data_input, subject]+[combi_str])
        fig.suptitle(fig_title)
        ax[0].plot(y_test)
        ax[0].plot(preds)
        ax[0].set_title('raw')

        if lbl_str == 'pss':
            br  = y_test_df['br'].values
            ax[1].plot(movingaverage(y_test, 12), color='tab:blue')
            ax[1].plot(br, 'k')
            ax[1].plot(movingaverage(preds, 12), color='tab:orange')
            ax[1].legend([lbl_str, 'br', 'pred'])
        else:
            ax[1].plot(y_test, 'k')
            ax[1].plot(movingaverage(preds, 12), color='tab:orange')
            ax[1].legend([lbl_str, 'pred'])
        ax[1].set_title('smoothened')
        fig_dir = join(project_dir, 'figures')
        if not exists(fig_dir): mkdir(fig_dir)
        fig.savefig(join(fig_dir, fig_title+".png"))
        plt.close()

def arg_parser():
    """Returns arguments in a Namespace to configure the subject specific model
    """
    parser = argparse.ArgumentParser()
    parser.add_argument("-m", '--model', type=str,
                        default='linreg',
                        choices=['linreg', 'ard', 'xgboost', 'knn',
                                 'svr', 'cnn1d', 'fnn', 'lstm', 'ridge',
                                 'elastic'],
                       )
    parser.add_argument("-s", '--subject', type=int,
                        default=2,
                        choices=list(range(1,4))+[-1],
                       )
    parser.add_argument("-f", '--feature_method', type=str,
                        default='minirocket',
                        choices=['tsfresh', 'minirocket', 'None']
                       )
    parser.add_argument("-o", '--overwrite', type=int,
                        default=0,
                       )
    parser.add_argument('--win_size', type=int,
                        default=12,
                       )
    parser.add_argument('--win_shift', type=float,
                        default=0.2,
                       )
    parser.add_argument('-l', '--lbl_str', type=str,
                        default='pss',
                       )
    parser.add_argument('-tl', '--train_len', type=int,
                        default=3,
                        help='minutes of data to use for calibration'
                       )
    parser.add_argument('-d', '--data_input', type=str,
                        default='imu',
                        help='imu, bvp, imu+bvp: select data cols for input'
                       )
    parser.add_argument('-ts', '--test_standing', type=int,
                        default=0,
                        help='1 or 0 input, choose if standing data will be '\
                        'recorded or not'
                       )
    args = parser.parse_args()
    return args

if __name__ == '__main__':
    np.random.seed(100)
    n_subject_max = 2
    args = arg_parser()

    # Load command line arguments
    mdl_str        = args.model
    subject        = args.subject
    feature_method = args.feature_method
    window_size    = args.win_size
    window_shift   = args.win_shift
    lbl_str        = args.lbl_str
    train_len      = args.train_len
    overwrite      = args.overwrite
    data_input     = args.data_input
    test_standing  = args.test_standing

    print(args)
    assert train_len>0,"--train_len must be an integer greater than 0"

    subject_pre_string = 'Pilot'

    if subject > 0:
        subject = subject_pre_string+str(subject).zfill(2)

        sens_rr_model(subject,
                      window_size=window_size,
                      window_shift=window_shift,
                      lbl_str=lbl_str,
                      mdl_str=mdl_str,
                      overwrite=overwrite,
                      feature_method=feature_method,
                      train_len=train_len,
                      test_standing=test_standing,
                      data_input=data_input,
                     )
    else:
        subjects = [subject_pre_string+str(i).zfill(2) for i in \
                    range(1, n_subject_max+1) if i not in imu_issues]

        rr_func = partial(sens_rr_model,
                          window_size=window_size,
                          window_shift=window_shift,
                          lbl_str=lbl_str,
                          mdl_str=mdl_str,
                          overwrite=overwrite,
                          feature_method=feature_method,
                          train_len=train_len,
                          test_standing=test_standing,
                          data_input=data_input,
                         )

        if mdl_str in ['fnn', 'lstm', 'cnn1d', 'elastic', 'ard', 'xgboost']:
            for subject in subjects:
                rr_func(subject)
        else:
            ncpu = min(len(subjects), cpu_count())
            with Pool(ncpu) as p:
                p.map(rr_func, subjects)

    print(args)