refactorings and renamings for unifying various generators

Author: asavinov

common/feature_generation.py common/gen_features.py

@@ -12,8 +12,8 @@
 import scipy.stats as stats
 
 from common.utils import *
-from common.feature_generation_rolling_agg import *
-from common.feature_generation_rolling_agg import _aggregate_last_rows
+from common.gen_features_rolling_agg import *
+from common.gen_features_rolling_agg import _aggregate_last_rows
 
 """
 Feature generators. 

common/feature_generation_rolling_agg.py common/gen_features_rolling_agg.py

@@ -7,8 +7,8 @@
 import pandas as pd
 
 from common.utils import *
-from common.feature_generation import *
-from common.feature_generation_rolling_agg import *
+from common.gen_features import *
+from common.gen_features_rolling_agg import *
 
 """
 Label generation. Labels are features which are used for training.

common/label_generation_highlow.py common/gen_labels_highlow.py

@@ -10,7 +10,7 @@
 
 from binance.helpers import date_to_milliseconds, interval_to_milliseconds
 
-from common.feature_generation import *
+from common.gen_features import *
 
 #
 # Decimals

common/label_generation_topbot.py common/gen_labels_topbot.py

@@ -11,7 +11,7 @@
 import numpy as np
 
 from common.utils import *
-from common.feature_generation import *
+from common.gen_features import *
 from common.depth_processing import *
 
 """

common/signal_generation.py common/gen_signals.py

@@ -6,13 +6,8 @@
 import pandas as pd
 
 from service.App import *
-from common.feature_generation import *
-from common.label_generation_highlow import generate_labels_highlow, generate_labels_highlow2
-from common.label_generation_topbot import generate_labels_topbot, generate_labels_topbot2
-from common.signal_generation import (
-    generate_smoothen_scores, generate_combine_scores,
-    generate_threshold_rule, generate_threshold_rule2
-)
+from common.generators import generate_feature_set
+
 
 #
 # Parameters
@@ -103,91 +98,5 @@ def main(config_file):
     print(f"Output file location: {out_path}")
 
 
-def generate_feature_set(df: pd.DataFrame, fs: dict, last_rows: int) -> Tuple[pd.DataFrame, list]:
-    """
-    Apply the specified resolved feature generator to the input data set.
-    """
-
-    #
-    # Select columns from the data set to be processed by the feature generator
-    #
-    cp = fs.get("column_prefix")
-    if cp:
-        cp = cp + "_"
-        f_cols = [col for col in df if col.startswith(cp)]
-        f_df = df[f_cols]  # Alternatively: f_df = df.loc[:, df.columns.str.startswith(cf)]
-        # Remove prefix because feature generators are generic (a prefix will be then added to derived features before adding them back to the main frame)
-        f_df = f_df.rename(columns=lambda x: x[len(cp):] if x.startswith(cp) else x)  # Alternatively: f_df.columns = f_df.columns.str.replace(cp, "")
-    else:
-        f_df = df[df.columns.to_list()]  # We want to have a different data frame object to add derived featuers and then join them back to the main frame with prefix
-
-    #
-    # Resolve and apply feature generator functions from the configuration
-    #
-    generator = fs.get("generator")
-    gen_config = fs.get('config', {})
-    if generator == "itblib":
-        features = generate_features_itblib(f_df, gen_config, last_rows=last_rows)
-    elif generator == "depth":
-        features = generate_features_depth(f_df)
-    elif generator == "tsfresh":
-        features = generate_features_tsfresh(f_df, gen_config, last_rows=last_rows)
-    elif generator == "talib":
-        features = generate_features_talib(f_df, gen_config, last_rows=last_rows)
-    elif generator == "itbstats":
-        features = generate_features_itbstats(f_df, gen_config, last_rows=last_rows)
-
-    # Labels
-    elif generator == "highlow":
-        horizon = gen_config.get("horizon")
-
-        # Binary labels whether max has exceeded a threshold or not
-        print(f"Generating 'highlow' labels with horizon {horizon}...")
-        features = generate_labels_highlow(f_df, horizon=horizon)
-
-        print(f"Finished generating 'highlow' labels. {len(features)} labels generated.")
-    elif generator == "highlow2":
-        print(f"Generating 'highlow2' labels...")
-        f_df, features = generate_labels_highlow2(f_df, gen_config)
-        print(f"Finished generating 'highlow2' labels. {len(features)} labels generated.")
-    elif generator == "topbot":
-        column_name = gen_config.get("columns", "close")
-
-        top_level_fracs = [0.01, 0.02, 0.03, 0.04, 0.05]
-        bot_level_fracs = [-x for x in top_level_fracs]
-
-        f_df, features = generate_labels_topbot(f_df, column_name, top_level_fracs, bot_level_fracs)
-    elif generator == "topbot2":
-        f_df, features = generate_labels_topbot2(f_df, gen_config)
-
-    # Signals
-    elif generator == "smoothen":
-        f_df, features = generate_smoothen_scores(f_df, gen_config)
-    elif generator == "combine":
-        f_df, features = generate_combine_scores(f_df, gen_config)
-    elif generator == "threshold_rule":
-        f_df, features = generate_threshold_rule(f_df, gen_config)
-    elif generator == "threshold_rule2":
-        f_df, features = generate_threshold_rule2(f_df, gen_config)
-
-    else:
-        print(f"Unknown feature generator {generator}")
-        return
-
-    #
-    # Add generated features to the main data frame with all other columns and features
-    #
-    f_df = f_df[features]
-    fp = fs.get("feature_prefix")
-    if fp:
-        f_df = f_df.add_prefix(fp + "_")
-
-    new_features = f_df.columns.to_list()
-
-    df = df.join(f_df)  # Attach all derived features to the main frame
-
-    return df, new_features
-
-
 if __name__ == '__main__':
     main()

common/utils.py

@@ -5,8 +5,6 @@
 
 from service.App import *
 from scripts.features import generate_feature_set
-from common.label_generation_highlow import *
-from common.label_generation_topbot import *
 
 """
 This script will load a feature file (or any file with close price), and add

scripts/depth_to_features.py

@@ -7,8 +7,8 @@
 import pandas as pd
 
 from service.App import *
+from common.gen_features import *
 from common.classifiers import *
-from common.feature_generation import *
 from common.model_store import *
 
 """

scripts/features.py

@@ -8,8 +8,8 @@
 
 from service.App import *
 from common.utils import *
+from common.gen_features import *
 from common.classifiers import *
-from common.feature_generation import *
 from common.model_store import *
 
 """

scripts/labels.py

@@ -9,13 +9,11 @@
 from sklearn.model_selection import ParameterGrid
 
 from service.App import *
-from common.label_generation_topbot import *
-from common.signal_generation import *
+from common.gen_signals import *
 
 """
-Use predictions to process scores, generate signals and simulate trades over the whole period.
-The results of the trade simulation with signals and performances is stored in the output file.
-The results can be used to further analyze (also visually) the selected signal and trade strategy.
+Generate new derived columns according to the signal definitions.
+The transformations are applied to the results of ML predictions.
 """
 
 class P:
@@ -74,6 +72,11 @@ def main(config_file):
     # Maximum possible on labels themselves
     #performance_long, performance_short, long_count, short_count, long_profitable, short_profitable, longs, shorts = performance_score(df, 'top10_2', 'bot10_2', 'close')
 
+
+
+
+
+
     #
     # Aggregate and post-process
     #
@@ -115,6 +118,12 @@ def main(config_file):
     else:  # Default one dim rule
         apply_rule_with_score_thresholds(df, score_column_names, trade_model)
 
+
+
+
+
+
+
     #
     # Simulate trade and compute performance using close price and two boolean signals
     # Add a pair of two dicts: performance dict and model parameters dict

scripts/predict.py

@@ -7,8 +7,8 @@
 import pandas as pd
 
 from service.App import *
+from common.gen_features import *
 from common.classifiers import *
-from common.feature_generation import *
 from common.model_store import *
 
 """

scripts/predict_rolling.py

@@ -10,9 +10,8 @@
 
 from service.App import *
 from common.utils import *
+from common.gen_signals import *
 from common.classifiers import *
-from common.label_generation_topbot import *
-from common.signal_generation import *
 
 """
 Input data:

scripts/signals.py

@@ -10,9 +10,8 @@
 
 from service.App import *
 from common.utils import *
+from common.generators import generate_feature_set
 from common.classifiers import *
-from common.feature_generation import *
-from common.signal_generation import *
 from common.model_store import *
 
 from scripts.merge import *

scripts/train.py

@@ -2,8 +2,8 @@
 import numpy.testing as npt
 
 from common.utils import *
-from common.signal_generation import *
-from common.label_generation_topbot import *
+from common.gen_signals import *
+from common.gen_labels_topbot import *
 
 
 def test_extremum_labels():

scripts/train_signals.py

@@ -3,7 +3,7 @@
 
 from common.utils import *
 from common.utils import add_area_ratio
-from common.signal_generation import *
+from common.gen_signals import *
 
 
 def test_decimal():