refactorings and renamings for unifying various generators

asavinov · asavinov · commit cd5db136c4d1 · 2023-12-10T11:47:25.000+01:00
diff --git a/common/gen_features.py b/common/gen_features.py
@@ -12,8 +12,8 @@
 import scipy.stats as stats
 
 from common.utils import *
-from common.feature_generation_rolling_agg import *
-from common.feature_generation_rolling_agg import _aggregate_last_rows
+from common.gen_features_rolling_agg import *
+from common.gen_features_rolling_agg import _aggregate_last_rows
 
 """
 Feature generators. 
diff --git a/common/gen_features_rolling_agg.py b/common/gen_features_rolling_agg.py
diff --git a/common/gen_labels_highlow.py b/common/gen_labels_highlow.py
@@ -7,8 +7,8 @@
 import pandas as pd
 
 from common.utils import *
-from common.feature_generation import *
-from common.feature_generation_rolling_agg import *
+from common.gen_features import *
+from common.gen_features_rolling_agg import *
 
 """
 Label generation. Labels are features which are used for training.
diff --git a/common/gen_labels_topbot.py b/common/gen_labels_topbot.py
diff --git a/common/gen_signals.py b/common/gen_signals.py
diff --git a/common/utils.py b/common/utils.py
@@ -10,7 +10,7 @@
 
 from binance.helpers import date_to_milliseconds, interval_to_milliseconds
 
-from common.feature_generation import *
+from common.gen_features import *
 
 #
 # Decimals
diff --git a/scripts/depth_to_features.py b/scripts/depth_to_features.py
@@ -11,7 +11,7 @@
 import numpy as np
 
 from common.utils import *
-from common.feature_generation import *
+from common.gen_features import *
 from common.depth_processing import *
 
 """
diff --git a/scripts/features.py b/scripts/features.py
@@ -6,13 +6,8 @@
 import pandas as pd
 
 from service.App import *
-from common.feature_generation import *
-from common.label_generation_highlow import generate_labels_highlow, generate_labels_highlow2
-from common.label_generation_topbot import generate_labels_topbot, generate_labels_topbot2
-from common.signal_generation import (
-    generate_smoothen_scores, generate_combine_scores,
-    generate_threshold_rule, generate_threshold_rule2
-)
+from common.generators import generate_feature_set
+
 
 #
 # Parameters
@@ -103,91 +98,5 @@ def main(config_file):
     print(f"Output file location: {out_path}")
 
 
-def generate_feature_set(df: pd.DataFrame, fs: dict, last_rows: int) -> Tuple[pd.DataFrame, list]:
-    """
-    Apply the specified resolved feature generator to the input data set.
-    """
-
-    #
-    # Select columns from the data set to be processed by the feature generator
-    #
-    cp = fs.get("column_prefix")
-    if cp:
-        cp = cp + "_"
-        f_cols = [col for col in df if col.startswith(cp)]
-        f_df = df[f_cols]  # Alternatively: f_df = df.loc[:, df.columns.str.startswith(cf)]
-        # Remove prefix because feature generators are generic (a prefix will be then added to derived features before adding them back to the main frame)
-        f_df = f_df.rename(columns=lambda x: x[len(cp):] if x.startswith(cp) else x)  # Alternatively: f_df.columns = f_df.columns.str.replace(cp, "")
-    else:
-        f_df = df[df.columns.to_list()]  # We want to have a different data frame object to add derived featuers and then join them back to the main frame with prefix
-
-    #
-    # Resolve and apply feature generator functions from the configuration
-    #
-    generator = fs.get("generator")
-    gen_config = fs.get('config', {})
-    if generator == "itblib":
-        features = generate_features_itblib(f_df, gen_config, last_rows=last_rows)
-    elif generator == "depth":
-        features = generate_features_depth(f_df)
-    elif generator == "tsfresh":
-        features = generate_features_tsfresh(f_df, gen_config, last_rows=last_rows)
-    elif generator == "talib":
-        features = generate_features_talib(f_df, gen_config, last_rows=last_rows)
-    elif generator == "itbstats":
-        features = generate_features_itbstats(f_df, gen_config, last_rows=last_rows)
-
-    # Labels
-    elif generator == "highlow":
-        horizon = gen_config.get("horizon")
-
-        # Binary labels whether max has exceeded a threshold or not
-        print(f"Generating 'highlow' labels with horizon {horizon}...")
-        features = generate_labels_highlow(f_df, horizon=horizon)
-
-        print(f"Finished generating 'highlow' labels. {len(features)} labels generated.")
-    elif generator == "highlow2":
-        print(f"Generating 'highlow2' labels...")
-        f_df, features = generate_labels_highlow2(f_df, gen_config)
-        print(f"Finished generating 'highlow2' labels. {len(features)} labels generated.")
-    elif generator == "topbot":
-        column_name = gen_config.get("columns", "close")
-
-        top_level_fracs = [0.01, 0.02, 0.03, 0.04, 0.05]
-        bot_level_fracs = [-x for x in top_level_fracs]
-
-        f_df, features = generate_labels_topbot(f_df, column_name, top_level_fracs, bot_level_fracs)
-    elif generator == "topbot2":
-        f_df, features = generate_labels_topbot2(f_df, gen_config)
-
-    # Signals
-    elif generator == "smoothen":
-        f_df, features = generate_smoothen_scores(f_df, gen_config)
-    elif generator == "combine":
-        f_df, features = generate_combine_scores(f_df, gen_config)
-    elif generator == "threshold_rule":
-        f_df, features = generate_threshold_rule(f_df, gen_config)
-    elif generator == "threshold_rule2":
-        f_df, features = generate_threshold_rule2(f_df, gen_config)
-
-    else:
-        print(f"Unknown feature generator {generator}")
-        return
-
-    #
-    # Add generated features to the main data frame with all other columns and features
-    #
-    f_df = f_df[features]
-    fp = fs.get("feature_prefix")
-    if fp:
-        f_df = f_df.add_prefix(fp + "_")
-
-    new_features = f_df.columns.to_list()
-
-    df = df.join(f_df)  # Attach all derived features to the main frame
-
-    return df, new_features
-
-
 if __name__ == '__main__':
     main()
diff --git a/scripts/labels.py b/scripts/labels.py
@@ -5,8 +5,6 @@
 
 from service.App import *
 from scripts.features import generate_feature_set
-from common.label_generation_highlow import *
-from common.label_generation_topbot import *
 
 """
 This script will load a feature file (or any file with close price), and add
diff --git a/scripts/predict.py b/scripts/predict.py
@@ -7,8 +7,8 @@
 import pandas as pd
 
 from service.App import *
+from common.gen_features import *
 from common.classifiers import *
-from common.feature_generation import *
 from common.model_store import *
 
 """
diff --git a/scripts/predict_rolling.py b/scripts/predict_rolling.py
@@ -8,8 +8,8 @@
 
 from service.App import *
 from common.utils import *
+from common.gen_features import *
 from common.classifiers import *
-from common.feature_generation import *
 from common.model_store import *
 
 """
diff --git a/scripts/signals.py b/scripts/signals.py
@@ -9,13 +9,11 @@
 from sklearn.model_selection import ParameterGrid
 
 from service.App import *
-from common.label_generation_topbot import *
-from common.signal_generation import *
+from common.gen_signals import *
 
 """
-Use predictions to process scores, generate signals and simulate trades over the whole period.
-The results of the trade simulation with signals and performances is stored in the output file.
-The results can be used to further analyze (also visually) the selected signal and trade strategy.
+Generate new derived columns according to the signal definitions.
+The transformations are applied to the results of ML predictions.
 """
 
 class P:
@@ -74,6 +72,11 @@ def main(config_file):
     # Maximum possible on labels themselves
     #performance_long, performance_short, long_count, short_count, long_profitable, short_profitable, longs, shorts = performance_score(df, 'top10_2', 'bot10_2', 'close')
 
+
+
+
+
+
     #
     # Aggregate and post-process
     #
@@ -115,6 +118,12 @@ def main(config_file):
     else:  # Default one dim rule
         apply_rule_with_score_thresholds(df, score_column_names, trade_model)
 
+
+
+
+
+
+
     #
     # Simulate trade and compute performance using close price and two boolean signals
     # Add a pair of two dicts: performance dict and model parameters dict
diff --git a/scripts/train.py b/scripts/train.py
@@ -7,8 +7,8 @@
 import pandas as pd
 
 from service.App import *
+from common.gen_features import *
 from common.classifiers import *
-from common.feature_generation import *
 from common.model_store import *
 
 """
diff --git a/scripts/train_signals.py b/scripts/train_signals.py
@@ -10,9 +10,8 @@
 
 from service.App import *
 from common.utils import *
+from common.gen_signals import *
 from common.classifiers import *
-from common.label_generation_topbot import *
-from common.signal_generation import *
 
 """
 Input data:
diff --git a/service/analyzer.py b/service/analyzer.py
@@ -10,9 +10,8 @@
 
 from service.App import *
 from common.utils import *
+from common.generators import generate_feature_set
 from common.classifiers import *
-from common.feature_generation import *
-from common.signal_generation import *
 from common.model_store import *
 
 from scripts.merge import *
diff --git a/tests/test_label_generation.py b/tests/test_label_generation.py
@@ -2,8 +2,8 @@
 import numpy.testing as npt
 
 from common.utils import *
-from common.signal_generation import *
-from common.label_generation_topbot import *
+from common.gen_signals import *
+from common.gen_labels_topbot import *
 
 
 def test_extremum_labels():
diff --git a/tests/test_utils.py b/tests/test_utils.py
@@ -3,7 +3,7 @@
 
 from common.utils import *
 from common.utils import add_area_ratio
-from common.signal_generation import *
+from common.gen_signals import *
 
 
 def test_decimal():

Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,7 @@`
`10`	`10`
`11`	`11`	`from binance.helpers import date_to_milliseconds, interval_to_milliseconds`
`12`	`12`
`13`		`-from common.feature_generation import *`
	`13`	`+from common.gen_features import *`
`14`	`14`
`15`	`15`	`#`
`16`	`16`	`# Decimals`