-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathgenerate_dataset.py
More file actions
70 lines (51 loc) · 2.08 KB
/
generate_dataset.py
File metadata and controls
70 lines (51 loc) · 2.08 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
import argparse
import sys
import pandas as pd
from core.features import FeatureEngineer
from utilities.settings import logger
def main():
parser = argparse.ArgumentParser(description="Generate technical features for crypto 1h data.")
# Arguments
parser.add_argument("-i", "--input_file", type=str, required=True, help="Path to raw CSV file")
parser.add_argument("-o", "--output_file", type=str, required=True, help="Path to save feature CSV")
args = parser.parse_args()
# 1. Init Classes
engineer = FeatureEngineer()
print(f"--- Loading: {args.input_file} ---")
try:
df = pd.read_csv(args.input_file)
except FileNotFoundError:
sys.exit(f"Error: File {args.input_file} not found.")
# Standardize Timestamp
# Assuming 'timestamp' exists as per prompt. If 'timestamp_open' is the main one, adjust here.
if 'timestamp' in df.columns:
df['ts'] = pd.to_datetime(df['timestamp'])
elif 'timestamp_open' in df.columns:
df['ts'] = pd.to_datetime(df['timestamp_open'])
else:
sys.exit("Error: Could not find 'timestamp' or 'timestamp_open' column.")
# Sort
df_raw = df.sort_values('ts').reset_index(drop=True)
df_raw.sort_index(inplace=True)
if df_raw.empty:
sys.exit("Error: No data received from API.")
logger.info(f"loaded {len(df_raw)} rows.")
# 3. Apply Features
logger.info("--- Engineering Features ---")
df_features = engineer.gen_features(df_raw)
# 4. Clean (Drop NaNs from lookback periods)
initial_len = len(df_features)
df_clean = engineer.clean_data(df_features)
dropped = initial_len - len(df_clean)
logger.info(f"Dropped {dropped} rows due to rolling window warmup.")
# 5. Save
df_clean.to_csv(args.output_file, index=False)
# Summary
logger.info(f"--- Success ---")
logger.info(f"Saved to: {args.output_file}")
logger.info(f"Final Shape: {df_clean.shape}")
logger.info("\nSample Data:")
cols = ['uid', 'close', 'log_ret', 'accel', 'rsi14']
print(df_clean[cols].tail())
if __name__ == "__main__":
main()