Skip to content

Commit 368f7bf

Browse files
MDUYNMDUYN
committed
Fix permutation testing saving
1 parent d8ff6ca commit 368f7bf

File tree

4 files changed

+296
-9
lines changed

4 files changed

+296
-9
lines changed

‎investing_algorithm_framework/domain/backtesting/backtest.py‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -362,15 +362,14 @@ def save(self, directory_path: Union[str, Path]) -> None:
362362
self.backtest_summary.save(summary_file)
363363

364364
if self.backtest_permutation_tests:
365-
permutation_path = os.path.join(
365+
permutation_dir_path = os.path.join(
366366
directory_path, "permutation_tests"
367367
)
368-
os.makedirs(permutation_path, exist_ok=True)
368+
os.makedirs(permutation_dir_path, exist_ok=True)
369369

370370
for pm in self.backtest_permutation_tests:
371371
dir_name = pm.create_directory_name()
372-
pm_path = os.path.join(permutation_path, dir_name)
373-
os.makedirs(pm_path, exist_ok=True)
372+
pm_path = os.path.join(permutation_dir_path, dir_name)
374373
pm.save(pm_path)
375374

376375
# Save metadata if available
@@ -407,6 +406,19 @@ def save(self, directory_path: Union[str, Path]) -> None:
407406
{'algorithm_id': self.algorithm_id}, f, indent=4
408407
)
409408

409+
# Save the permutation tests if available
410+
if self.backtest_permutation_tests:
411+
permutation_tests_path = os.path.join(
412+
directory_path, "permutation_tests"
413+
)
414+
os.makedirs(permutation_tests_path, exist_ok=True)
415+
416+
for bpt in self.backtest_permutation_tests:
417+
dir_name = bpt.create_directory_name()
418+
bpt_path = os.path.join(permutation_tests_path, dir_name)
419+
os.makedirs(bpt_path, exist_ok=True)
420+
bpt.save(bpt_path)
421+
410422
def __repr__(self):
411423
"""
412424
Return a string representation of the Backtest instance.

‎investing_algorithm_framework/domain/backtesting/backtest_permutation_test.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,12 @@ def summary(
133133
def save(self, path: str) -> None:
134134
"""
135135
Save the permutation test results to disk (JSON + Parquet).
136+
137+
Args:
138+
path (str): The directory path where to save the results.
139+
140+
Returns:
141+
None
136142
"""
137143
os.makedirs(path, exist_ok=True)
138144

@@ -153,6 +159,12 @@ def save(self, path: str) -> None:
153159
def open(path: str) -> "BacktestPermutationTest":
154160
"""
155161
Load the permutation test results from disk (JSON + Parquet).
162+
163+
Args:
164+
path (str): The directory path where the results are saved.
165+
166+
Returns:
167+
BacktestPermutationTest: The loaded permutation test results.
156168
"""
157169
original_metrics = os.path.join(path, "original_metrics.json")
158170

@@ -198,6 +210,9 @@ def create_directory_name(self) -> str:
198210
def to_dict(self) -> Dict:
199211
"""
200212
Convert the permutation test results to a dictionary.
213+
214+
Returns:
215+
dict: A dictionary representation of the permutation test results.
201216
"""
202217
return {
203218
"real_metrics": self.real_metrics.to_dict(),

‎investing_algorithm_framework/domain/backtesting/combine_backtests.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,11 @@ def combine_backtests(backtests):
6464
)
6565
unique_date_ranges.add(date_range)
6666

67-
# Get first non-empty metadata
67+
# Merge all metadata dictionaries
68+
metadata = {}
6869
for backtest in backtests:
6970
if backtest.metadata:
70-
metadata = backtest.metadata
71-
break
71+
metadata.update(backtest.metadata)
7272

7373
# Get the first risk-free rate
7474
for backtest in backtests:

‎tests/domain/models/backtesting/test_backtest.py‎

Lines changed: 262 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,231 @@ def setUp(self):
4747
"backtest_data/OHLCV_BTC-EUR_BINANCE_2h_2020-12-15-06-00_2021-01-01-00-30.csv"
4848
)
4949

50+
# Test models
51+
self.backtest_metrics_run_one = BacktestMetrics(
52+
backtest_start_date=datetime(2020, 1, 1),
53+
backtest_end_date=datetime(2020, 12, 31),
54+
equity_curve=[
55+
(0.0, datetime(2020, 1, 1)),
56+
(1.0, datetime(2020, 12, 31)),
57+
(0.5, datetime(2020, 6, 30)),
58+
(0.2, datetime(2020, 3, 31))
59+
],
60+
total_growth=1.0,
61+
total_growth_percentage=1.0,
62+
total_net_gain=1.0,
63+
total_net_gain_percentage=1.0,
64+
final_value=1.0,
65+
cagr=1.0,
66+
sharpe_ratio=0.0,
67+
rolling_sharpe_ratio=[
68+
(0.0, datetime(2020, 1, 1)),
69+
(1.0, datetime(2020, 12, 31)),
70+
(0.5, datetime(2020, 6, 30)),
71+
(0.2, datetime(2020, 3, 31))
72+
],
73+
sortino_ratio=0.0,
74+
calmar_ratio=0.0,
75+
profit_factor=0.0,
76+
gross_profit=0.0,
77+
gross_loss=0.0,
78+
annual_volatility=0.0,
79+
monthly_returns=[
80+
(0.0, datetime(2020, 1, 1)), (0.0, datetime(2020, 2, 1)),
81+
(0.0, datetime(2020, 3, 1)),
82+
(0.0, datetime(2020, 4, 1)), (0.0, datetime(2020, 5, 1)),
83+
(0.0, datetime(2020, 6, 1)),
84+
(0.0, datetime(2020, 7, 1)), (0.0, datetime(2020, 8, 1)),
85+
(0.0, datetime(2020, 9, 1)),
86+
(0.0, datetime(2020, 10, 1)), (0.0, datetime(2020, 11, 1)),
87+
(0.0, datetime(2020, 12, 1))
88+
],
89+
yearly_returns=[
90+
(0.0, date(2020, 1, 1)), (0.0, date(2020, 12, 31))
91+
],
92+
drawdown_series=[
93+
(0.0, datetime(2020, 1, 1)), (0.0, datetime(2020, 2, 1)),
94+
(0.0, datetime(2020, 3, 1)),
95+
(0.0, datetime(2020, 4, 1)), (0.0, datetime(2020, 5, 1)),
96+
(0.0, datetime(2020, 6, 1)),
97+
(0.0, datetime(2020, 7, 1)), (0.0, datetime(2020, 8, 1)),
98+
(0.0, datetime(2020, 9, 1)),
99+
(0.0, datetime(2020, 10, 1)), (0.0, datetime(2020, 11, 1)),
100+
(0.0, datetime(2020, 12, 1))
101+
],
102+
max_drawdown=0.0,
103+
max_drawdown_absolute=0.0,
104+
max_daily_drawdown=0.0,
105+
max_drawdown_duration=0,
106+
trades_per_year=0.0,
107+
trade_per_day=0.0,
108+
exposure_ratio=0.0,
109+
average_trade_gain=0.0,
110+
average_trade_gain_percentage=0.0,
111+
average_trade_loss=0.0,
112+
average_trade_loss_percentage=0.0,
113+
best_trade=Trade(
114+
id=10,
115+
open_price=0.0,
116+
opened_at=datetime(2020, 1, 1),
117+
closed_at=datetime(2020, 12, 31),
118+
orders=[],
119+
target_symbol="BTC",
120+
trading_symbol="EUR",
121+
amount=10.0,
122+
cost=1.0,
123+
available_amount=1.0,
124+
remaining=9.0,
125+
filled_amount=1,
126+
status="closed"
127+
),
128+
worst_trade=Trade(
129+
id=10,
130+
open_price=0.0,
131+
opened_at=datetime(2020, 1, 1),
132+
closed_at=datetime(2020, 12, 31),
133+
orders=[],
134+
target_symbol="BTC",
135+
trading_symbol="EUR",
136+
amount=10.0,
137+
cost=1.0,
138+
available_amount=1.0,
139+
remaining=9.0,
140+
filled_amount=1,
141+
status="closed"
142+
),
143+
average_trade_duration=0.0,
144+
number_of_trades=0,
145+
win_rate=0.0,
146+
win_loss_ratio=0.0,
147+
percentage_winning_months=0.0,
148+
percentage_winning_years=0.0,
149+
average_monthly_return=0.0,
150+
average_monthly_return_losing_months=0.0,
151+
average_monthly_return_winning_months=0.0,
152+
best_month=(0.0, datetime(2020, 1, 1)),
153+
best_year=(0.0, date(2020, 1, 1)),
154+
worst_month=(0.0, datetime(2020, 1, 1)),
155+
worst_year=(0.0, date(2020, 1, 1))
156+
)
157+
158+
self.backtest_run_one = BacktestRun(
159+
backtest_start_date=datetime(2020, 1, 1),
160+
backtest_end_date=datetime(2020, 12, 31),
161+
trading_symbol="EUR",
162+
initial_unallocated=1000.0,
163+
number_of_runs=50,
164+
portfolio_snapshots=[
165+
PortfolioSnapshot(
166+
created_at=datetime(2020, 1, 1),
167+
total_value=1000.0,
168+
unallocated=1000.0,
169+
pending_value=100.0,
170+
cash_flow=0.0,
171+
total_cost=0.0,
172+
),
173+
PortfolioSnapshot(
174+
created_at=datetime(2020, 12, 31),
175+
total_value=1100.0,
176+
unallocated=100.0,
177+
pending_value=100.0,
178+
cash_flow=0.0,
179+
total_cost=0.0,
180+
)
181+
],
182+
trades=[
183+
Trade(
184+
id=10,
185+
open_price=0.0,
186+
opened_at=datetime(2020, 1, 1),
187+
closed_at=datetime(2020, 12, 31),
188+
orders=[],
189+
target_symbol="BTC",
190+
trading_symbol="EUR",
191+
amount=10.0,
192+
cost=1.0,
193+
available_amount=1.0,
194+
remaining=9.0,
195+
filled_amount=1,
196+
status="closed"
197+
)
198+
],
199+
orders=[
200+
Order(
201+
id=1,
202+
order_type="LIMIT",
203+
price=100.0,
204+
amount=0.1,
205+
target_symbol="BTC",
206+
trading_symbol="EUR",
207+
created_at=datetime(2020, 1, 1),
208+
updated_at=datetime(2020, 1, 1),
209+
status="CLOSED",
210+
remaining=10.0,
211+
filled=10,
212+
cost=1000.0,
213+
order_side="BUY"
214+
)
215+
],
216+
positions=[
217+
Position(
218+
symbol="BTC/EUR",
219+
amount=0.1,
220+
)
221+
],
222+
created_at=datetime(2020, 1, 1),
223+
symbols=["BTC/EUR"],
224+
number_of_days=0,
225+
number_of_trades=0,
226+
number_of_trades_closed=0,
227+
number_of_trades_open=0,
228+
number_of_orders=0,
229+
number_of_positions=0,
230+
backtest_metrics=self.backtest_metrics_run_one
231+
)
232+
233+
# ohlcv_csv_path = os
234+
ohlcv_df = pl.read_csv(self.ohlcv_csv_path).to_pandas()
235+
236+
self.permutation_test_metrics_one = BacktestPermutationTest(
237+
real_metrics=self.backtest_metrics_run_one,
238+
permutated_metrics=[self.backtest_metrics_run_one, self.backtest_metrics_run_one],
239+
p_values={
240+
"cagr": 0.05,
241+
"sharpe_ratio": 0.05,
242+
"sortino_ratio": 0.05,
243+
"calmar_ratio": 0.05,
244+
"profit_factor": 0.05,
245+
"annual_volatility": 0.05,
246+
"max_drawdown": 0.05,
247+
"win_rate": 0.05,
248+
"win_loss_ratio": 0.05,
249+
"average_monthly_return": 0.05
250+
},
251+
ohlcv_original_datasets={
252+
"BTC/EUR": ohlcv_df
253+
},
254+
ohlcv_permutated_datasets={
255+
"BTC/EUR": [ohlcv_df]
256+
}
257+
)
258+
259+
self.backtest_summary_metrics_one = BacktestSummaryMetrics(
260+
cagr=0.0,
261+
sharpe_ratio=0.0,
262+
sortino_ratio=0.0,
263+
calmar_ratio=0.0,
264+
profit_factor=0.0,
265+
annual_volatility=0.0,
266+
max_drawdown=0.0,
267+
max_drawdown_duration=0,
268+
trades_per_year=0.0,
269+
number_of_trades=0,
270+
win_rate=0.0,
271+
win_loss_ratio=0.0,
272+
)
273+
274+
50275
def tearDown(self):
51276
self.temp_dir.cleanup()
52277

@@ -316,6 +541,9 @@ def test_save_and_open(self):
316541
len(loaded_backtest.get_all_backtest_permutation_tests()),
317542
1
318543
)
544+
self.assertEqual(
545+
1, len(loaded_backtest.backtest_permutation_tests)
546+
)
319547

320548
first_backtest_run = loaded_backtest.get_all_backtest_runs()[0]
321549
self.assertEqual(
@@ -774,7 +1002,6 @@ def test_open_with_backtest_date_ranges(self):
7741002
win_loss_ratio=0.0,
7751003
)
7761004

777-
7781005
# Create a Backtest instance
7791006
backtest = Backtest(
7801007
backtest_runs=[backtest_run, backtest_run_two],
@@ -800,7 +1027,7 @@ def test_open_with_backtest_date_ranges(self):
8001027
# Check that there is a permutation tests directory
8011028
self.assertTrue((self.dir_path / "permutation_tests").exists())
8021029

803-
# Check if there are 2 permutated_metrics folders
1030+
# Check if there are 1 permutated_metrics folders
8041031
permutated_metrics_dir = (self.dir_path / "permutation_tests")
8051032

8061033
self.assertEqual(1, (len(os.listdir(permutated_metrics_dir))))
@@ -1341,3 +1568,36 @@ def test_backtest_hash(self):
13411568
self.assertTrue(
13421569
backtest_two in backtest_dict
13431570
)
1571+
1572+
def test_add_permutation_metrics_after_backtest_has_been_save(self):
1573+
backtest = Backtest(
1574+
backtest_runs=[self.backtest_run_one],
1575+
backtest_permutation_tests=[],
1576+
backtest_summary=self.backtest_summary_metrics_one,
1577+
metadata={"strategy": "test_strategy"},
1578+
risk_free_rate=0.02
1579+
)
1580+
1581+
# Save the backtest
1582+
backtest.save(self.dir_path)
1583+
1584+
loaded_backtest = Backtest.open(self.dir_path)
1585+
1586+
self.assertEqual(
1587+
len(loaded_backtest.get_all_backtest_permutation_tests()), 0
1588+
)
1589+
1590+
# Add a permutation test
1591+
loaded_backtest.add_permutation_test(
1592+
self.permutation_test_metrics_one
1593+
)
1594+
1595+
# Save again
1596+
loaded_backtest.save(self.dir_path)
1597+
1598+
# Load again
1599+
reloaded_backtest = Backtest.open(self.dir_path)
1600+
self.assertEqual(
1601+
len(reloaded_backtest.get_all_backtest_permutation_tests()),
1602+
1
1603+
)

0 commit comments

Comments
 (0)