@@ -219,7 +219,6 @@ def prepare_backtest_data(
219219 )
220220
221221 if data is None :
222-
223222 # Disable pandas if it is set to True, because logic
224223 # depends on polars DataFrame
225224 has_pandas_flag = self .pandas
@@ -806,20 +805,8 @@ def _get_data_from_storage(
806805 """
807806 Helper function to retrieve the data from the storage path if
808807 it exists. If the data does not exist, it returns None.
809-
810- Args:
811- storage_path (str): The path to the storage.
812- symbol (str): The symbol for which to retrieve the data.
813- market (str): The market for which to retrieve the data.
814- time_frame (TimeFrame): The time frame for which to retrieve the
815- start_date (datetime): The start date for the data.
816- end_date (datetime): The end date for the data.
817-
818- Returns:
819- Union[pl.DataFrame, None]: The data from the storage path as a
820- Polars DataFrame, or None if the data does not exist.
821808 """
822-
809+ data = None
823810 if storage_path is None :
824811 return None
825812
@@ -846,36 +833,72 @@ def _get_data_from_storage(
846833 if data_source_spec .symbol .upper () == symbol .upper () and \
847834 data_source_spec .market .upper () == market .upper () and \
848835 data_source_spec .time_frame .equals (time_frame ):
836+
849837 # Check if the data source specification matches
850838 # the start and end date if its specified
851- if data_source_spec .start_date is not None and \
852- data_source_spec .end_date is not None and \
839+ if ( data_source_spec .start_date is not None and
840+ data_source_spec .end_date is not None and
853841 (data_source_spec .start_date <= start_date
854- and data_source_spec .end_date >= end_date ):
842+ and data_source_spec .end_date >= end_date )):
843+
855844 # If the data source specification matches,
856845 # read the file
857846 file_path = os .path .join (storage_path , file_name )
858847 self .data_file_path = file_path
859- data = pl .read_csv (
860- file_path ,
861- schema_overrides = {"Datetime" : pl .Datetime },
862- low_memory = True
863- ).with_columns (
864- pl .col ("Datetime" ).cast (
865- pl .Datetime (
866- time_unit = "ms" , time_zone = "UTC"
848+
849+ # Read CSV as-is first
850+ data = pl .read_csv (file_path , low_memory = True )
851+
852+ # Check what columns we have
853+ if "Datetime" in data .columns :
854+ # Try to parse the datetime column
855+ try :
856+ # Try the ISO format with timezone first
857+ data = data .with_columns (
858+ pl .col ("Datetime" ).str .to_datetime (
859+ format = "%Y-%m-%dT%H:%M:%S%.f%z" ,
860+ time_zone = "UTC"
861+ )
867862 )
863+ except Exception as e1 :
864+ try :
865+ # Fallback: let Polars infer the format
866+ data = data .with_columns (
867+ pl .col ("Datetime" ).str .to_datetime (
868+ time_zone = "UTC"
869+ )
870+ )
871+ except Exception as e2 :
872+ logger .warning (
873+ f"Could not parse Datetime "
874+ f"column in { file_name } : "
875+ f"Format error: { str (e1 )} , "
876+ f"Infer error: { str (e2 )} "
877+ )
878+ continue
879+ else :
880+ logger .warning (
881+ f"No 'Datetime' column "
882+ f"found in { file_name } . "
883+ f"Available columns: { data .columns } "
868884 )
869- )
885+ continue
886+
887+ # Filter by date range
870888 data = data .filter (
871889 (pl .col ("Datetime" ) >= start_date ) &
872890 (pl .col ("Datetime" ) <= end_date )
873891 )
874- return data
892+ break
893+
875894 except Exception as e :
876- logger .warning (e )
895+ logger .warning (
896+ f"Error reading data from { file_name } : { str (e )} "
897+ )
877898 continue
878899
900+ return data
901+
879902 def _get_data_source_specification_from_file_name (
880903 self , file_name : str
881904 ) -> Union [DataSource , None ]:
0 commit comments