Improve error message when loading data fails

Author: staadecker

switch_model/generators/core/dispatch.py

@@ -729,16 +729,24 @@ def graph_total_dispatch(tools):
 
     tools.bar_label()
 
+
 @graph(
     "energy_balance",
     title="Energy Balance For Every Month",
     supports_multi_scenario=True,
-    is_long=True
+    is_long=True,
 )
 def energy_balance(tools):
     # Get dispatch dataframe
-    cols = ["timestamp", "gen_tech", "gen_energy_source", "DispatchGen_MW", "scenario_name", "scenario_index",
-            "Curtailment_MW"]
+    cols = [
+        "timestamp",
+        "gen_tech",
+        "gen_energy_source",
+        "DispatchGen_MW",
+        "scenario_name",
+        "scenario_index",
+        "Curtailment_MW",
+    ]
     df = tools.get_dataframe("dispatch.csv", drop_scenario_info=False)[cols]
     df = tools.transform.gen_type(df)
 
@@ -749,11 +757,16 @@ def energy_balance(tools):
     # Sum dispatch across all the projects of the same type and timepoint
     key_columns = ["timestamp", "gen_type", "scenario_name", "scenario_index"]
     df = df.groupby(key_columns, as_index=False).sum()
-    df = df.melt(id_vars=key_columns, value_vars=["Dispatch", "Dispatch Limit"], var_name="Type")
+    df = df.melt(
+        id_vars=key_columns, value_vars=["Dispatch", "Dispatch Limit"], var_name="Type"
+    )
     df = df.rename({"gen_type": "Source"}, axis=1)
 
-    discharge = df[(df["Source"] == "Storage") & (df["Type"] == "Dispatch")].drop(["Source", "Type"], axis=1).rename(
-        {"value": "discharge"}, axis=1)
+    discharge = (
+        df[(df["Source"] == "Storage") & (df["Type"] == "Dispatch")]
+        .drop(["Source", "Type"], axis=1)
+        .rename({"value": "discharge"}, axis=1)
+    )
 
     # Get load dataframe
     load = tools.get_dataframe("load_balance.csv", drop_scenario_info=False)
@@ -764,23 +777,21 @@ def energy_balance(tools):
     load = load.groupby(key_columns, as_index=False).sum()
 
     # Subtract storage dispatch from generation and add it to the storage charge to get net flow
-    load = load.merge(
-        discharge,
-        how="left",
-        on=key_columns,
-        validate="one_to_one"
-    )
+    load = load.merge(discharge, how="left", on=key_columns, validate="one_to_one")
     load["ZoneTotalCentralDispatch"] -= load["discharge"]
     load["StorageNetCharge"] += load["discharge"]
     load = load.drop("discharge", axis=1)
 
     # Rename and convert from wide to long format
-    load = load.rename({
-        "ZoneTotalCentralDispatch": "Total Generation (excl. storage discharge)",
-        "TXPowerNet": "Transmission Losses",
-        "StorageNetCharge": "Storage Net Flow",
-        "zone_demand_mw": "Demand",
-    }, axis=1).sort_index(axis=1)
+    load = load.rename(
+        {
+            "ZoneTotalCentralDispatch": "Total Generation (excl. storage discharge)",
+            "TXPowerNet": "Transmission Losses",
+            "StorageNetCharge": "Storage Net Flow",
+            "zone_demand_mw": "Demand",
+        },
+        axis=1,
+    ).sort_index(axis=1)
     load = load.melt(id_vars=key_columns, var_name="Source")
     load["Type"] = "Dispatch"
 
@@ -796,26 +807,34 @@ def energy_balance(tools):
     FREQUENCY = "1W"
 
     def groupby_time(df):
-        return df.groupby([
-            "scenario_name",
-            "period",
-            "Source",
-            "Type",
-            tools.pd.Grouper(key="datetime", freq=FREQUENCY, origin="start")
-        ])["value"]
+        return df.groupby(
+            [
+                "scenario_name",
+                "period",
+                "Source",
+                "Type",
+                tools.pd.Grouper(key="datetime", freq=FREQUENCY, origin="start"),
+            ]
+        )["value"]
 
     df = groupby_time(df).sum().reset_index()
 
     # Get the state of charge data
-    soc = tools.get_dataframe("StateOfCharge.csv", dtype={"STORAGE_GEN_TPS_1": str}, drop_scenario_info=False)
-    soc = soc.rename({"STORAGE_GEN_TPS_2": "timepoint", "StateOfCharge": "value"}, axis=1)
+    soc = tools.get_dataframe(
+        "StateOfCharge.csv", dtype={"STORAGE_GEN_TPS_1": str}, drop_scenario_info=False
+    )
+    soc = soc.rename(
+        {"STORAGE_GEN_TPS_2": "timepoint", "StateOfCharge": "value"}, axis=1
+    )
     # Sum over all the projects that are in the same scenario with the same timepoint
     soc = soc.groupby(["timepoint", "scenario_name"], as_index=False).sum()
     soc["Source"] = "State Of Charge"
     soc["value"] /= 1e6  # Convert to TWh
 
     # Group by time
-    soc = tools.transform.timestamp(soc, use_timepoint=True, key_col="timepoint").astype({"period": str})
+    soc = tools.transform.timestamp(
+        soc, use_timepoint=True, key_col="timepoint"
+    ).astype({"period": str})
     soc["Type"] = "Dispatch"
     soc = groupby_time(soc).mean().reset_index()
 
@@ -827,35 +846,47 @@ def groupby_time(df):
     # Plot
     # Get the colors for the lines
     colors = tools.get_colors()
-    colors.update({
-        "Transmission Losses": "brown",
-        "Storage Net Flow": "cadetblue",
-        "Demand": "black",
-        "Total Generation (excl. storage discharge)": "black",
-        "State Of Charge": "green"
-    })
+    colors.update(
+        {
+            "Transmission Losses": "brown",
+            "Storage Net Flow": "cadetblue",
+            "Demand": "black",
+            "Total Generation (excl. storage discharge)": "black",
+            "State Of Charge": "green",
+        }
+    )
 
     # plot
     num_periods = df["period"].nunique()
     pn = tools.pn
-    plot = pn.ggplot(df) + \
-           pn.geom_line(pn.aes(x="day", y="value", color="Source", linetype="Type")) + \
-           pn.facet_grid("period ~ scenario_name") + \
-           pn.labs(y="Contribution to Energy Balance (TWh)") + \
-           pn.scales.scale_color_manual(values=colors, aesthetics="color", na_value=colors["Other"]) + \
-           pn.scales.scale_x_continuous(
-               name="Month",
-               labels=["J", "F", "M", "A", "M", "J", "J", "A", "S", "O", "N", "D"],
-               breaks=(15, 46, 76, 106, 137, 167, 198, 228, 259, 289, 319, 350),
-               limits=(0, 366)) + \
-           pn.scales.scale_linetype_manual(
-               values={"Dispatch Limit": "dotted", "Dispatch": "solid"}
-           ) + \
-           pn.theme(
-               figure_size=(pn.options.figure_size[0] * tools.num_scenarios, pn.options.figure_size[1] * num_periods))
+    plot = (
+        pn.ggplot(df)
+        + pn.geom_line(pn.aes(x="day", y="value", color="Source", linetype="Type"))
+        + pn.facet_grid("period ~ scenario_name")
+        + pn.labs(y="Contribution to Energy Balance (TWh)")
+        + pn.scales.scale_color_manual(
+            values=colors, aesthetics="color", na_value=colors["Other"]
+        )
+        + pn.scales.scale_x_continuous(
+            name="Month",
+            labels=["J", "F", "M", "A", "M", "J", "J", "A", "S", "O", "N", "D"],
+            breaks=(15, 46, 76, 106, 137, 167, 198, 228, 259, 289, 319, 350),
+            limits=(0, 366),
+        )
+        + pn.scales.scale_linetype_manual(
+            values={"Dispatch Limit": "dotted", "Dispatch": "solid"}
+        )
+        + pn.theme(
+            figure_size=(
+                pn.options.figure_size[0] * tools.num_scenarios,
+                pn.options.figure_size[1] * num_periods,
+            )
+        )
+    )
 
     tools.save_figure(plot.draw())
 
+
 @graph(
     "curtailment_per_period",
     title="Percent of total dispatchable capacity curtailed",

switch_model/utilities/load_data.py

@@ -99,16 +99,16 @@ def __str__(self):
         return repr(self.value)
 
 
-def load_data(switch_data, optional, auto_select, optional_params, **kwds):
-    path = kwds["filename"]
+def load_data(switch_data, optional, auto_select, optional_params, **kwargs):
+    path = kwargs["filename"]
     # Skip if the file is missing
     if optional and not os.path.isfile(path):
         return
     # If this is a .dat file, then skip the rest of this fancy business; we'll
     # only check if the file is missing and optional for .csv files.
     filename, extension = os.path.splitext(path)
     if extension == ".dat":
-        switch_data.load(**kwds)
+        switch_data.load(**kwargs)
         return
 
     # copy the optional_params to avoid side-effects when the list is altered below
@@ -134,14 +134,14 @@ def load_data(switch_data, optional, auto_select, optional_params, **kwds):
     # Try to get a list of parameters. If param was given as a
     # singleton or a tuple, make it into a list that can be edited.
     params = []
-    if "param" in kwds:
+    if "param" in kwargs:
         # Tuple -> list
-        if isinstance(kwds["param"], tuple):
-            kwds["param"] = list(kwds["param"])
+        if isinstance(kwargs["param"], tuple):
+            kwargs["param"] = list(kwargs["param"])
         # Singleton -> list
-        elif not isinstance(kwds["param"], list):
-            kwds["param"] = [kwds["param"]]
-        params = kwds["param"]
+        elif not isinstance(kwargs["param"], list):
+            kwargs["param"] = [kwargs["param"]]
+        params = kwargs["param"]
     # optional_params may include Param objects instead of names. In
     # those cases, convert objects to names.
     for (i, p) in enumerate(optional_params):
@@ -157,11 +157,11 @@ def load_data(switch_data, optional, auto_select, optional_params, **kwds):
             optional_params.append(p.name)
     # How many index columns do we expect?
     # Grab the dimensionality of the index param if it was provided.
-    if "index" in kwds:
-        num_indexes = kwds["index"].dimen
+    if "index" in kwargs:
+        num_indexes = kwargs["index"].dimen
         if num_indexes == UnknownSetDimen:
             raise Exception(
-                f"Index {kwds['index'].name} has unknown dimension. Specify dimen= during its creation."
+                f"Index {kwargs['index'].name} has unknown dimension. Specify dimen= during its creation."
             )
     # Next try the first parameter's index.
     elif len(params) > 0:
@@ -185,7 +185,7 @@ def load_data(switch_data, optional, auto_select, optional_params, **kwds):
     # within the file (e.g., "cost" and "limit"). We could also require the data file
     # to be called "rfm_supply_tier.csv" for greater consistency/predictability.
     if auto_select:
-        if "select" in kwds:
+        if "select" in kwargs:
             raise InputError(
                 "You may not specify a select parameter if "
                 + "auto_select is set to True."
@@ -197,15 +197,15 @@ def get_column_name(p):
             else:
                 return p.name
 
-        kwds["select"] = headers[0:num_indexes] + [get_column_name(p) for p in params]
+        kwargs["select"] = headers[0:num_indexes] + [get_column_name(p) for p in params]
     # Check to see if expected column names are in the file. If a column
     # name is missing and its parameter is optional, then drop it from
     # the select & param lists.
-    if "select" in kwds:
-        if isinstance(kwds["select"], tuple):
-            kwds["select"] = list(kwds["select"])
+    if "select" in kwargs:
+        if isinstance(kwargs["select"], tuple):
+            kwargs["select"] = list(kwargs["select"])
         del_items = []
-        for (i, col) in enumerate(kwds["select"]):
+        for (i, col) in enumerate(kwargs["select"]):
             p_i = i - num_indexes
             if col not in headers:
                 if len(params) > p_i >= 0 and params[p_i].name in optional_params:
@@ -218,17 +218,20 @@ def get_column_name(p):
         # to first so that the indexes won't get messed up as we go.
         del_items.sort(reverse=True)
         for (i, p_i) in del_items:
-            del kwds["select"][i]
-            del kwds["param"][p_i]
+            del kwargs["select"][i]
+            del kwargs["param"][p_i]
 
     if optional and file_has_no_data_rows:
         # Skip the file.  Note that we are only doing this after having
         # validated the file's column headings.
         return
 
     # Use our custom DataManager to allow 'inf' in csvs.
-    if kwds["filename"][-4:] == ".csv":
-        kwds["using"] = "switch_csv"
+    if kwargs["filename"][-4:] == ".csv":
+        kwargs["using"] = "switch_csv"
     # All done with cleaning optional bits. Pass the updated arguments
     # into the DataPortal.load() function.
-    switch_data.load(**kwds)
+    try:
+        switch_data.load(**kwargs)
+    except:
+        raise Exception(f"Failed to load data from file {path}.")