Merge branch 'wecc' into improve_load_aug

# Conflicts:
#	switch_model/utilities/__init__.py

Author: staadecker

.gitignore

@@ -15,6 +15,7 @@
 *.vcf
 *.xml
 *.pickle
+examples/**/temp/**
 examples/**/outputs/*.csv
 examples/**/outputs/*.txt
 examples/**/graphs/*.png

database/2021-07-07_add_wind_solar_ratio_column.sql

@@ -0,0 +1,12 @@
+/*
+####################
+Add wind_to_solar_ratio column
+
+Date applied: 2021-07-07
+Description:
+Adds a column called wind_to_solar_ratio to the database which is used by
+switch_model.policies.wind_to_solar_ratio
+#################
+*/
+
+ALTER TABLE switch.scenario ADD COLUMN wind_to_solar_ratio real;

switch_model/balancing/load_zones.py

@@ -244,10 +244,11 @@ def get_component_per_year(m, z, p, component):
     title="Energy balance duals per period",
     note="Note: Outliers and zero-valued duals are ignored."
 )
-def graph(tools):
+def graph_energy_balance(tools):
     load_balance = tools.get_dataframe('load_balance.csv')
     load_balance = tools.transform.timestamp(load_balance)
-    load_balance["energy_balance_duals"] = tools.pd.to_numeric(load_balance["normalized_energy_balance_duals_dollar_per_mwh"], errors="coerce") / 10
+    load_balance["energy_balance_duals"] = tools.pd.to_numeric(
+        load_balance["normalized_energy_balance_duals_dollar_per_mwh"], errors="coerce") / 10
     load_balance = load_balance[["energy_balance_duals", "time_row"]]
     load_balance = load_balance.pivot(columns="time_row", values="energy_balance_duals")
     # Don't include the zero-valued duals
@@ -259,3 +260,44 @@ def graph(tools):
             ylabel='Energy balance duals (cents/kWh)',
             showfliers=False
         )
+
+
+@graph(
+    "daily_demand",
+    title="Total daily demand",
+    supports_multi_scenario=True
+)
+def demand(tools):
+    df = tools.get_dataframe("loads.csv", from_inputs=True, drop_scenario_info=False)
+    df = df.groupby(["TIMEPOINT", "scenario_name"], as_index=False).sum()
+    df = tools.transform.timestamp(df, key_col="TIMEPOINT", use_timepoint=True)
+    df = df.groupby(["season", "hour", "scenario_name", "time_row"], as_index=False).mean()
+    df["zone_demand_mw"] /= 1e3
+    pn = tools.pn
+
+    plot = pn.ggplot(df) + \
+           pn.geom_line(pn.aes(x="hour", y="zone_demand_mw", color="scenario_name")) + \
+           pn.facet_grid("time_row ~ season") + \
+           pn.labs(x="Hour (PST)", y="Demand (GW)", color="Scenario")
+    tools.save_figure(plot.draw())
+
+
+@graph(
+    "demand",
+    title="Total demand",
+    supports_multi_scenario=True
+)
+def yearly_demand(tools):
+    df = tools.get_dataframe("loads.csv", from_inputs=True, drop_scenario_info=False)
+    df = df.groupby(["TIMEPOINT", "scenario_name"], as_index=False).sum()
+    df = tools.transform.timestamp(df, key_col="TIMEPOINT", use_timepoint=True)
+    df["zone_demand_mw"] *= df["tp_duration"] / 1e3
+    df["day"] = df["datetime"].dt.day_of_year
+    df = df.groupby(["day", "scenario_name", "time_row"], as_index=False)["zone_demand_mw"].sum()
+    pn = tools.pn
+
+    plot = pn.ggplot(df) + \
+           pn.geom_line(pn.aes(x="day", y="zone_demand_mw", color="scenario_name")) + \
+           pn.facet_grid("time_row ~ .") + \
+           pn.labs(x="Day of Year", y="Demand (GW)", color="Scenario")
+    tools.save_figure(plot.draw())

switch_model/generators/core/build.py

@@ -666,14 +666,15 @@ def graph_capacity(tools):
         color=tools.get_colors(len(capacity_df.index)),
     )
 
+    tools.bar_label()
 
 @graph(
     "buildout_gen_per_period",
     title="Built Capacity per Period",
     supports_multi_scenario=True
 )
 def graph_buildout(tools):
-    build_gen = tools.get_dataframe("BuildGen.csv")
+    build_gen = tools.get_dataframe("BuildGen.csv", dtype={"GEN_BLD_YRS_1": str})
     build_gen = build_gen.rename(
         {"GEN_BLD_YRS_1": "GENERATION_PROJECT", "GEN_BLD_YRS_2": "build_year", "BuildGen": "Amount"},
         axis=1

switch_model/generators/core/dispatch.py

@@ -609,7 +609,6 @@ def graph_hourly_curtailment(tools):
 @graph(
     "total_dispatch",
     title="Total dispatched electricity",
-    is_long=True,
 )
 def graph_total_dispatch(tools):
     # ---------------------------------- #
@@ -649,6 +648,134 @@ def graph_total_dispatch(tools):
         ylabel="Total dispatched electricity (TWh)"
     )
 
+    tools.bar_label()
+
+@graph(
+    "energy_balance",
+    title="Energy Balance For Every Month",
+    supports_multi_scenario=True,
+    is_long=True
+)
+def energy_balance(tools):
+    # Get dispatch dataframe
+    cols = ["timestamp", "gen_tech", "gen_energy_source", "DispatchGen_MW", "scenario_name", "scenario_index",
+            "Curtailment_MW"]
+    df = tools.get_dataframe("dispatch.csv", drop_scenario_info=False)[cols]
+    df = tools.transform.gen_type(df)
+
+    # Rename and add needed columns
+    df["Dispatch Limit"] = df["DispatchGen_MW"] + df["Curtailment_MW"]
+    df = df.drop("Curtailment_MW", axis=1)
+    df = df.rename({"DispatchGen_MW": "Dispatch"}, axis=1)
+    # Sum dispatch across all the projects of the same type and timepoint
+    key_columns = ["timestamp", "gen_type", "scenario_name", "scenario_index"]
+    df = df.groupby(key_columns, as_index=False).sum()
+    df = df.melt(id_vars=key_columns, value_vars=["Dispatch", "Dispatch Limit"], var_name="Type")
+    df = df.rename({"gen_type": "Source"}, axis=1)
+
+    discharge = df[(df["Source"] == "Storage") & (df["Type"] == "Dispatch")].drop(["Source", "Type"], axis=1).rename(
+        {"value": "discharge"}, axis=1)
+
+    # Get load dataframe
+    load = tools.get_dataframe("load_balance.csv", drop_scenario_info=False)
+    load = load.drop("normalized_energy_balance_duals_dollar_per_mwh", axis=1)
+
+    # Sum load across all the load zones
+    key_columns = ["timestamp", "scenario_name", "scenario_index"]
+    load = load.groupby(key_columns, as_index=False).sum()
+
+    # Subtract storage dispatch from generation and add it to the storage charge to get net flow
+    load = load.merge(
+        discharge,
+        how="left",
+        on=key_columns,
+        validate="one_to_one"
+    )
+    load["ZoneTotalCentralDispatch"] -= load["discharge"]
+    load["StorageNetCharge"] += load["discharge"]
+    load = load.drop("discharge", axis=1)
+
+    # Rename and convert from wide to long format
+    load = load.rename({
+        "ZoneTotalCentralDispatch": "Total Generation (excl. storage discharge)",
+        "TXPowerNet": "Transmission Losses",
+        "StorageNetCharge": "Storage Net Flow",
+        "zone_demand_mw": "Demand",
+    }, axis=1).sort_index(axis=1)
+    load = load.melt(id_vars=key_columns, var_name="Source")
+    load["Type"] = "Dispatch"
+
+    # Merge dispatch contributions with load contributions
+    df = pd.concat([load, df])
+
+    # Add the timestamp information and make period string to ensure it doesn't mess up the graphing
+    df = tools.transform.timestamp(df).astype({"period": str})
+
+    # Convert to TWh (incl. multiply by timepoint duration)
+    df["value"] *= df["tp_duration"] / 1e6
+
+    FREQUENCY = "1W"
+
+    def groupby_time(df):
+        return df.groupby([
+            "scenario_name",
+            "period",
+            "Source",
+            "Type",
+            tools.pd.Grouper(key="datetime", freq=FREQUENCY, origin="start")
+        ])["value"]
+
+    df = groupby_time(df).sum().reset_index()
+
+    # Get the state of charge data
+    soc = tools.get_dataframe("StateOfCharge.csv", dtype={"STORAGE_GEN_TPS_1": str}, drop_scenario_info=False)
+    soc = soc.rename({"STORAGE_GEN_TPS_2": "timepoint", "StateOfCharge": "value"}, axis=1)
+    # Sum over all the projects that are in the same scenario with the same timepoint
+    soc = soc.groupby(["timepoint", "scenario_name"], as_index=False).sum()
+    soc["Source"] = "State Of Charge"
+    soc["value"] /= 1e6  # Convert to TWh
+
+    # Group by time
+    soc = tools.transform.timestamp(soc, use_timepoint=True, key_col="timepoint").astype({"period": str})
+    soc["Type"] = "Dispatch"
+    soc = groupby_time(soc).mean().reset_index()
+
+    # Add state of charge to dataframe
+    df = pd.concat([df, soc])
+    # Add column for day since that's what we really care about
+    df["day"] = df["datetime"].dt.dayofyear
+
+    # Plot
+    # Get the colors for the lines
+    colors = tools.get_colors()
+    colors.update({
+        "Transmission Losses": "brown",
+        "Storage Net Flow": "cadetblue",
+        "Demand": "black",
+        "Total Generation (excl. storage discharge)": "black",
+        "State Of Charge": "green"
+    })
+
+    # plot
+    num_periods = df["period"].nunique()
+    pn = tools.pn
+    plot = pn.ggplot(df) + \
+           pn.geom_line(pn.aes(x="day", y="value", color="Source", linetype="Type")) + \
+           pn.facet_grid("period ~ scenario_name") + \
+           pn.labs(y="Contribution to Energy Balance (TWh)") + \
+           pn.scales.scale_color_manual(values=colors, aesthetics="color", na_value=colors["Other"]) + \
+           pn.scales.scale_x_continuous(
+               name="Month",
+               labels=["J", "F", "M", "A", "M", "J", "J", "A", "S", "O", "N", "D"],
+               breaks=(15, 46, 76, 106, 137, 167, 198, 228, 259, 289, 319, 350),
+               limits=(0, 366)) + \
+           pn.scales.scale_linetype_manual(
+               values={"Dispatch Limit": "dotted", "Dispatch": "solid"}
+           ) + \
+           pn.theme(
+               figure_size=(pn.options.figure_size[0] * tools.num_scenarios, pn.options.figure_size[1] * num_periods))
+
+    tools.save_figure(plot.draw())
 
 @graph(
     "curtailment_per_period",

switch_model/generators/extensions/storage.py

@@ -429,7 +429,7 @@ def graph_state_of_charge(tools):
     df = tools.get_dataframe("storage_dispatch")
     df = df.groupby(["timepoint", "scenario_name"], as_index=False)["StateOfCharge"].sum()
     # Add datetime information
-    df = tools.transform.timestamp(df, timestamp_col="timepoint")
+    df = tools.transform.timestamp(df, key_col="timepoint")
     # Count num rows
     num_periods = len(df["period"].unique())
 
@@ -445,16 +445,24 @@ def graph_state_of_charge(tools):
         how="left"
     )
 
-    # Convert values to GWh
-    df["StateOfCharge"] /= 1e3
-    df["OnlineEnergyCapacityMWh"] /= 1e3
+    # Convert values to TWh
+    df["StateOfCharge"] /= 1e6
+    df["OnlineEnergyCapacityMWh"] /= 1e6
+
+    # Determine information for the label
+    y_axis_lim = df["OnlineEnergyCapacityMWh"].max()
+    offset = y_axis_lim * 0.05
+    df["label_position"] = df["OnlineEnergyCapacityMWh"] + offset
+    df["label"] = df["OnlineEnergyCapacityMWh"].round(decimals=2)
+    label_x_pos = df["datetime"].median()
 
     # Plot with plotnine
     pn = tools.pn
     plot = pn.ggplot(df, pn.aes(x="datetime", y="StateOfCharge")) \
            + pn.geom_line() \
-           + pn.labs(y="State of Charge (GWh)", x="Time of Year") \
-           + pn.geom_hline(pn.aes(yintercept="OnlineEnergyCapacityMWh"), linetype="dashed", color='blue')
+           + pn.labs(y="State of Charge (TWh)", x="Time of Year") \
+           + pn.geom_hline(pn.aes(yintercept="OnlineEnergyCapacityMWh"), linetype="dashed", color='blue') \
+           + pn.geom_text(pn.aes(label="label", x=label_x_pos, y="label_position"), fontweight="light", size="10")
 
     tools.save_figure(by_scenario_and_period(tools, plot, num_periods).draw())
 
@@ -477,7 +485,7 @@ def graph_state_of_charge_per_duration(tools):
     # Get the total state of charge at each timepoint for each project
     df = tools.get_dataframe("storage_dispatch")[
         ["generation_project", "timepoint", "StateOfCharge", "scenario_name"]]
-    df = tools.transform.timestamp(df, timestamp_col="timepoint")
+    df = tools.transform.timestamp(df, key_col="timepoint")
 
     # Add the capacity information to the state of charge information
     df = df.merge(
@@ -489,7 +497,9 @@ def graph_state_of_charge_per_duration(tools):
     df = df.groupby(["duration", "scenario_name", "datetime", "period"], as_index=False)[
         ["StateOfCharge", "OnlineEnergyCapacityMWh"]].sum()
     # Convert to GWh
-    df["StateOfCharge"] /= 1e3
+    # df["StateOfCharge"] /= 1e3
+    # Convert to percent
+    df["StateOfCharge"] /= df["OnlineEnergyCapacityMWh"]
 
     # Plot with plotnine
     pn = tools.pn
@@ -510,7 +520,7 @@ def graph_dispatch_cycles(tools):
     # Aggregate by timepoint
     df = df.groupby("timepoint", as_index=False).sum()
     # Add datetime column
-    df = tools.transform.timestamp(df, timestamp_col="timepoint")
+    df = tools.transform.timestamp(df, key_col="timepoint")
     # Find charge in GWh
     df["StateOfCharge"] /= 1e3
 
@@ -538,7 +548,12 @@ def graph_dispatch_cycles(tools):
                         title="Storage cycle duration based on fourier transform"
                               " of state of charge")
     ax.semilogx(1 / xfreq, yfreq)
+    # Plot some key cycle lengths
+    ax.axvline(24, linestyle="dotted", label="24 hours", color="red")  # A day
+    ax.axvline(24 * 21, linestyle="dotted", label="3 weeks", color="green")  # 3 weeks
+    ax.axvline(24 * 182.5, linestyle="dotted", label="1/2 Year", color="purple")
     ax.set_xlabel("Hours per cycle")
+    ax.legend()
     ax.grid(True, which="both", axis="x")
 
 
@@ -556,6 +571,7 @@ def graph_buildout(tools):
     df = df[df["IncrementalPowerCapacityMW"] != 0]
     df["duration"] = df["IncrementalEnergyCapacityMWh"] / df["IncrementalPowerCapacityMW"]
     df["power"] = df["IncrementalPowerCapacityMW"] / 1e3
+    df["energy"] = df["IncrementalEnergyCapacityMWh"] / 1e3
     df = tools.transform.build_year(df)
     pn = tools.pn
     num_regions = len(df["region"].unique())
@@ -576,6 +592,14 @@ def graph_buildout(tools):
     tools.save_figure(by_scenario(tools, plot).draw(), "storage_duration_histogram")
     tools.save_figure(by_scenario_and_region(tools, plot, num_regions).draw(), "storage_duration_histogram_by_region")
 
+    plot = pn.ggplot(df, pn.aes(x="duration")) \
+           + pn.geom_histogram(pn.aes(weight="energy"), binwidth=5) \
+           + pn.labs(title="Storage Duration Histogram", x="Duration (h)", y="Energy Capacity (GWh)")
+
+    tools.save_figure(by_scenario(tools, plot).draw(), "storage_duration_histogram_by_energy")
+    tools.save_figure(by_scenario_and_region(tools, plot, num_regions).draw(), "storage_duration_histogram_by_region_and_energy")
+
+
 
 def by_scenario(tools, plot):
     pn = tools.pn