Merge remote-tracking branch 'rael/wecc' into wind_to_solar_ratio

Author: staadecker

README.md

@@ -18,4 +18,6 @@ SWITCH. This will build HTML documentation files from python doc strings which
 will include descriptions of each module, their intentions, model
 components they define, and what input files they expect.
 
-To learn about **numerical solvers** read [`docs/Numerical Solvers.md`](/docs/Numerical%20Solvers.md)
+To learn about **numerical solvers** read [`docs/Numerical Solvers.md`](/docs/Numerical%20Solvers.md)
+
+To learn about **numerical issues** read [`docs/Numerical Issues.md`](/docs/Numerical%20Issues.md)

REAM Model Changelog.md

@@ -0,0 +1,18 @@
+# REAM Model Breaking Changes
+
+This file specifies the breaking changes that we have done to the model.
+A breaking change is any change that would change the results of previously
+completed runs.
+
+## List of breaking changes to model
+
+Changes are listed from oldest (first line) to newest (last line of table).
+
+| PR | Month | Change description |
+| ---- | --- | ---------------------|
+| #12 | March 2021 | Use the middle of the period instead of the start as the cutoff for retirement and predetermined buildout. |
+| #15 | March 2021 | Fix bug where storage costs where being over-counted. |
+| #36 | May 2021 | Correct inputs to only list transmission lines in one direction. |
+| #56 | June 2021 | Convert 2020 predetermined build years to 2019 in `get_inputs.py` to avoid conflicts with 2020 period. |
+| #57 | June 2021 | Specify predetermined storage energy capacity in inputs (previously left unspecified). |
+| #68 | June 2021 | Change financial params to 2018 dollars & 5% interest rate. Start using terrain multipliers (which now include the economic multiplier). |

database/2021-06-15_add_generation_plants_groups.sql

@@ -0,0 +1,63 @@
+/*
+####################
+Add generation plants groups
+
+Date applied:
+Description:
+This script adds the option to specify generation plant groups.
+The generation groups are specified in the table generation_plant_group.
+Plants are assigned to a group by adding them to the many-to-many table generation_plant_group_member.
+Groups are assigned to a generation_plant_scenario_id by specifying them in generation_plant_scenario_group_member
+#################
+*/
+
+CREATE TABLE switch.generation_plant_group
+(
+    generation_plant_group_id serial                NOT NULL,
+    description               text                  NOT NULL,
+    name                      character varying(30) NOT NULL,
+    PRIMARY KEY (generation_plant_group_id)
+);
+
+COMMENT ON TABLE switch.generation_plant_group
+    IS 'This table specifies all the generation plant groups. Every group has a set of generation plants (see generation_plant_group_member). Groups can be assigned to a generation_plant_scenario (see generation_plant_scenario_group_member).';
+
+CREATE TABLE switch.generation_plant_group_member
+(
+    generation_plant_group_id integer,
+    generation_plant_id       integer,
+    PRIMARY KEY (generation_plant_group_id, generation_plant_id)
+);
+
+ALTER TABLE switch.generation_plant_group_member
+    ADD CONSTRAINT generation_plant_group_member_group_id_fkey
+        FOREIGN KEY (generation_plant_group_id)
+            REFERENCES switch.generation_plant_group (generation_plant_group_id);
+
+ALTER TABLE switch.generation_plant_group_member
+    ADD CONSTRAINT generation_plant_group_member_generation_plant_id_fkey
+        FOREIGN KEY (generation_plant_id)
+            REFERENCES switch.generation_plant (generation_plant_id);
+
+COMMENT ON TABLE switch.generation_plant_group_member
+    IS 'This table is a many-to-many table that specifies the generation plants that are associated with a generation group.';
+
+CREATE TABLE switch.generation_plant_scenario_group_member
+(
+    generation_plant_scenario_id integer,
+    generation_plant_group_id    integer,
+    PRIMARY KEY (generation_plant_scenario_id, generation_plant_group_id)
+);
+
+ALTER TABLE switch.generation_plant_scenario_group_member
+    ADD CONSTRAINT generation_plant_scenario_group_member_scenario_id_fkey
+        FOREIGN KEY (generation_plant_scenario_id)
+            REFERENCES switch.generation_plant_scenario (generation_plant_scenario_id);
+
+ALTER TABLE switch.generation_plant_scenario_group_member
+    ADD CONSTRAINT generation_plant_scenario_group_member_group_id_fkey
+        FOREIGN KEY (generation_plant_group_id)
+            REFERENCES switch.generation_plant_group (generation_plant_group_id);
+
+COMMENT ON TABLE switch.generation_plant_scenario_group_member
+    IS 'This table is a many-to-many table that specifies which generation plant groups belong to which generation plant scenarios';

database/2021-06-18_add_gen_store_energy_to_power_ratio.sql

@@ -0,0 +1,13 @@
+/*
+####################
+Add column gen_store_energy_to_power_ratio
+
+Date applied: 2021-06-18
+Description:
+This script adds a column to the generation_plant
+table called gen_storage_energy_to_power_ratio specifying
+the storage duration
+#################
+*/
+
+ALTER TABLE switch.generation_plant ADD COLUMN gen_storage_energy_to_power_ratio real;

database/2021-06-24_add_transmission_options.sql

@@ -0,0 +1,19 @@
+/*
+####################
+Add transmission options
+
+Date applied: 2021-06-23
+Description:
+Adds two rows to table transmission_base_capital_cost_scenario_id
+1. A scenario where transmission costs are zero.
+2. A scenario where transmission costs are infinity (building not allowed).
+#################
+*/
+
+INSERT INTO switch.transmission_base_capital_cost (transmission_base_capital_cost_scenario_id,
+                                                   trans_capital_cost_per_mw_km, description)
+VALUES (3, 'Infinity', 'For scenarios where building transmission is forbidden.');
+
+INSERT INTO switch.transmission_base_capital_cost (transmission_base_capital_cost_scenario_id,
+                                                   trans_capital_cost_per_mw_km, description)
+VALUES (4, 0, 'For scenarios where transmission is unlimited.');

database/2021-06-29_add_transmission_options_v2.sql

@@ -0,0 +1,13 @@
+/*
+####################
+Add transmission options
+
+Date applied: 2021-06-29
+Description:
+Adds an extra scenario to the database for a 10x increase in transmission costs.
+#################
+*/
+
+INSERT INTO switch.transmission_base_capital_cost (transmission_base_capital_cost_scenario_id,
+                                                   trans_capital_cost_per_mw_km, description)
+VALUES (5, 9600, '10x the costs of scenario #2. Approximates the no TX case.');

docs/Contribute.md

@@ -14,7 +14,9 @@ procedure.
 3. Once your changes are final and ready to be added to the switch main
 branch, create a pull request on Github.
 
-4. Get someone to review and then merge your changes on Github.
+4. If your change is a breaking change add it to `REAM Model Changelog.md`.
+   
+5. Get someone to review and then merge your changes on Github.
 
 For more information read [this excellent guide](https://guides.github.com/introduction/flow/) (5 min read).
 

docs/Database.md

@@ -21,7 +21,7 @@ their strong points. DBVisualizer can also create a graph of all the relationshi
 tables.
 
 Further, it is often useful to read the comments on tables (PGAdmin: right-click table -> Properties)
-as they sometimes give details on the tables role. Finally, if the table is used in [`get_inputs.py`](/switch_model/wecc/get_inputs.py)
+as they sometimes give details on the tables role. Finally, if the table is used in [`get_inputs.py`](/switch_model/wecc/get_inputs/get_inputs.py)
 one can discover what it does by looking at how get_inputs.py uses the table to generate the SWITCH inputs.
 
 ## Connecting to the database

docs/Graphs.md

@@ -18,92 +18,128 @@ and `ca_policies` have already been solved).
 
 ## Adding new graphs
 
-Graphs can be defined in any module by adding the following function to the file.
-
+New graphs can be added with the `@graph(...)` annotation.
 ```python
-def graph(tools):
+from switch_model.tools.graph import graph
+
+@graph(
+  name="my_custom_graph",
+  title="An example plot",
+  note="Some optional note to add below the graph",
+  # Other options are possible see code documentation
+)
+def my_graphing_function(tools):
   # Your graphing code
   ...
 ```
 
-In `graph()` you can use the `tools` object to create graphs. Here are some important methods.
-
-- `tools.get_dataframe(csv=filename)` will return a pandas dataframe for the file called `filename`. You can also
-  specify `folder=tools.folders.INPUTS` to load a csv from the inputs directory.
-
-- `tools.get_new_axes(out, title, note)` will return a matplotlib axes. This should be the axes used while
-  graphing. `out` is the name of the `.png` file that will be created with this graph. `title` and `note` are optional
-  and will be the title and footnote for the graph.
-
-- `tools.pd`, `tools.sns`, `tools.np`, `tools.mplt` are references to the pandas, seaborn, numpy and matplotlib
-  libraries. This is useful if your graphing code needs to access these libraries since it doesn't require adding an
+In `my_graphing_function()` you can use the `tools` object to create graphs. Here are some important methods.
+
+- `tools.get_dataframe(filename)` will return a pandas dataframe for the file called `filename`. You can also
+  specify `from_inputs=True` to load a csv from the inputs directory.
+  
+- `tools.get_axes()` or `tools.get_figure()` will return a matplotlib axes or figure
+  that should be used while graphing. When possible, always use `get_axes` instead of `get_figure` since
+  this allows plots from different scenarios to share the same figure.
+  
+- `tools.save_figure(fig)`. Some libraries (e.g. plotnine) 
+  always generate their own figures. In this case we can add the figure
+  to our outputs with this function. When possible, use `tools.get_axes()` instead.
+
+- `tools.pd`, `tools.sns`, `tools.np`, `tools.mplt`, `tools.pn` are references to the pandas, seaborn, numpy, matplotlib
+  and plotnine graphing libraries. This is useful if your graphing code needs to access these libraries since it doesn't require adding an
   import to your file.
 
-- `tools.add_gen_type_column(df)` adds a column called `gen_type` to a dataframe with columns
-  `gen_tech` and `gen_energy_source`. `gen_type` is a user-friendly name for the technology (e.g. Nuclear instead of
-  Uranium). The mapping of `gen_energy_source` and `gen_tech` to `gen_type` is defined in
-  a `inputs/graph_tech_types.csv`. If this file isn't present, a default mapping will be used. You can also use other
-  mappings found in `graph_tech_types.csv` by specifying `map_name=` when calling `add_gen_type_column()`.
-
+- `tools.transform` is a reference to a `TransformTools` object that provides
+  useful helper methods for modyfing a dataframe for graphing. Full documentation
+  can be found in the `TransformTools` class but some examples include.
+  
+  - `tools.transform.build_year(df)` which will convert build years that aren't
+  a period to the string `Pre-existing`.
+    
+  - `tools.transform.gen_type(df)` which adds a column called `gen_type` to the dataframe. 
+    `gen_type` is a user-friendly name for the technology (e.g. Nuclear instead of
+  Uranium) and is determined using the mappings in `inputs/graph_tech_types.csv`.
+    
+  - `tools.transform.timestamp(df)`: which adds columns such as the hour, the timestamp in datetime format
+  in the correct timezone, etc.
+    
+  - `tools.transform.load_zone(df)`: Adds a column called 'region' to the dataframe which
+  normally corresponds to the load zone state.
+    
 - `tools.get_colors()` returns a mapping of `gen_type` to its color. This is useful for graphing and can normally be
-  passed straight to `color=` in standard plotting libraries. You can also specify a different color mapping using a
-  similar process to above (`map_name=`)
+  passed straight to `color=` in standard plotting libraries. The color mapping is based on `inputs/graph_tech_colors.csv`.
 
 ## Adding a comparison graph
 
-By default, `tools.get_dataframe` will return the data for only one scenario (the one you are graphing).
-
-Sometimes, you may wish to create a graph that compares multiple scenarios. To do this create a function
-called `compare`.
+Sometimes you may want to create graphs that compare data from multiple scenarios.
+To do this, add `supports_multi_scenario=True` inside the `@graph()` decorator.
 
 ```python
-def compare(tools):
-  # Your graphing code
-  ...
+from switch_model.tools.graph import graph
+
+@graph(
+  name="my_custom_comparison_graph",
+  title="My Comparison plot",
+  supports_multi_scenario=True,
+  # Instead of supports_multi_scenario, you can use
+  # requires_multi_scenario if you want the graphing function
+  # to *only* be run when we have multiple scenarios.
+  # requires_multi_scenario=True,
+)
+def my_graphing_comparison_function(tools):
+    # Read data from all the scenarios
+    df = tools.get_dataframe("some_file.csv")
+    # Plot data
+    ...
 ```
 
-If you call `tools.get_dataframe(...)` from within `compare`, then
-`tools.get_dataframe` will return a dataframe containing the data from *all*
-the scenarios. The dataframe will contain a column called `scenario` to indicate which rows correspond to which
-scenarios. You can then use this column to create a graph comparing the different scenarios (still
-using `tools.get_new_axes`).
+Now everytime you call `tools.get_dataframe(filename)`, data for *all* the scenarios
+gets returned. The way this works is that the 
+returned dataframe will contain a column called `scenario_name` 
+to indicate which rows correspond to which scenarios. 
+You can then use this column to create a graph comparing the different scenarios (still
+using `tools.get_axes`).
 
-At this point, when you run `switch compare`, your `compare(tools)` function will be called and your comparison graph
+At this point, when you run `switch compare`, your `my_graphing_comparison_function` function will be called and your comparison graph
 will be generated.
 
 ## Example
 
 In this example we create a graph that shows the power capacity during each period broken down by technology.
 
 ```python
+from switch_model.tools.graph import graph
+
+@graph(
+  "capacity_per_period",
+  title="Capacity per period"
+)
 def graph(tools):
-  # Get a dataframe of gen_cap.csv
-  gen_cap = tools.get_dataframe(csv="gen_cap")
-
-  # Add a 'gen_type' column to your dataframe
-  gen_cap = tools.add_gen_type_column(gen_cap)
-
-  # Aggregate the generation capacity by gen_type and PERIOD
-  capacity_df = gen_cap.pivot_table(
-    index='PERIOD',
-    columns='gen_type',
-    values='GenCapacity',
-    aggfunc=tools.np.sum,
-    fill_value=0  # Missing values become 0
-  )
-
-  # Get a new pair of axis to plot onto
-  ax = tools.get_new_axes(out="capacity_per_period")
-
-  # Plot
-  capacity_df.plot(
-    kind='bar',
-    ax=ax, # Notice we pass in the axis
-    stacked=True,
-    ylabel="Capacity Online (MW)",
-    xlabel="Period",
-    color=tools.get_colors(len(capacity_df.index))
-  )
+    # Get a dataframe of gen_cap.csv
+    df = tools.get_dataframe("gen_cap.csv")
+
+    # Add a 'gen_type' column to your dataframe
+    df = tools.transform.gen_type(df)
+
+    # Aggregate the generation capacity by gen_type and PERIOD
+    df = df.pivot_table(
+        index='PERIOD',
+        columns='gen_type',
+        values='GenCapacity',
+        aggfunc=tools.np.sum,
+        fill_value=0  # Missing values become 0
+    )
+
+    # Plot
+    df.plot(
+        kind='bar',
+        ax=tools.get_axes(),
+        stacked=True,
+        ylabel="Capacity Online (MW)",
+        xlabel="Period",
+        color=tools.get_colors(len(df.index))
+    )
 ```
 
 Running `switch graph` would run the `graph()` function above and create 
@@ -112,3 +148,37 @@ Running `switch graph` would run the `graph()` function above and create
 Running `switch compare` would create `capacity_per_period.png` containing
 your plot side-by-side with the same plot but for the scenario you're comparing to.
 
+### Testing your graphs
+
+To test your graphs, you can run `switch graph` or `switch compare`. However,
+this takes quite some time. If you want to test just one graphing function
+you can run `switch graph/compare -f FIGURE`. This will run only the graphing function
+you've defined. Here `FIGURE` should be the name of the graph (the first
+argument in `@graph()`, so `capacity_per_period` in the example above).
+
+### Creating graphs outside of SWITCH
+
+Sometimes you may want to create graphs but don't want to permently add
+them to the switch code. To do this create the following Python file anywhere
+on your computer.
+
+```python
+from switch_model.tools.graph import graph
+from switch_model.tools.graph.cli_graph import main as graph
+
+@graph(
+  ...
+)
+def my_first_graph(tools):
+    ...
+
+@graph(
+  ...
+)
+def my_second_graph(tools):
+  ...
+
+if __name__=="__main__":
+    graph(["--ignore-modules-txt"])
+```
+