Merge pull request #105 from staadecker/aggregate_projects

Add get_inputs post-process step that aggregates similar plants within the same zone

Author: pesap

switch_model/__main__.py

@@ -33,7 +33,7 @@ def runner():
     "solve-scenarios": get_module_runner("switch_model.solve_scenarios"),
     "test": get_module_runner("switch_model.test"),
     "upgrade": get_module_runner("switch_model.upgrade"),
-    "get_inputs": get_module_runner("switch_model.wecc.get_inputs"),
+    "get_inputs": get_module_runner("switch_model.wecc.get_inputs.cli"),
     "drop": get_module_runner("switch_model.tools.drop"),
     "new": get_module_runner("switch_model.tools.new"),
     "graph": get_module_runner("switch_model.tools.graph.cli_graph"),

switch_model/tools/drop.py

@@ -1,4 +1,6 @@
 import os
+import warnings
+
 import pandas
 from switch_model.utilities import query_yes_no
 from argparse import ArgumentParser, RawTextHelpFormatter
@@ -71,7 +73,8 @@
         ('timepoints.csv', 'timepoint_id'),
         [
             ('loads.csv', 'TIMEPOINT'),
-            ('variable_capacity_factors.csv', 'timepoint')
+            ('variable_capacity_factors.csv', 'timepoint'),
+            ('hydro_timepoints.csv', 'timepoint_id')
         ]
     ),
     "projects": (
@@ -100,6 +103,8 @@ def main(args=None):
 
     parser.add_argument('--run', default=False, action='store_true', help='Drop the data.')
     parser.add_argument('--inputs-dir', default='inputs', help='Directory of the input files. Defaults to "inputs".')
+    parser.add_argument('--silent', default=False, action='store_true', help='Suppress output')
+    parser.add_argument('--no-confirm', default=False, action='store_true', help="Skip confirmation prompts")
     args = parser.parse_args(args)
 
     if not args.run:
@@ -109,7 +114,7 @@ def main(args=None):
     if not os.path.isdir(args.inputs_dir):
         raise NotADirectoryError("{} is not a directory".format(args.inputs_dir))
 
-    should_continue = query_yes_no("WARNING: This will permanently delete data from directory '{}' "
+    should_continue = args.no_confirm or query_yes_no("WARNING: This will permanently delete data from directory '{}' "
                                    "WITHOUT backing it up. Are you sure you want to continue?".format(args.inputs_dir))
 
     if not should_continue:
@@ -123,30 +128,34 @@ def main(args=None):
     warn_about_periods = False
     pass_count = 0
     while pass_count == 0 or rows_removed_in_pass != 0:
-        print("Pass {}...".format(pass_count), flush=True)
+        if not args.silent:
+            print("Pass {}...".format(pass_count), flush=True)
         rows_removed_in_pass = 0
         for name, data_type in data_types.items():
-            print("Checking '{}'...".format(name), flush=True)
+            if not args.silent:
+                print("Checking '{}'...".format(name), flush=True)
             rows_removed = drop_data(data_type, args)
             rows_removed_in_pass += rows_removed
 
             if name == "periods" and rows_removed != 0:
                 warn_about_periods = True
-        print("Removed {} rows during pass.".format(rows_removed_in_pass))
+        if not args.silent:
+            print("Removed {} rows during pass.".format(rows_removed_in_pass))
 
         total_rows_removed += rows_removed_in_pass
         pass_count += 1
 
-    print("\n\nRemove {} rows in total from the input files.".format(total_rows_removed))
-    print("\n\nNote: If SWITCH fails to load the model when solving it is possible that some input files were missed."
-          " If this is the case, please add the missing input files to 'data_types' in 'switch_model/tools/drop.py'.")
+    if not args.silent:
+        print("\n\nRemove {} rows in total from the input files.".format(total_rows_removed))
+        print("\n\nNote: If SWITCH fails to load the model when solving it is possible that some input files were missed."
+              " If this is the case, please add the missing input files to 'data_types' in 'switch_model/tools/drop.py'.")
 
     # It is impossible to know if a row in gen_build_costs.csv is for predetermined generation or for
     # a period that was removed. So instead we don't touch it and let the user manually edit
     # the input file.
     if warn_about_periods:
-        print("\n\nWARNING: Could not update gen_build_costs.csv. Please manually edit gen_build_costs.csv to remove "
-              "references to the removed periods.")
+        warnings.warn("\n\nCould not update gen_build_costs.csv. Please manually edit gen_build_costs.csv to remove "
+                      "references to the removed periods.")
 
 
 def drop_data(id_type, args):
@@ -170,7 +179,7 @@ def get_valid_ids(primary_file, args):
         print("\n Warning: {} was not found.".format(filename))
         return None
 
-    valid_ids = pandas.read_csv(path)[primary_key]
+    valid_ids = pandas.read_csv(path, dtype=str)[primary_key]
     return valid_ids
 
 
@@ -180,17 +189,21 @@ def drop_from_file(filename, foreign_key, valid_ids, args):
     if not os.path.exists(path):
         return 0
 
-    df = pandas.read_csv(path)
+    df = pandas.read_csv(path, dtype=str)
     count = len(df)
+    if foreign_key not in df.columns:
+        raise Exception(f"Column {foreign_key} not in file {filename}")
     df = df[df[foreign_key].isin(valid_ids)]
     rows_removed = count - len(df)
 
     if rows_removed != 0:
         df.to_csv(path, index=False)
 
-        print("Removed {} rows {}.".format(rows_removed, filename))
+        if not args.silent:
+            print("Removed {} rows {}.".format(rows_removed, filename))
         if rows_removed == count:
-            print("WARNING: {} is now empty.".format(filename))
+            if not args.silent:
+                print("WARNING: {} is now empty.".format(filename))
 
     return rows_removed
 

switch_model/tools/graphing.md

@@ -48,4 +48,13 @@ get_inputs:
   # costs_scenario: 0
   # plants_scenario: 0
   # constant_scenario: 0
-  # minimums_scenario: 0
+  # minimums_scenario: 0
+# When the following line is uncommented (regardless of its value) then only California load zones are kept
+# only_california: 0
+# When the following lines are uncommented all the Central_PV and Wind projects within the same load zone gets
+# aggregated into a single project. This helps reduce the model complexity.
+# cf_quantile is the percentile for the capacity factor to use. 1 will use the largest capacity factor
+# of all the available candidate plants, 0.5 will use the median plant and 0 will use the worst plant.
+# aggregate_projects_by_zone:
+#  agg_techs: ["Central_PV"]
+#  cf_method: "file" # Other options are "weighted_mean" and "95_quantile"

switch_model/tools/templates/config.yaml

@@ -1,61 +0,0 @@
-""" Script to retrieve the input data from the switch-wecc database and apply post-processing steps.
-"""
-import argparse
-import os
-
-from switch_model.utilities import query_yes_no, StepTimer
-from switch_model.wecc.get_inputs.get_inputs import query_db
-from switch_model.wecc.get_inputs.register_post_process import run_post_process
-from switch_model.wecc.utilities import load_config
-from switch_model.wecc.get_inputs.post_process_steps import *
-
-
-def main():
-    timer = StepTimer()
-
-    # Create command line tool, just provides help information
-    parser = argparse.ArgumentParser(
-        description="Write SWITCH input files from database tables.",
-        epilog="""
-        This tool will populate the inputs folder with the data from the PostgreSQL database.
-        config.yaml specifies the scenario parameters.
-        The environment variable DB_URL specifies the url to connect to the database. """,
-    )
-    parser.add_argument("--skip-cf", default=False, action='store_true',
-                        help="Skip creation variable_capacity_factors.csv. Useful when debugging and one doesn't"
-                             "want to wait for the command.")
-    parser.add_argument("--post-only", default=False, action='store_true',
-                        help="Only run the post solve functions (don't query db)")
-    parser.add_argument("--overwrite", default=False, action='store_true',
-                        help="Overwrite previous input files without prompting to confirm.")
-    args = parser.parse_args()  # Makes switch get_inputs --help works
-
-    # Load values from config.yaml
-    full_config = load_config()
-    switch_to_input_dir(full_config, overwrite=args.overwrite)
-
-    if not args.post_only:
-        query_db(full_config, skip_cf=args.skip_cf)
-    run_post_process(full_config)
-    print(f"\nScript took {timer.step_time_as_str()} seconds to build input tables.")
-
-
-def switch_to_input_dir(config, overwrite):
-    inputs_dir = config["inputs_dir"]
-
-    # Create inputs_dir if it doesn't exist
-    if not os.path.exists(inputs_dir):
-        os.makedirs(inputs_dir)
-        print("Inputs directory created.")
-    else:
-        if not overwrite and not query_yes_no(
-                "Inputs directory already exists. Allow contents to be overwritten?"
-        ):
-            raise SystemExit("User cancelled run.")
-
-    os.chdir(inputs_dir)
-    return inputs_dir
-
-
-if __name__ == "__main__":
-    main()

switch_model/wecc/get_inputs/__init__.py

@@ -0,0 +1,61 @@
+""" Script to retrieve the input data from the switch-wecc database and apply post-processing steps.
+"""
+import argparse
+import os
+
+from switch_model.utilities import query_yes_no, StepTimer
+from switch_model.wecc.get_inputs.get_inputs import query_db
+from switch_model.wecc.get_inputs.register_post_process import run_post_process
+from switch_model.wecc.utilities import load_config
+from switch_model.wecc.get_inputs.post_process_steps import *
+
+
+def main():
+    timer = StepTimer()
+
+    # Create command line tool, just provides help information
+    parser = argparse.ArgumentParser(
+        description="Write SWITCH input files from database tables.",
+        epilog="""
+        This tool will populate the inputs folder with the data from the PostgreSQL database.
+        config.yaml specifies the scenario parameters.
+        The environment variable DB_URL specifies the url to connect to the database. """,
+    )
+    parser.add_argument("--skip-cf", default=False, action='store_true',
+                        help="Skip creation variable_capacity_factors.csv. Useful when debugging and one doesn't"
+                             "want to wait for the command.")
+    parser.add_argument("--post-process", default=None, help="Run only this post process step.")
+    parser.add_argument("--overwrite", default=False, action='store_true',
+                        help="Overwrite previous input files without prompting to confirm.")
+    args = parser.parse_args()  # Makes switch get_inputs --help works
+
+    # Load values from config.yaml
+    full_config = load_config()
+    switch_to_input_dir(full_config, overwrite=args.overwrite)
+
+    if args.post_process is None:
+        query_db(full_config, skip_cf=args.skip_cf)
+    print("Post-processing...")
+    run_post_process(full_config, step_name=args.post_process)
+    print(f"\nScript took {timer.step_time_as_str()} seconds to build input tables.")
+
+
+def switch_to_input_dir(config, overwrite):
+    inputs_dir = config["inputs_dir"]
+
+    # Create inputs_dir if it doesn't exist
+    if not os.path.exists(inputs_dir):
+        os.makedirs(inputs_dir)
+        print("Inputs directory created.")
+    else:
+        if not overwrite and not query_yes_no(
+                "Inputs directory already exists. Allow contents to be overwritten?"
+        ):
+            raise SystemExit("User cancelled run.")
+
+    os.chdir(inputs_dir)
+    return inputs_dir
+
+
+if __name__ == "__main__":
+    main()

switch_model/wecc/get_inputs/cli.py

@@ -25,8 +25,8 @@ def fetch_df(tab_name, key, config):
     url = f"https://docs.google.com/spreadsheet/ccc?key={SHEET_ID}&output=csv&gid={gid}"
 
     df: pd.DataFrame = pd.read_csv(url, index_col=False) \
-        .replace("FALSE", 0) \
-        .replace("TRUE", 1)
+        .replace("FALSE", False) \
+        .replace("TRUE", True)
 
     if "description" in df.columns:
         df = df.drop("description", axis=1)
@@ -87,8 +87,6 @@ def drop_previous_candidate_storage():
     should_drop = (gen["gen_tech"] == STORAGE_TECH) & ~gen["GENERATION_PROJECT"].isin(predetermined_gen)
     # Find projects that we should drop (candidate storage)
     gen_to_drop = gen[should_drop]["GENERATION_PROJECT"]
-    # Verify we're dropping the right amount
-    assert len(gen_to_drop) == 50  # 50 is the number of load zones. we expect one candidate per load zone
 
     # Drop and write output
     gen = gen[~should_drop]