Merge pull request #152 from assemblerflow/recipes

Added recipes system

Author: ODiogoSilva

changelog.md

@@ -2,6 +2,13 @@
 
 ## Changes in upcoming release (`dev` branch) 
 
+### New features
+
+- Added new `recipe` system to flowcraft along with 6 starting recipes.
+Recipes are pre-made and curated pipelines that address specific questions.
+To create a recipe, the `-r <recipe_name>` can be used. To list available
+recipes, the `--recipe-list` and `--recipe-list-short` options were added. 
+
 ### Components changes
 
 - Added new `disableRR` param in the `spades` component that disables repeat

docs/dev/create_recipe.rst

@@ -0,0 +1,116 @@
+Recipe creation guidelines
+===========================
+
+Recipes are pre-made pipeline strings that may be associated with specific
+parameters and directives and are used to rapidly build a certain type of
+pipeline.
+
+Instead of building a pipeline like::
+
+    -t "integrity_coverage fastqc_trimmomatic fastqc spades pilon"
+
+The user simply can specific a recipe with that pipeline::
+
+    -r assembly
+
+Recipe creation
+---------------
+
+The creation of new recipes is a very simple and straightforward process.
+You need to create a new file in the ``flowcraft/generator/recipes`` folder
+with any name and create a basic class with three attributes::
+
+    try:
+        from generator.recipe import Recipe
+    except ImportError:
+        from flowcraft.generator.recipe import Recipe
+
+
+    class Innuca(Recipe):
+
+        def __init__(self):
+            super().__init__()
+
+            # Recipe name
+            self.name = "innuca"
+
+            # Recipe pipeline
+            self.pipeline_str = <pipeline string>
+
+            # Recipe parameters and directives
+            self.directives = { <directives> }
+
+And that's it! Now there is a new recipe available with the ``innuca`` name and
+we can build this pipeline using the option ``-r innuca``.
+
+Name
+^^^^
+
+This is the name of the recipe, which is used to make a match with the recipe
+name provided by the user via the ``-r`` option.
+
+Pipeline_str
+^^^^^^^^^^^^
+
+The pipeline string as if provided via the ``-t`` option.
+
+Directives
+^^^^^^^^^^
+
+A dictionary containing the parameters and directives for each process in the
+pipeline string. **Setting this attribute is optional and components
+that are not specified here will assume their default values**. In general, each
+element in this dictionary should have the following format::
+
+    self.directives = {
+        "component_name": {
+            "params": {
+                "paramA": "value"
+            },
+            "directives": {
+                "directiveA": "value"
+            }
+        }
+    }
+
+This will set the provided parameters and directives to the component, but it is
+possible to provide only one.
+
+A more concrete example of a real component and directives follows::
+
+    self.pipeline_str = "integrity_coverage fastqc"
+
+    # Set parameters and directives only for integrity_coverage
+    # and leave fastqc with the defaults
+    self.directives = {
+        "integrity_coverage": {
+            "params": {
+                "minCoverage": 20
+            },
+            "directives": {
+                "memory": "1GB"
+            }
+        }
+    }
+
+Duplicate components
+~~~~~~~~~~~~~~~~~~~~
+
+In some cases, the same component may be present multiple times in the pipeline
+string of a recipe. In these cases, directives can be assigned to each individual
+component by adding a ``#<id>`` suffix to the component::
+
+    self.pipeline_str = "integrity_coverage ( trimmomatic spades#1 | spades#2)"
+
+    self.directives = {
+        "spades#1": {
+            "directives": {
+                "memory": "10GB"
+            }
+        },
+        "spades#2": {
+            "directives": {
+                "version": "3.7.0"
+            }
+        }
+    }

docs/index.rst

@@ -44,6 +44,7 @@ A NextFlow pipeline assembler for genomics.
    dev/general_orientation
    dev/create_process
    dev/create_template
+   dev/create_recipe
    dev/containers
    dev/process_dotfiles
    dev/pipeline_reporting

docs/resources/reports/mash_dist_table.png

@@ -9,6 +9,7 @@ Pipeline reports
 .. include:: reports/fastqc.rst
 .. include:: reports/fastqc_trimmomatic.rst
 .. include:: reports/integrity_coverage.rst
+.. include:: reports/mash_dist.rst
 .. include:: reports/mlst.rst
 .. include:: reports/patho_typing.rst
 .. include:: reports/pilon.rst
@@ -18,6 +19,7 @@ Pipeline reports
 .. include:: reports/process_spades.rst
 .. include:: reports/process_viral_assembly.rst
 .. include:: reports/seq_typing.rst
+.. include:: reports/sistr.rst
 .. include:: reports/trimmomatic.rst
 .. include:: reports/true_coverage.rst
 

docs/resources/reports/sliding_window_mash_dist.png

@@ -0,0 +1,20 @@
+mash_dist
+---------
+
+Table data
+^^^^^^^^^^
+
+Plasmids table:
+    - **Mash Dist**: Number of plasmid hits
+
+.. image:: ../resources/reports/mash_dist_table.png
+    :align: center
+
+Plot data
+^^^^^^^^^
+
+- **Sliding window Plasmid annotation**: Provides annotation of plasmid
+  hits along the genome assembly. This report component is only available
+  when the ``mash_dist`` component is used.
+
+.. image:: ../resources/reports/sliding_window_mash_dist.png

docs/user/pipeline_reports.rst

@@ -0,0 +1,11 @@
+sistr
+-----
+
+Table data
+^^^^^^^^^^
+
+Typing table:
+    - **sistr**: The sequence typing result.
+
+.. image:: ../resources/reports/typing_table.png
+    :align: center

docs/user/reports/mash_dist.rst

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+
+import argparse
+from itertools import groupby
+import os
+
+
+def replace_char(text):
+    for ch in ['/', '`', '*', '{', '}', '[', ']', '(', ')', '#', '+', '-', '.', '!', '$', ':']:
+        text = text.replace(ch, "_")
+    return text
+
+def getSequence(ref, fasta):
+
+    entry = (x[1] for x in groupby(fasta, lambda line: line[0] == ">"))
+
+    for header in entry:
+        headerStr = header.__next__()[1:].strip()
+        seq = "".join(s.strip() for s in entry.__next__())
+
+        if ref == headerStr.replace('>',''):
+            filename = os.path.join(os.getcwd(), ref.replace('/','_').split('|')[0])
+            fasta_header = replace_char(headerStr)
+            output_file = open(filename + '.fa', "w")
+            output_file.write(">" + fasta_header + "\n" + seq.upper() + "\n")
+            output_file.close()
+
+def main():
+
+    parser = argparse.ArgumentParser(prog='parse_fasta.py', description="Parse FASTA files for a specific header", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('--version', help='Version information', action='version', version=str('%(prog)s v0.1'))
+
+    parser_required = parser.add_argument_group('Required options')
+    parser_required.add_argument('-t', type=str, metavar='header of sequence to be retrieved',
+                             help='Uncompressed fastq file containing mate 1 reads', required=True)
+    parser_required.add_argument('-f', type=argparse.FileType('r'), metavar='/path/to/input/file.fasta',
+                             help='Fasta with the sequences', required=True)
+
+    args = parser.parse_args()
+
+    getSequence(args.t, args.f)
+
+
+
+if __name__ == "__main__":
+    main()

docs/user/reports/sistr.rst

@@ -16,7 +16,7 @@
     from generator.engine import NextflowGenerator, process_map
     from generator.inspect import NextflowInspector
     from generator.report import FlowcraftReport
-    from generator.recipe import brew_recipe, available_recipes
+    from generator.recipe import brew_innuendo, brew_recipe, list_recipes
     from generator.pipeline_parser import parse_pipeline, SanityError
     from generator.process_details import proc_collector, colored_print
     import generator.error_handling as eh
@@ -25,7 +25,8 @@
     from flowcraft.generator.engine import NextflowGenerator, process_map
     from flowcraft.generator.inspect import NextflowInspector
     from flowcraft.generator.report import FlowcraftReport
-    from flowcraft.generator.recipe import brew_recipe, available_recipes
+    from flowcraft.generator.recipe import brew_innuendo, \
+        brew_recipe, list_recipes
     from flowcraft.generator.pipeline_parser import parse_pipeline, \
         SanityError
     from flowcraft.generator.process_details import proc_collector, \
@@ -77,13 +78,22 @@ def get_args(args=None):
         const=True, help="Check only the validity of the pipeline "
                          "string and exit.")
     group_lists.add_argument(
-        "-L", "--detailed-list", action="store_const", dest="detailed_list",
+        "-L", "--component-list", action="store_const", dest="detailed_list",
         const=True, help="Print a detailed description for all the "
-                         "currently available processes")
+                         "currently available processes.")
     group_lists.add_argument(
-        "-l", "--short-list", action="store_const", dest="short_list",
+        "-l", "--component-list-short", action="store_const", dest="short_list",
         const=True, help="Print a short list of the currently "
-                         "available processes")
+                         "available processes.")
+    group_lists.add_argument(
+        "--recipe-list", dest="recipe_list", action="store_const", const=True,
+        help="Print a short list of the currently available recipes."
+    )
+    group_lists.add_argument(
+        "--recipe-list-short", dest="recipe_list_short", action="store_const",
+        const=True, help="Print a condensed list of the currently available "
+                         "recipes"
+    )
     build_parser.add_argument(
         "-cr", "--check-recipe", dest="check_recipe",
         action="store_const", const=True,
@@ -277,6 +287,12 @@ def build(args):
     if args.export_params or args.export_directives:
         logger.setLevel(logging.ERROR)
 
+    if args.recipe_list_short:
+        list_recipes()
+
+    if args.recipe_list:
+        list_recipes(full=True)
+
     welcome = [
         "========= F L O W C R A F T =========",
         "Build mode\n"
@@ -293,13 +309,14 @@ def build(args):
     # appropriate recipe
     if args.recipe:
         if args.recipe == "innuendo":
-            pipeline_string = brew_recipe(args, available_recipes)
+            pipeline_string = brew_innuendo(args)
         else:
-            pipeline_string = available_recipes[args.recipe]
+            # pipeline_string = available_recipes[args.recipe]
+            pipeline_string = brew_recipe(args.recipe)
             if args.tasks:
                 logger.warning(colored_print(
-                    "-t parameter will be ignored for recipe: {}\n"
-                        .format(args.recipe), "yellow_bold")
+                    "-t parameter will be ignored for recipe: {}\n".format(
+                        args.recipe), "yellow_bold")
                 )
 
         if args.check_recipe: