Merge pull request #133 from assemblerflow/dev

1.3.1

Author: cimendes

changelog.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## Changes in upcoming release (`dev` branch) 
+
+### Features
+
+- Added a new `clearInput` parameter to components that change their input.
+The aim of this option is to allow the controlled removal of temporary files,
+which is particularly useful in very large workflows.
+
+### Components changes
+
+- Updated images for components `mash_dist`, `mash_screen` and 
+`mapping_patlas`.
+
+### New components
+- Added component `fast_ani`.
+
+### Minor/Other changes
+
+- Added `--export-directives` option to `build` mode to export component's 
+directives in JSON format to standard output.
+- Added more date information in `inspect` mode, including the year and the
+locale of the executing system.
+
 ## 1.3.0
 
 ### Features

docker/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.6-alpine3.7
+MAINTAINER Bruno Gonçalves <bfgoncalves@medicina.ulisboa.pt>
+
+RUN apk add --no-cache git
+
+WORKDIR /flowcraft
+
+# Clone FlowCraft
+RUN git clone https://github.com/assemblerflow/flowcraft.git
+WORKDIR ./flowcraft
+
+# Install flowcraft
+RUN python setup.py install
+
+WORKDIR /flowcraft
+
+# Remove unnecessary packages
+RUN apk del git

docs/user/available_components.rst

@@ -99,7 +99,13 @@ Distance Estimation
   plasmid database and generates a JSON input file for pATLAS. This component
   searches for containment of a given sequence in read sequencing data.
   However if a different database is provided it can use mash screen for other
-  purporses.
+  purposes.
+
+- :doc:`components/fast_ani`: Performs pairwise comparisons between fastas,
+given a multifasta as input for fastANI. It will split the multifasta into
+single fastas that will then be provided as a matrix. The output will be the
+all pairwise comparisons that pass the minimum of 50 aligned sequences with a
+default length of 200 bp.
 
 - :doc:`components/mash_sketch_fasta`: Performs mash sketch for fasta files.
 

docs/user/components/fast_ani.rst

@@ -0,0 +1,47 @@
+fast_ani
+========
+
+Purpose
+-------
+
+This component performs pairwise comparisons between fastas,
+given a multifasta as input for fastANI. It will split the multifasta into
+single fastas that will then be provided as a matrix. The output will be the
+all pairwise comparisons that pass the minimum of 50 aligned sequences with a
+default length of 200 bp.
+
+Input/Output type
+------------------
+
+- Input type: ``fasta``
+- Output type: ``None``
+
+
+Parameters
+----------
+
+- ``fragLen``: Sets the minimum size of the fragment to be passed to
+`--fragLen` argument of fastANI.
+
+
+Published results
+-----------------
+
+- ``results/fast_ani/``: A text file with the extension `.out`, which has all
+the pairwise comparisons between sequences, reporting ANI.
+
+
+Published reports
+-----------------
+
+None.
+
+
+Default directives
+------------------
+
+- ``fastAniMatrix``:
+    - ``container``: flowcraft/fast_ani
+    - ``version``: 1.1.0-2
+    - ``cpus``: 20
+    - ``memory``: { 30.GB * task.attempt }

docs/user/components/mapping_patlas.rst

@@ -60,10 +60,10 @@ Default directives
 
 - ``mappingBowtie``:
     - ``container``: flowcraft/mapping-patlas
-    - ``version``: 1.4.1
+    - ``version``: 1.6.0-1
 - ``samtoolsView``:
     - ``container``: flowcraft/mapping-patlas
-    - ``version``: 1.4.1
+    - ``version``: 1.6.0-1
 - ``jsonDumpingMapping``:
     - ``container``: flowcraft/mapping-patlas
-    - ``version``: 1.4.1
+    - ``version``: 1.6.0-1

docs/user/components/mash_dist.rst

@@ -65,7 +65,7 @@ Default directives
 
 - ``runMashDist``:
     - ``container``: flowcraft/mash-patlas
-    - ``version``: 1.4.1
+    - ``version``: 1.6.0-1
 - ``mashDistOutputJson``:
     - ``container``: flowcraft/mash-patlas
-    - ``version``: 1.4.1
+    - ``version``: 1.6.0-1

docs/user/components/mash_screen.rst

@@ -60,7 +60,7 @@ Default directives
 
 - ``mashScreen``:
     - ``container``: flowcraft/mash-patlas
-    - ``version``: 1.4.1
+    - ``version``: 1.6.0-1
 - ``mashOutputJson``:
     - ``container``: flowcraft/mash-patlas
-    - ``version``: 1.4.1
+    - ``version``: 1.6.0-1

flowcraft/flowcraft.py

@@ -97,6 +97,12 @@ def get_args(args=None):
                          "components (via -t option) in JSON format to stdout. "
                          "No pipeline will be generated with this option."
     )
+    build_parser.add_argument(
+        "--export-directives", dest="export_directives", action="store_const",
+        const=True, help="Only export the directives for the provided "
+                         "components (via -t option) in JSON format to stdout. "
+                         "No pipeline will be generated with this option."
+    )
 
     # GENERAL OPTIONS
     parser.add_argument(
@@ -177,7 +183,7 @@ def validate_build_arguments(args):
 
     # Skill all checks when exporting parameters AND providing at least one
     # component
-    if args.export_params:
+    if args.export_params or args.export_directives:
         # Check if components provided
         if not args.tasks:
             logger.error(colored_print(
@@ -266,8 +272,9 @@ def copy_project(path):
 
 def build(args):
 
-    # Disable standard logging for stdout when the following modes are executed:
-    if args.export_params:
+    # Disable standard logging for stdout when the following modes are
+    #  executed:
+    if args.export_params or args.export_directives:
         logger.setLevel(logging.ERROR)
 
     welcome = [
@@ -330,6 +337,9 @@ def build(args):
     if args.export_params:
         nfg.export_params()
         sys.exit(0)
+    elif args.export_directives:
+        nfg.export_directives()
+        sys.exit(0)
     else:
         # building the actual pipeline nf file
         nfg.build()

flowcraft/generator/components/assembly.py

@@ -50,6 +50,14 @@ def __init__(self, **kwargs):
                     "If 'auto' the SPAdes k-mer lengths will be determined "
                     "from the maximum read length of each assembly. If "
                     "'default', SPAdes will use the default k-mer lengths. "
+            },
+            "clearInput": {
+                "default": "false",
+                "description":
+                    "Permanently removes temporary input files. This option "
+                    "is only useful to remove temporary files in large "
+                    "workflows and prevents nextflow's resume functionality. "
+                    "Use with caution."
             }
         }
 
@@ -81,6 +89,18 @@ def __init__(self, **kwargs):
             "scratch": "true"
         }}
 
+        self.params = {
+            "clearInput": {
+                "default": "false",
+                "description":
+                    "Permanently removes temporary input files. This option "
+                    "is only useful to remove temporary files in large "
+                    "workflows and prevents nextflow's resume functionality. "
+                    "Use with caution."
+            }
+        }
+
+
 class ViralAssembly(Process):
     """
     Process to assemble viral genomes, based on SPAdes and megahit
@@ -95,7 +115,8 @@ def __init__(self, **kwargs):
 
         self.dependencies = ["integrity_coverage"]
 
-        self.status_channels = ["va_spades" , "va_megahit", "report_viral_assembly"]
+        self.status_channels = ["va_spades", "va_megahit",
+                                "report_viral_assembly"]
 
         self.link_end.append({"link": "SIDE_max_len", "alias": "SIDE_max_len"})
 
@@ -105,7 +126,7 @@ def __init__(self, **kwargs):
             "container": "flowcraft/viral_assembly",
             "version": "0.1-1",
             "scratch": "true"
-        },"va_megahit": {
+        }, "va_megahit": {
             "cpus": 4,
             "memory": "{ 5.GB * task.attempt }",
             "container": "flowcraft/viral_assembly",
@@ -146,5 +167,13 @@ def __init__(self, **kwargs):
                     "from the maximum read length of each assembly. If "
                     "'default', megahit will use the default k-mer lengths. "
                     "(default: $params.megahitKmers)"
+            },
+            "clearInput": {
+                "default": "false",
+                "description":
+                    "Permanently removes temporary input files. This option "
+                    "is only useful to remove temporary files in large "
+                    "workflows and prevents nextflow's resume functionality. "
+                    "Use with caution."
             }
-        }
+        }

flowcraft/generator/components/assembly_processing.py

@@ -204,6 +204,17 @@ def __init__(self, **kwargs):
         self.link_end.append({"link": "SIDE_BpCoverage",
                               "alias": "SIDE_BpCoverage"})
 
+        self.params = {
+            "clearInput": {
+                "default": "false",
+                "description":
+                    "Permanently removes temporary input files. This option "
+                    "is only useful to remove temporary files in large "
+                    "workflows and prevents nextflow's resume functionality. "
+                    "Use with caution."
+            }
+        }
+
         self.directives = {
             "pilon": {
                 "cpus": 4,