Skip to content

Commit 667f53d

Browse files
giannisdoukasgiannisdoukas
committed
init dumpables
1 parent ea3cafd commit 667f53d

File tree

3 files changed

+117
-31
lines changed

3 files changed

+117
-31
lines changed

‎ipython2cwl/cwltoolextractor.py‎

Lines changed: 40 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -6,21 +6,27 @@
66
import tempfile
77
from collections import namedtuple
88
from pathlib import Path
9-
from typing import Dict, Any
9+
from typing import Dict, Any, List
1010

1111
import astor
1212
import nbconvert
1313
import yaml
1414
from nbformat.notebooknode import NotebookNode
1515

16-
from .iotypes import CWLFilePathInput, CWLBooleanInput, CWLIntInput, CWLStringInput, CWLFilePathOutput
16+
from .iotypes import CWLFilePathInput, CWLBooleanInput, CWLIntInput, CWLStringInput, CWLFilePathOutput, CWLDumpableFile, \
17+
CWLDumpableBinaryFile
1718
from .requirements_manager import RequirementsManager
1819

1920
with open(os.sep.join([os.path.abspath(os.path.dirname(__file__)), 'templates', 'template.dockerfile'])) as f:
2021
DOCKERFILE_TEMPLATE = f.read()
2122
with open(os.sep.join([os.path.abspath(os.path.dirname(__file__)), 'templates', 'template.setup'])) as f:
2223
SETUP_TEMPLATE = f.read()
2324

25+
_VariableNameTypePair = namedtuple(
26+
'VariableNameTypePair',
27+
['name', 'cwl_typeof', 'argparse_typeof', 'required', 'is_input', 'is_output', 'value']
28+
)
29+
2430

2531
# TODO: check if supports recursion if main function exists
2632

@@ -55,9 +61,15 @@ class AnnotatedVariablesExtractor(ast.NodeTransformer):
5561
CWLFilePathOutput.__name__
5662
}
5763

64+
dumpable_mapper = {
65+
(CWLDumpableFile.__name__,): "with open('{var_name}', 'w') as f:\n\tf.write({var_name})",
66+
(CWLDumpableBinaryFile.__name__,): "with open('{var_name}', 'wb') as f:\n\tf.write({var_name})",
67+
}
68+
5869
def __init__(self, *args, **kwargs):
5970
super().__init__(*args, **kwargs)
60-
self.extracted_nodes = []
71+
self.extracted_variables: List = []
72+
self.to_dump: List = []
6173

6274
def __get_annotation__(self, type_annotation):
6375
annotation = None
@@ -77,15 +89,27 @@ def visit_AnnAssign(self, node):
7789
annotation = self.__get_annotation__(node.annotation)
7890
if annotation in self.input_type_mapper:
7991
mapper = self.input_type_mapper[annotation]
80-
self.extracted_nodes.append(
81-
(node, mapper[0], mapper[1], not mapper[0].endswith('?'), True, False)
92+
self.extracted_variables.append(_VariableNameTypePair(
93+
node.target.id, mapper[0], mapper[1], not mapper[0].endswith('?'), True, False, None)
8294
)
8395
return None
84-
96+
elif annotation in self.dumpable_mapper:
97+
dump_tree = ast.parse(self.dumpable_mapper[annotation].format(var_name=node.target.id))
98+
self.to_dump.append(dump_tree.body)
99+
self.extracted_variables.append(_VariableNameTypePair(
100+
node.target.id, None, None, None, False, True, node.target.id)
101+
)
102+
# removing type annotation
103+
return ast.Assign(
104+
col_offset=node.col_offset,
105+
lineno=node.lineno,
106+
targets=[node.target],
107+
value=node.value
108+
)
85109
elif (isinstance(node.annotation, ast.Name) and node.annotation.id in self.output_type_mapper) or \
86110
(isinstance(node.annotation, ast.Str) and node.annotation.s in self.output_type_mapper):
87-
self.extracted_nodes.append(
88-
(node, None, None, None, False, True)
111+
self.extracted_variables.append(_VariableNameTypePair(
112+
node.target.id, None, None, None, False, True, node.value.s)
89113
)
90114
# removing type annotation
91115
return ast.Assign(
@@ -123,12 +147,6 @@ class AnnotatedIPython2CWLToolConverter:
123147
"""
124148

125149
_code: str
126-
127-
_VariableNameTypePair = namedtuple(
128-
'VariableNameTypePair',
129-
['name', 'cwl_typeof', 'argparse_typeof', 'required', 'is_input', 'is_output', 'value']
130-
)
131-
132150
"""The annotated python code to convert."""
133151

134152
def __init__(self, annotated_ipython_code: str):
@@ -137,19 +155,15 @@ def __init__(self, annotated_ipython_code: str):
137155

138156
self._code = annotated_ipython_code
139157
extractor = AnnotatedVariablesExtractor()
140-
self._tree = ast.fix_missing_locations(extractor.visit(ast.parse(self._code)))
158+
self._tree = extractor.visit(ast.parse(self._code))
159+
[self._tree.body.extend(d) for d in extractor.to_dump]
160+
self._tree = ast.fix_missing_locations(self._tree)
141161
self._variables = []
142-
for node, cwl_type, click_type, required, is_input, is_output in extractor.extracted_nodes:
143-
if is_input:
144-
self._variables.append(
145-
self._VariableNameTypePair(node.target.id, cwl_type, click_type, required, is_input, is_output,
146-
None)
147-
)
148-
if is_output:
149-
self._variables.append(
150-
self._VariableNameTypePair(node.target.id, cwl_type, click_type, required, is_input, is_output,
151-
node.value.s)
152-
)
162+
for variable in extractor.extracted_variables: # type: _VariableNameTypePair
163+
if variable.is_input:
164+
self._variables.append(variable)
165+
if variable.is_output:
166+
self._variables.append(variable)
153167

154168
@classmethod
155169
def from_jupyter_notebook_node(cls, node: NotebookNode) -> 'AnnotatedIPython2CWLToolConverter':

‎ipython2cwl/iotypes.py‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,38 @@
1+
class _CWLInput:
2+
pass
3+
4+
5+
class CWLFilePathInput(_CWLInput):
6+
pass
7+
8+
9+
class CWLBooleanInput(_CWLInput):
10+
pass
11+
12+
13+
class CWLStringInput(_CWLInput):
14+
pass
15+
16+
17+
class CWLIntInput(_CWLInput):
18+
pass
19+
120

2-
class CWLFilePathInput:
21+
class _CWLOutput:
322
pass
423

524

6-
class CWLBooleanInput:
25+
class CWLFilePathOutput(_CWLOutput):
726
pass
827

928

10-
class CWLStringInput:
29+
class _CWLDumpable(_CWLOutput):
1130
pass
1231

1332

14-
class CWLIntInput:
33+
class CWLDumpableFile(_CWLDumpable):
1534
pass
1635

1736

18-
class CWLFilePathOutput:
37+
class CWLDumpableBinaryFile(_CWLDumpable):
1938
pass

‎tests/test_cwltoolextractor.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,3 +377,56 @@ def test_AnnotatedIPython2CWLToolConverter_optional_array_input(self):
377377
self.assertListEqual([], AnnotatedIPython2CWLToolConverter(os.linesep.join([
378378
'x1: "RANDOM CHARACTERS!!!!!!" = True'
379379
]))._variables)
380+
381+
def test_AnnotatedIPython2CWLToolConverter_dumpables(self):
382+
script = os.linesep.join([
383+
'message: CWLDumpableFile = "this is a text from a dumpable"',
384+
'message2: "CWLDumpableFile" = "this is a text from a dumpable 2"',
385+
'binary_message: CWLDumpableBinaryFile = b"this is a text from a binary dumpable"',
386+
'print("Message:", message)',
387+
'print(b"Binary Message:" + binary_message)',
388+
])
389+
converter = AnnotatedIPython2CWLToolConverter(script)
390+
generated_script = AnnotatedIPython2CWLToolConverter._wrap_script_to_method(
391+
converter._tree, converter._variables
392+
)
393+
for f in ['message', 'binary_message', 'message2']:
394+
try:
395+
os.remove(f)
396+
except FileNotFoundError:
397+
pass
398+
exec(generated_script)
399+
print(generated_script)
400+
locals()['main']()
401+
with open('message') as f:
402+
self.assertEqual('this is a text from a dumpable', f.read())
403+
with open('message2') as f:
404+
self.assertEqual('this is a text from a dumpable 2', f.read())
405+
with open('binary_message', 'rb') as f:
406+
self.assertEqual(b'this is a text from a binary dumpable', f.read())
407+
408+
cwl_tool = converter.cwl_command_line_tool()
409+
print(cwl_tool)
410+
self.assertDictEqual(
411+
{
412+
'message': {
413+
'type': 'File',
414+
'outputBinding': {
415+
'glob': 'message'
416+
}
417+
},
418+
'message2': {
419+
'type': 'File',
420+
'outputBinding': {
421+
'glob': 'message2'
422+
}
423+
},
424+
'binary_message': {
425+
'type': 'File',
426+
'outputBinding': {
427+
'glob': 'binary_message'
428+
}
429+
}
430+
},
431+
cwl_tool['outputs']
432+
)

0 commit comments

Comments
 (0)