Add --data key=value CLI flag.

* Implement `--data key=value` CLI flag.

This reworks how `--ssh-*` and `--winrm-*` CLI flags are applied as data,
removing specific code for them from the main `pyinfra` package.

* Add inventory API tests.

Author: Fizzadar

pyinfra/api/inventory.py

@@ -28,22 +28,19 @@ class Inventory(object):
 
     Args:
         names_data: tuple of ``(names, data)``
-        ssh_user: override SSH user
-        ssh_port: override SSH port
-        ssh_key: override SSH key filename
-        ssh_key_password: override password for the SSH key
-        ssh_password: override SSH password
-        winrm_username: override WINRM username
-        winrm_password: override WINRM pasword
-        winrm_port: override WINRM port
-        winrm_transport: override WINRM transport
-        **groups: map of group names -> ``(names, data)``
+        override_data: dictionary of data overrides
+        ssh_*: deprecated, use ``override_data.ssh_*``
+        winrm_*: deprecated, use ``override_data.winrm_*``
+        **groups: map of group name -> ``(names, data)``
     '''
 
     state = None
 
     def __init__(
         self, names_data,
+        override_data=None,
+        # The below are deprecated, override_data.X is now preferred
+        # TODO: remove these in v2
         ssh_user=None, ssh_port=None, ssh_key=None,
         ssh_key_password=None, ssh_password=None,
         winrm_username=None, winrm_password=None,
@@ -55,24 +52,27 @@ def __init__(
         self.host_data = defaultdict(dict)  # dict of name -> data
         self.group_data = defaultdict(dict)  # dict of name -> data
 
-        # In CLI mode these are --user, --key, etc
-        override_data = {
-            'ssh_user': ssh_user,
-            'ssh_key': ssh_key,
-            'ssh_key_password': ssh_key_password,
-            'ssh_port': ssh_port,
-            'ssh_password': ssh_password,
-            'winrm_username': winrm_username,
-            'winrm_password': winrm_password,
-            'winrm_port': winrm_port,
-            'winrm_transport': winrm_transport,
-        }
-        # Strip None values
-        override_data = {
-            key: value
-            for key, value in six.iteritems(override_data)
-            if value is not None
-        }
+        override_data = override_data or {}
+
+        # TODO: remove these in v2
+        for key, value in (
+            ('ssh_user', ssh_user),
+            ('ssh_key', ssh_key),
+            ('ssh_key_password', ssh_key_password),
+            ('ssh_port', ssh_port),
+            ('ssh_password', ssh_password),
+            ('winrm_username', winrm_username),
+            ('winrm_password', winrm_password),
+            ('winrm_port', winrm_port),
+            ('winrm_transport', winrm_transport),
+        ):
+            if value is not None:
+                logger.warning((
+                    'Use of `{0}` is deprecated when creating `Inventory` classes, '
+                    'please use `override_data.{0}`'
+                ).format(key))
+                override_data[key] = value
+
         self.override_data = override_data
 
         names, data = names_data

pyinfra_cli/inventory.py

@@ -77,27 +77,12 @@ def _get_groups_from_filename(inventory_filename):
     }
 
 
-def make_inventory(
-    inventory_filename,
-    deploy_dir=None,
-    ssh_port=None,
-    ssh_user=None,
-    ssh_key=None,
-    ssh_key_password=None,
-    ssh_password=None,
-    winrm_username=None,
-    winrm_password=None,
-    winrm_port=None,
-    winrm_transport=None,
-):
+def make_inventory(inventory_filename, override_data=None, deploy_dir=None):
     '''
     Builds a ``pyinfra.api.Inventory`` from the filesystem. If the file does not exist
     and doesn't contain a / attempts to use that as the only hostname.
     '''
 
-    if ssh_port is not None:
-        ssh_port = int(ssh_port)
-
     file_groupname = None
 
     # If we're not a valid file we assume a list of comma separated hostnames
@@ -181,14 +166,6 @@ def make_inventory(
 
     return Inventory(
         groups.pop('all'),
-        ssh_user=ssh_user,
-        ssh_key=ssh_key,
-        ssh_key_password=ssh_key_password,
-        ssh_port=ssh_port,
-        ssh_password=ssh_password,
-        winrm_username=winrm_username,
-        winrm_password=winrm_password,
-        winrm_port=winrm_port,
-        winrm_transport=winrm_transport,
+        override_data=override_data,
         **groups
     ), file_groupname and file_groupname.lower()

pyinfra_cli/main.py

@@ -53,6 +53,7 @@
     get_operation_and_args,
     list_dirs_above_file,
     load_deploy_file,
+    parse_cli_arg,
 )
 from .virtualenv import init_virtualenv
 
@@ -118,6 +119,11 @@ def _print_support(ctx, param, value):
     multiple=True,
 )
 @click.option('--fail-percent', type=int, help='% of hosts allowed to fail.')
+@click.option(
+    '--data',
+    multiple=True,
+    help='Override data values, format key=value',
+)
 # Auth args
 @click.option(
     '--sudo', is_flag=True, default=False,
@@ -264,7 +270,7 @@ def _main(
     winrm_username, winrm_password, winrm_port,
     winrm_transport, shell_executable,
     sudo, sudo_user, use_sudo_password, su_user,
-    parallel, fail_percent,
+    parallel, fail_percent, data,
     dry, limit, no_wait, serial, quiet,
     debug, debug_data, debug_facts, debug_operations,
     facts=None, print_operations=None, support=None,
@@ -431,19 +437,36 @@ def _main(
     if not quiet:
         click.echo('--> Loading inventory...', err=True)
 
+    override_data = {}
+
+    for arg in data:
+        key, value = arg.split('=', 1)
+        override_data[key] = value
+
+    override_data = {
+        key: parse_cli_arg(value)
+        for key, value in override_data.items()
+    }
+
+    for key, value in (
+        ('ssh_user', ssh_user),
+        ('ssh_key', ssh_key),
+        ('ssh_key_password', ssh_key_password),
+        ('ssh_port', ssh_port),
+        ('ssh_password', ssh_password),
+        ('winrm_username', winrm_username),
+        ('winrm_password', winrm_password),
+        ('winrm_port', winrm_port),
+        ('winrm_transport', winrm_transport),
+    ):
+        if value:
+            override_data[key] = value
+
     # Load up the inventory from the filesystem
     inventory, inventory_group = make_inventory(
         inventory,
         deploy_dir=deploy_dir,
-        ssh_port=ssh_port,
-        ssh_user=ssh_user,
-        ssh_key=ssh_key,
-        ssh_key_password=ssh_key_password,
-        ssh_password=ssh_password,
-        winrm_username=winrm_username,
-        winrm_password=winrm_password,
-        winrm_port=winrm_port,
-        winrm_transport=winrm_transport,
+        override_data=override_data,
     )
 
     # Attach to pseudo inventory

pyinfra_cli/util.py

@@ -121,9 +121,9 @@ def json_encode(obj):
         raise TypeError('Cannot serialize: {0} ({1})'.format(type(obj), obj))
 
 
-def _parse_arg(arg):
+def parse_cli_arg(arg):
     if isinstance(arg, list):
-        return [_parse_arg(a) for a in arg]
+        return [parse_cli_arg(a) for a in arg]
 
     if arg.lower() == 'false':
         return False
@@ -177,12 +177,12 @@ def get_operation_and_args(commands):
             pass
 
     args = [
-        _parse_arg(arg)
+        parse_cli_arg(arg)
         for arg in operation_args if '=' not in arg
     ]
 
     kwargs = {
-        key: _parse_arg(value)
+        key: parse_cli_arg(value)
         for key, value in [
             arg.split('=', 1)
             for arg in operation_args if '=' in arg

tests/test_api/test_api_inventory.py

@@ -0,0 +1,48 @@
+from unittest import TestCase
+
+from pyinfra.api import Inventory
+
+
+class TestInventoryApi(TestCase):
+    def test_create_inventory(self):
+        inventory = Inventory((['somehost', 'anotherhost', 'anotherotherhost'], {}))
+        assert len(inventory) == 3
+
+    def test_create_inventory_data(self):
+        default_data = {'default_data': 'default_data'}
+        inventory = Inventory((['somehost'], default_data))
+        assert inventory.get_data() == default_data
+        assert inventory.get_host_data('somehost') == {}
+
+    def test_create_inventory_host_data(self):
+        default_data = {'default_data': 'default_data', 'host_data': 'none'}
+        somehost_data = {'host_data': 'host_data'}
+
+        inventory = Inventory((
+            [('somehost', somehost_data), 'anotherhost'],
+            default_data,
+        ))
+
+        assert inventory.get_data() == default_data
+        assert inventory.get_host_data('somehost') == somehost_data
+        assert inventory.get_host_data('anotherhost') == {}
+        assert inventory.get_host('anotherhost').data.host_data == 'none'
+
+    def test_create_inventory_override_data(self):
+        default_data = {'default_data': 'default_data', 'override_data': 'ignored'}
+        override_data = {'override_data': 'override_data'}
+        somehost_data = {'host_data': 'host_data'}
+
+        inventory = Inventory((
+            [('somehost', somehost_data), 'anotherhost'],
+            default_data,
+        ), override_data=override_data)
+
+        assert inventory.get_data() == default_data
+        assert inventory.get_override_data() == override_data
+
+        assert inventory.get_host('somehost').data.host_data == 'host_data'
+        assert inventory.get_host('anotherhost').data.host_data is None
+
+        assert inventory.get_host('somehost').data.override_data == 'override_data'
+        assert inventory.get_host('anotherhost').data.override_data == 'override_data'

tests/test_cli/test_cli.py

@@ -347,7 +347,7 @@ def test_deploy_operation_direct(self):
                 sudo=False, sudo_user=None, use_sudo_password=False, su_user=None,
                 parallel=None, fail_percent=0, dry=False, limit=None, no_wait=False, serial=False,
                 winrm_username=None, winrm_password=None, winrm_port=None, winrm_transport=None,
-                shell_executable=None, quiet=False,
+                shell_executable=None, quiet=False, data=tuple(),
                 debug=False, debug_data=False, debug_facts=False, debug_operations=False,
             )
             assert e.args == (0,)