playbook improvements (#470)

Signed-off-by: David Chung <david.chung@docker.com>

Author: David Chung

Makefile

@@ -60,6 +60,13 @@ ifeq (${DOCKER_TAG_LATEST},true)
 endif
 endif
 
+build-docker-dev:
+	@echo "+ $@"
+	GOOS=linux GOARCH=amd64 make build-in-container
+	@docker build ${DOCKER_BUILD_FLAGS} \
+	-t ${DOCKER_IMAGE}:${DOCKER_TAG} \
+	-f ${CURDIR}/dockerfiles/Dockerfile.bundle .
+
 get-tools:
 	@echo "+ $@"
 	@go get -u \

cmd/cli/base/template.go

@@ -7,8 +7,8 @@ import (
 	"path"
 	"strings"
 
+	"github.com/docker/infrakit/pkg/cli"
 	"github.com/docker/infrakit/pkg/discovery"
-	metadata_template "github.com/docker/infrakit/pkg/plugin/metadata/template"
 	"github.com/docker/infrakit/pkg/template"
 	"github.com/ghodss/yaml"
 	"github.com/spf13/pflag"
@@ -134,25 +134,7 @@ func TemplateProcessor(plugins func() discovery.Plugins) (*pflag.FlagSet, ToJSON
 				}
 			}
 
-			engine.WithFunctions(func() []template.Function {
-				return []template.Function{
-					{
-						Name: "metadata",
-						Description: []string{
-							"Metadata function takes a path of the form \"plugin_name/path/to/data\"",
-							"and calls GET on the plugin with the path \"path/to/data\".",
-							"It's identical to the CLI command infrakit metadata cat ...",
-						},
-						Func: metadata_template.MetadataFunc(plugins),
-					},
-					{
-						Name: "resource",
-						Func: func(s string) string {
-							return fmt.Sprintf("{{ resource `%s` }}", s)
-						},
-					},
-				}
-			})
+			cli.ConfigureTemplate(engine, plugins)
 
 			view, err = engine.Render(nil)
 			if err != nil {

cmd/cli/main/main.go

@@ -5,17 +5,26 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"strings"
 
 	"github.com/docker/infrakit/cmd/cli/base"
-	"github.com/docker/infrakit/cmd/cli/playbook"
 	"github.com/docker/infrakit/pkg/cli"
 	cli_local "github.com/docker/infrakit/pkg/cli/local"
-	cli_remote "github.com/docker/infrakit/pkg/cli/remote"
 	"github.com/docker/infrakit/pkg/discovery"
 	discovery_local "github.com/docker/infrakit/pkg/discovery/local"
 	"github.com/docker/infrakit/pkg/discovery/remote"
 	logutil "github.com/docker/infrakit/pkg/log"
 	"github.com/spf13/cobra"
+
+	_ "github.com/docker/infrakit/cmd/cli/event"
+	_ "github.com/docker/infrakit/cmd/cli/flavor"
+	_ "github.com/docker/infrakit/cmd/cli/group"
+	_ "github.com/docker/infrakit/cmd/cli/manager"
+	_ "github.com/docker/infrakit/cmd/cli/playbook"
+	_ "github.com/docker/infrakit/cmd/cli/plugin"
+	_ "github.com/docker/infrakit/cmd/cli/resource"
+	_ "github.com/docker/infrakit/cmd/cli/template"
+	_ "github.com/docker/infrakit/cmd/cli/util"
 )
 
 func init() {
@@ -55,10 +64,19 @@ func main() {
 
 		if len(remotes) > 0 {
 			for _, h := range remotes {
+				addProtocol := false
+				if !strings.Contains(h, "://") {
+					h = "http://" + h
+					addProtocol = true
+				}
 				u, err := url.Parse(h)
 				if err != nil {
 					return err
 				}
+				if addProtocol {
+					u.Scheme = "http"
+				}
+
 				ulist = append(ulist, u)
 			}
 		}
@@ -96,48 +114,11 @@ func main() {
 		cmd.AddCommand(c)
 	})
 
-	mods := []*cobra.Command{}
-	// additional modules
-	if os.Getenv(cli.CliDirEnvVar) != "" {
-		modules, err := cli_local.NewModules(cli_local.Dir())
-		if err != nil {
-			log.Crit("error executing", "err", err)
-			os.Exit(1)
-		}
-		localModules, err := modules.List()
-		log.Debug("modules", "local", localModules)
-		if err != nil {
-			log.Crit("error executing", "err", err)
-			os.Exit(1)
-		}
-		mods = append(mods, localModules...)
-	}
-
-	// any remote modules?
-	pmod, err := playbook.Load()
-	if err != nil {
-		log.Warn("playbooks failed to load", "err", err)
-	} else {
-		if playbooks, err := cli_remote.NewModules(pmod, os.Stdin); err != nil {
-			log.Warn("error loading playbooks", "err", err)
-		} else {
-			if more, err := playbooks.List(); err != nil {
-				log.Warn("cannot list playbooks", "err", err)
-			} else {
-				mods = append(mods, more...)
-			}
-		}
-	}
-
-	for _, mod := range mods {
-		log.Debug("Adding", "module", mod.Use)
-		cmd.AddCommand(mod)
-	}
-
-	cmd.SetUsageTemplate(usageTemplate)
+	// Help template includes the usage string, which is configure below
 	cmd.SetHelpTemplate(helpTemplate)
+	cmd.SetUsageTemplate(usageTemplate)
 
-	err = cmd.Execute()
+	err := cmd.Execute()
 	if err != nil {
 		log.Crit("error executing", "cmd", cmd.Use, "err", err)
 		fmt.Println(err.Error())

cmd/cli/playbook/playbook.go

@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 
 	"github.com/docker/infrakit/cmd/cli/base"
+	"github.com/docker/infrakit/pkg/cli"
+	"github.com/docker/infrakit/pkg/cli/local"
 	"github.com/docker/infrakit/pkg/cli/remote"
 	"github.com/docker/infrakit/pkg/discovery"
 	logutil "github.com/docker/infrakit/pkg/log"
@@ -185,5 +187,51 @@ func Command(plugins func() discovery.Plugins) *cobra.Command {
 
 	cmd.AddCommand(add, remove, list)
 
+	reserved := map[*cobra.Command]int{add: 1, remove: 1, list: 1}
+
+	// Modules
+	mods := []*cobra.Command{}
+	// additional modules
+	if os.Getenv(cli.CliDirEnvVar) != "" {
+		modules, err := local.NewModules(plugins, local.Dir())
+		if err != nil {
+			log.Crit("error executing", "err", err)
+			os.Exit(1)
+		}
+		localModules, err := modules.List()
+		log.Debug("modules", "local", localModules)
+		if err != nil {
+			log.Crit("error executing", "err", err)
+			os.Exit(1)
+		}
+		mods = append(mods, localModules...)
+	}
+
+	// any remote modules?
+	pmod, err := Load()
+	if err != nil {
+		log.Warn("playbooks failed to load", "err", err)
+	} else {
+		if playbooks, err := remote.NewModules(plugins, pmod, os.Stdin); err != nil {
+			log.Warn("error loading playbooks", "err", err)
+		} else {
+			if more, err := playbooks.List(); err != nil {
+				log.Warn("cannot list playbooks", "err", err)
+			} else {
+				mods = append(mods, more...)
+			}
+		}
+	}
+
+	for _, mod := range mods {
+		if _, has := reserved[mod]; has {
+			log.Warn("cannot override reserverd command; igored", "conflict", mod.Use)
+			continue
+		}
+
+		log.Debug("Adding", "module", mod.Use)
+		cmd.AddCommand(mod)
+	}
+
 	return cmd
 }

cmd/cli/util/util.go

@@ -1,6 +1,8 @@
 package util
 
 import (
+	"net/http"
+	"os"
 	"time"
 
 	"github.com/docker/infrakit/cmd/cli/base"
@@ -16,13 +18,39 @@ func init() {
 	base.Register(Command)
 }
 
+func fileServerCommand(plugins func() discovery.Plugins) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "fileserver <path>",
+		Short: "Fileserver over http",
+	}
+
+	listen := cmd.Flags().StringP("listen", "l", ":8080", "Listening port")
+
+	cmd.RunE = func(c *cobra.Command, args []string) error {
+
+		if len(args) != 1 {
+			c.Usage()
+			os.Exit(-1)
+		}
+
+		logger.Info("Starting file server", "listen", *listen)
+
+		rootFS := args[0]
+		handler := http.FileServer(http.Dir(rootFS))
+		return http.ListenAndServe(*listen, handler)
+	}
+
+	return cmd
+}
+
 func muxCommand(plugins func() discovery.Plugins) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "mux",
 		Short: "API mux service",
 	}
 
-	listen := cmd.Flags().StringP("listen", "l", ":4358", "Listening port")
+	// http://www.speedguide.net/port.php?port=24864 - unassigned by IANA
+	listen := cmd.Flags().StringP("listen", "l", ":24864", "Listening port")
 	autoStop := cmd.Flags().BoolP("auto-stop", "a", false, "True to stop when no plugins are running")
 	interval := cmd.Flags().DurationP("scan", "s", 1*time.Minute, "Scan interval to check for plugins")
 
@@ -74,7 +102,7 @@ func Command(plugins func() discovery.Plugins) *cobra.Command {
 		Short: "Utilties",
 	}
 
-	util.AddCommand(muxCommand(plugins))
+	util.AddCommand(muxCommand(plugins), fileServerCommand(plugins))
 
 	return util
 }

dockerfiles/Dockerfile.bundle

@@ -1,13 +1,24 @@
-FROM alpine:3.5
-
-RUN apk add --update ca-certificates
+FROM docker:latest
 
 RUN mkdir -p /infrakit/plugins /infrakit/configs /infrakit/logs /infrakit/cli
 
 VOLUME /infrakit
 
 ENV INFRAKIT_HOME /infrakit
+
+# Defined in pkg/discovery
 ENV INFRAKIT_PLUGINS_DIR /infrakit/plugins
+
+# Defined in pkg/cli
 ENV INFRAKIT_CLI_DIR /infrakit/cli
 
+# Defined in cmd/cli/playbook
+ENV INFRAKIT_PLAYBOOKS_FILE /infrakit/playbooks
+
+# When using the manager 'os' option
+ENV INFRAKIT_LEADER_FILE /infrakit/leader
+ENV INFRAKIT_STORE_DIR /infrakit/configs
+
+
+
 ADD build/* /usr/local/bin/

pkg/cli/context.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/docker/infrakit/pkg/discovery"
 	"github.com/docker/infrakit/pkg/template"
 	"github.com/docker/infrakit/pkg/util/exec"
 	"github.com/spf13/cobra"
@@ -21,20 +22,22 @@ const (
 
 // Context is the context for the running module
 type Context struct {
-	cmd    *cobra.Command
-	src    string
-	input  io.Reader
-	exec   bool
-	run    func(string) error
-	script string
+	cmd     *cobra.Command
+	src     string
+	input   io.Reader
+	exec    bool
+	run     func(string) error
+	script  string
+	plugins func() discovery.Plugins
 }
 
 // NewContext creates a context
-func NewContext(cmd *cobra.Command, src string, input io.Reader) *Context {
+func NewContext(plugins func() discovery.Plugins, cmd *cobra.Command, src string, input io.Reader) *Context {
 	return &Context{
-		cmd:   cmd,
-		src:   src,
-		input: input,
+		plugins: plugins,
+		cmd:     cmd,
+		src:     src,
+		input:   input,
 	}
 }
 
@@ -317,7 +320,8 @@ func (c *Context) BuildFlags() error {
 	if err != nil {
 		return err
 	}
-	_, err = t.Render(c)
+
+	_, err = ConfigureTemplate(t, c.plugins).Render(c)
 	return err
 }
 
@@ -334,8 +338,9 @@ func (c *Context) Execute() error {
 	if err != nil {
 		return err
 	}
+
 	c.exec = true
-	script, err := t.Render(c)
+	script, err := ConfigureTemplate(t, c.plugins).Render(c)
 	if err != nil {
 		return err
 	}