move runtime definition from cli-v2 to csdp-official (#620)

* move runtime definition from cli-v2 to csdp-official

* wip

* codegen

* fix

* fix

* fix

* fix

* add cli version constraint

* fix constraint

* fix

* add script to get manifests location

* fixes according to code review

* fix

Author: danielm-codefresh

Makefile

@@ -1,4 +1,4 @@
-VERSION=v0.0.569
+VERSION=v0.1.0
 
 OUT_DIR=dist
 YEAR?=$(shell date +"%Y")
@@ -7,11 +7,12 @@ CLI_NAME?=cf
 IMAGE_REPOSITORY?=quay.io
 IMAGE_NAMESPACE?=codefresh
 
-RUNTIME_DEF_URL="https://github.com/codefresh-io/cli-v2/releases/latest/download/runtime.yaml"
+RUNTIME_DEF_URL="https://raw.githubusercontent.com/codefresh-io/csdp-official/stable/csdp/hybrid/basic/runtime.yaml"
 ADD_CLUSTER_DEF_URL="https://github.com/codefresh-io/csdp-official/add-cluster/kustomize"
 FALLBACK_ADD_CLUSTER_DEF_URL="https://github.com/codefresh-io/cli-v2/manifests/add-cluster/kustomize"
 
-DEV_RUNTIME_DEF_URL="manifests/runtime.yaml"
+# when developing, point this to your local clone of csdp-official
+DEV_RUNTIME_DEF_URL="https://raw.githubusercontent.com/codefresh-io/csdp-official/stable/csdp/hybrid/basic/runtime.yaml"
 DEV_ADD_CLUSTER_DEF_URL="https://github.com/codefresh-io/csdp-official/add-cluster/kustomize" # specify dev branch using ?ref=<branch> here if you want to test a change
 
 CLI_SRCS := $(shell find . -name '*.go')

cmd/commands/git-source.go

@@ -27,7 +27,6 @@ import (
 	"github.com/codefresh-io/cli-v2/pkg/store"
 	"github.com/codefresh-io/cli-v2/pkg/util"
 	apu "github.com/codefresh-io/cli-v2/pkg/util/aputil"
-	eventsutil "github.com/codefresh-io/cli-v2/pkg/util/events"
 	routingutil "github.com/codefresh-io/cli-v2/pkg/util/routing"
 	wfutil "github.com/codefresh-io/cli-v2/pkg/util/workflow"
 
@@ -1161,10 +1160,6 @@ func createDemoGitlabSensor() *sensorsv1alpha1.Sensor {
 func createDemoEventSource(name string) *eventsourcev1alpha1.EventSource {
 	tpl := &eventsourcev1alpha1.Template{Container: &corev1.Container{}}
 
-	if store.Get().SetDefaultResources {
-		eventsutil.SetDefaultResourceRequirements(tpl.Container)
-	}
-
 	return &eventsourcev1alpha1.EventSource{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       eventsourcereg.Kind,
@@ -1186,10 +1181,6 @@ func createDemoSensor(name string, triggers []sensorsv1alpha1.Trigger, dependenc
 		ServiceAccountName: store.Get().WorkflowTriggerServiceAccount,
 	}
 
-	if store.Get().SetDefaultResources {
-		eventsutil.SetDefaultResourceRequirements(tpl.Container)
-	}
-
 	return &sensorsv1alpha1.Sensor{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       sensorreg.Kind,

cmd/commands/runtime.go

@@ -768,7 +768,6 @@ func deleteRuntimeFromPlatform(ctx context.Context, opts *RuntimeUninstallOption
 
 func NewRuntimeUpgradeCommand() *cobra.Command {
 	var (
-		versionStr      string
 		finalParameters map[string]string
 		opts            = &RuntimeUpgradeOptions{
 			featuresToInstall: make([]runtime.InstallFeature, 0),
@@ -813,8 +812,12 @@ func NewRuntimeUpgradeCommand() *cobra.Command {
 				"Repository URL":    opts.CloneOpts.Repo,
 			}
 
-			if versionStr != "" {
-				finalParameters["Version"] = versionStr
+			if opts.versionStr != "" {
+				finalParameters["Version"] = opts.versionStr
+			}
+
+			if opts.runtimeDef == "" {
+				opts.runtimeDef = runtime.GetRuntimeDefURL(opts.versionStr)
 			}
 
 			err = validateVersionIfExists(opts.versionStr)
@@ -844,15 +847,15 @@ func NewRuntimeUpgradeCommand() *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringVar(&opts.versionStr, "version", "", "The runtime version to upgrade to, defaults to latest")
+	cmd.Flags().StringVar(&opts.versionStr, "version", "", "The runtime version to upgrade to, defaults to stable")
 	cmd.Flags().StringVar(&opts.SuggestedSharedConfigRepo, "shared-config-repo", "", "URL to the shared configurations repo. (default: <installation-repo> or the existing one for this account)")
 	cmd.Flags().BoolVar(&opts.DisableTelemetry, "disable-telemetry", false, "If true, will disable analytics reporting for the upgrade process")
 	cmd.Flags().BoolVar(&store.Get().SetDefaultResources, "set-default-resources", false, "If true, will set default requests and limits on all of the runtime components")
-	cmd.Flags().StringVar(&opts.runtimeDef, "runtime-def", store.RuntimeDefURL, "Install runtime from a specific manifest")
+	cmd.Flags().StringVar(&opts.runtimeDef, "runtime-def", "", "Install runtime from a specific manifest")
 	cmd.Flags().BoolVar(&opts.SkipIngress, "skip-ingress", false, "Skips the creation of ingress resources")
 	opts.CloneOpts = apu.AddCloneFlags(cmd, &apu.CloneFlagsOptions{CloneForWrite: true})
-
 	util.Die(cmd.Flags().MarkHidden("runtime-def"))
+	util.Die(cmd.Flags().MarkHidden("set-default-resources"))
 	cmd.MarkFlagsMutuallyExclusive("version", "runtime-def")
 
 	return cmd
@@ -870,14 +873,19 @@ func runRuntimeUpgrade(ctx context.Context, opts *RuntimeUpgradeOptions) error {
 		return fmt.Errorf("failed to download runtime definition: %w", err)
 	}
 
-	if newRt.Spec.DefVersion.GreaterThan(store.Get().MaxDefVersion) {
+	if newRt.Spec.DefVersion != nil && newRt.Spec.DefVersion.GreaterThan(store.Get().MaxDefVersion) {
 		err = fmt.Errorf("please upgrade your cli version before upgrading to %s", newRt.Spec.Version)
 	}
 	handleCliStep(reporter.UpgradeStepRunPreCheckEnsureCliVersion, "Checking CLI version", err, true, false)
 	if err != nil {
 		return err
 	}
 
+	err = runtime.CheckRuntimeVersionCompatible(newRt.Spec.RequiredCLIVersion)
+	if err != nil {
+		return err
+	}
+
 	log.G(ctx).Info("Cloning installation repository")
 	r, fs, err := opts.CloneOpts.GetRepo(ctx)
 	handleCliStep(reporter.UpgradeStepGetRepo, "Getting repository", err, true, false)

cmd/commands/runtime_install.go

@@ -222,6 +222,10 @@ func NewRuntimeInstallCommand() *cobra.Command {
 				return util.DecorateErrorWithDocsLink(fmt.Errorf("pre installation error: %w", err), store.Get().RequirementsLink)
 			}
 
+			if installationOpts.runtimeDef == "" {
+				installationOpts.runtimeDef = runtime.GetRuntimeDefURL(installationOpts.versionStr)
+			}
+
 			finalParameters = map[string]string{
 				"Codefresh context":         cfConfig.CurrentContext,
 				"Kube context":              installationOpts.kubeContext,
@@ -253,7 +257,7 @@ func NewRuntimeInstallCommand() *cobra.Command {
 	cmd.Flags().StringVar(&installationOpts.GatewayNamespace, "gateway-namespace", "", "The namespace of the gateway")
 	cmd.Flags().StringVar(&installationOpts.GitIntegrationRegistrationOpts.Token, "personal-git-token", "", "The Personal git token for your user")
 	cmd.Flags().StringVar(&installationOpts.GitIntegrationRegistrationOpts.Username, "personal-git-user", "", "The Personal git user that match the token, required for bitbucket cloud")
-	cmd.Flags().StringVar(&installationOpts.versionStr, "version", "", "The runtime version to install (default: latest)")
+	cmd.Flags().StringVar(&installationOpts.versionStr, "version", "", "The runtime version to install (default: stable)")
 	cmd.Flags().StringVar(&installationOpts.SuggestedSharedConfigRepo, "shared-config-repo", "", "URL to the shared configurations repo. (default: <installation-repo> or the existing one for this account)")
 	cmd.Flags().BoolVar(&installationOpts.InstallDemoResources, "demo-resources", true, "Installs demo resources (default: true)")
 	cmd.Flags().BoolVar(&installationOpts.SkipClusterChecks, "skip-cluster-checks", false, "Skips the cluster's checks")
@@ -268,7 +272,7 @@ func NewRuntimeInstallCommand() *cobra.Command {
 	cmd.Flags().StringToStringVar(&installationOpts.NamespaceLabels, "namespace-labels", nil, "Optional labels that will be set on the namespace resource. (e.g. \"key1=value1,key2=value2\"")
 	cmd.Flags().StringToStringVar(&installationOpts.InternalIngressAnnotation, "internal-ingress-annotation", nil, "Add annotations to the internal ingress")
 	cmd.Flags().StringToStringVar(&installationOpts.ExternalIngressAnnotation, "external-ingress-annotation", nil, "Add annotations to the external ingress")
-	cmd.Flags().StringVar(&installationOpts.runtimeDef, "runtime-def", store.RuntimeDefURL, "Install runtime from a specific manifest")
+	cmd.Flags().StringVar(&installationOpts.runtimeDef, "runtime-def", "", "Install runtime from a specific manifest")
 	cmd.Flags().StringVar(&accessMode, "access-mode", string(platmodel.AccessModeIngress), "The access mode to the cluster, one of: ingress|tunnel")
 	cmd.Flags().StringVar(&installationOpts.TunnelRegisterHost, "tunnel-register-host", "register-tunnels.cf-cd.com", "The host name for registering a new tunnel")
 	cmd.Flags().StringVar(&installationOpts.TunnelDomain, "tunnel-domain", "tunnels.cf-cd.com", "The base domain for the tunnels")
@@ -293,6 +297,7 @@ func NewRuntimeInstallCommand() *cobra.Command {
 	util.Die(cmd.Flags().MarkHidden("tunnel-domain"))
 	util.Die(cmd.Flags().MarkHidden("ips-allow-list"))
 	util.Die(cmd.Flags().MarkHidden("runtime-def"))
+	util.Die(cmd.Flags().MarkHidden("set-default-resources"))
 	cmd.MarkFlagsMutuallyExclusive("runtime-def", "version")
 	cmd.MarkFlagsMutuallyExclusive("runtime-def", "set-default-resources")
 
@@ -1141,8 +1146,15 @@ func preInstallationChecks(ctx context.Context, opts *RuntimeInstallOptions) (*r
 		return nil, fmt.Errorf("failed to download runtime definition: %w", err)
 	}
 
-	if rt.Spec.DefVersion.GreaterThan(store.Get().MaxDefVersion) {
-		err = fmt.Errorf("your cli version is out of date. please upgrade to the latest version before installing")
+	if rt.Spec.DefVersion != nil {
+		if rt.Spec.DefVersion.GreaterThan(store.Get().MaxDefVersion) {
+			err = fmt.Errorf("your cli version is out of date. please upgrade to the latest version before installing")
+		} else if rt.Spec.DefVersion.LessThan(store.Get().MaxDefVersion) {
+			val := store.Get().DefVersionToLastCLIVersion[rt.Spec.DefVersion.String()]
+			err = fmt.Errorf("to install this version, please downgrade your cli to version %s", val)
+		}
+	} else {
+		err = runtime.CheckRuntimeVersionCompatible(rt.Spec.RequiredCLIVersion)
 	}
 
 	handleCliStep(reporter.InstallStepRunPreCheckEnsureCliVersion, "Checking CLI version", err, true, false)
@@ -2115,7 +2127,7 @@ func (opts *RuntimeInstallOptions) shouldInstallIngress() bool {
 }
 
 func (opts *RuntimeInstallOptions) IsCustomInstall() bool {
-	return opts.runtimeDef != store.RuntimeDefURL
+	return opts.runtimeDef != store.RuntimeDefURL && opts.runtimeDef != store.OldRuntimeDefURL
 }
 
 func getRuntimeDef(runtimeDef, versionStr string) string {
@@ -2135,6 +2147,9 @@ func getRuntimeDef(runtimeDef, versionStr string) string {
 		return runtimeDef
 	}
 
-	// specific version means the runtimeDef is the default value in cli-v2 repo
-	return strings.Replace(runtimeDef, "/releases/latest/download", "/releases/download/v"+version.String(), 1)
+	// specific version means the runtimeDef is the default value in cli-v2/csdp-official repo
+	if strings.Contains(runtimeDef, "cli-v2") {
+		return strings.Replace(runtimeDef, "/releases/latest/download", "/releases/download/v"+version.String(), 1)
+	}
+	return runtimeDef + "?ref=v" + version.String()
 }

docs/commands/cli-v2_runtime_install.md

@@ -51,12 +51,11 @@ cli-v2 runtime install [runtime_name] [flags]
       --provider string                              The git provider, one of: azure|bitbucket|bitbucket-server|gitea|github|gitlab
       --provider-api-url string                      Git provider API url
       --repo string                                  Repository URL [GIT_REPO]
-      --set-default-resources                        If true, will set default requests and limits on all of the runtime components
       --shared-config-repo string                    URL to the shared configurations repo. (default: <installation-repo> or the existing one for this account)
       --skip-cluster-checks                          Skips the cluster's checks
       --skip-ingress                                 Skips the creation of ingress resources
   -b, --upsert-branch                                If true will try to checkout the specified branch and create it if it doesn't exist
-      --version string                               The runtime version to install (default: latest)
+      --version string                               The runtime version to install (default: stable)
       --wait-timeout duration                        How long to wait for the runtime components to be ready (default 8m0s)
 ```
 

docs/commands/cli-v2_runtime_upgrade.md

@@ -33,11 +33,10 @@ cli-v2 runtime upgrade [RUNTIME_NAME] [flags]
   -u, --git-user string             Your git provider user name [GIT_USER] (not required in GitHub)
   -h, --help                        help for upgrade
       --repo string                 Repository URL [GIT_REPO]
-      --set-default-resources       If true, will set default requests and limits on all of the runtime components
       --shared-config-repo string   URL to the shared configurations repo. (default: <installation-repo> or the existing one for this account)
       --skip-ingress                Skips the creation of ingress resources
   -b, --upsert-branch               If true will try to checkout the specified branch and create it if it doesn't exist
-      --version string              The runtime version to upgrade to, defaults to latest
+      --version string              The runtime version to upgrade to, defaults to stable
 ```
 
 ### Options inherited from parent commands

docs/releases/release_notes.md

@@ -23,7 +23,7 @@ cf version
 
 ```bash
 # download and extract the binary
-curl -L --output - https://github.com/codefresh-io/cli-v2/releases/download/v0.0.569/cf-linux-amd64.tar.gz | tar zx
+curl -L --output - https://github.com/codefresh-io/cli-v2/releases/download/v0.1.0/cf-linux-amd64.tar.gz | tar zx
 
 # move the binary to your $PATH
 mv ./cf-linux-amd64 /usr/local/bin/cf
@@ -36,7 +36,7 @@ cf version
 
 ```bash
 # download and extract the binary
-curl -L --output - https://github.com/codefresh-io/cli-v2/releases/download/v0.0.569/cf-darwin-amd64.tar.gz | tar zx
+curl -L --output - https://github.com/codefresh-io/cli-v2/releases/download/v0.1.0/cf-darwin-amd64.tar.gz | tar zx
 
 # move the binary to your $PATH
 mv ./cf-darwin-amd64 /usr/local/bin/cf

hack/build.sh

@@ -16,6 +16,5 @@ go build -ldflags=" \
     -X 'github.com/codefresh-io/cli-v2/pkg/store.RuntimeDefURL=${RUNTIME_DEF_URL}' \
     -X 'github.com/codefresh-io/cli-v2/pkg/store.AddClusterDefURL=${ADD_CLUSTER_DEF_URL}' \
     -X 'github.com/codefresh-io/cli-v2/pkg/store.FallbackAddClusterDefURL=${FALLBACK_ADD_CLUSTER_DEF_URL}' \
-    -X 'github.com/codefresh-io/cli-v2/pkg/store.SegmentWriteKey=${SEGMENT_WRITE_KEY}' \
-    -X 'github.com/codefresh-io/cli-v2/pkg/store.devMode=${DEV_MODE}'" \
+    -X 'github.com/codefresh-io/cli-v2/pkg/store.SegmentWriteKey=${SEGMENT_WRITE_KEY}'" \
     -v -o ${OUT_FILE} ${MAIN}

hack/get-manifests-location.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+REPO="https://github.com/codefresh-io/csdp-official"
+BRANCH="$1"
+
+DEFAULT_MANIFESTS_LOCATION="https://raw.githubusercontent.com/codefresh-io/csdp-official/stable/csdp/hybrid/basic/runtime.yaml"
+CUSTOM_MANIFESTS_LOCATION="https://github.com/codefresh-io/csdp-official/csdp/hybrid/basic/runtime.yaml?ref=$BRANCH"
+
+git ls-remote --heads ${REPO} ${BRANCH} | grep ${BRANCH} >/dev/null
+
+if [ "$?" == "1" ]; then
+	# No matching branch was found in csdp-official
+	echo "$DEFAULT_MANIFESTS_LOCATION"
+	exit 0
+fi
+
+echo "$CUSTOM_MANIFESTS_LOCATION"