Merge pull request #294 from cloudogu/feature/content-repo-mirror

Content Repos: MIRROR + createJenkinsJob

Author: schnatterer

docs/configuration.schema.json

@@ -164,18 +164,18 @@
           "items" : {
             "type" : "object",
             "properties" : {
-              "folderBased" : {
+              "createJenkinsJob" : {
                 "type" : [ "boolean", "null" ],
-                "description" : "When true, interpret the folder structure of each repo as repos. That is, root folder becomes namespace in SCM, sub folders become repository names in SCM"
+                "description" : "If true, creates a Jenkins job, if jenkinsfile exists in one of the content repo's branches."
               },
-              "overrideMode" : {
+              "overwriteMode" : {
                 "anyOf" : [ {
                   "type" : "null"
                 }, {
                   "type" : "string",
                   "enum" : [ "INIT", "RESET", "UPGRADE" ]
                 } ],
-                "description" : "This defines, how customer repos will be updated.\nINIT - push only if repo does not exist.\nRESET - delete all files after cloning source - files not in content are deleted\nUPGRADE - clone and copy - existing files will be overwritten, files not in content are kept"
+                "description" : "This defines, how customer repos will be updated.\nINIT - push only if repo does not exist.\nRESET - delete all files after cloning source - files not in content are deleted\nUPGRADE - clone and copy - existing files will be overwritten, files not in content are kept. For type: MIRROR reset and upgrade have same result: in both cases source repo will be force pushed to target repo."
               },
               "password" : {
                 "type" : [ "string", "null" ],
@@ -187,19 +187,32 @@
               },
               "ref" : {
                 "type" : [ "string", "null" ],
-                "description" : "Reference for a specific branch, tag, or commit. Emtpy defaults to default branch of the repo"
+                "description" : "Reference for a specific branch, tag, or commit. Emtpy defaults to default branch of the repo. With type MIRROR: ref must not be a commit hash; Choosing a ref only mirrors the ref but does not delete other branches/tags!"
               },
               "target" : {
                 "type" : [ "string", "null" ],
-                "description" : "Target path for the repository"
+                "description" : "Target repo for the repository in the for of namespace/name. Must contain one slash to separate namespace from name."
+              },
+              "targetRef" : {
+                "type" : [ "string", "null" ],
+                "description" : "Reference for a specific branch or tag in the target repo of a MIRROR or COPY repo. If ref is a tag, targetRef is treated as tag as well. Except: targetRef is full ref like refs/heads/my-branch or refs/tags/my-tag. Empty defaults to the source ref."
               },
               "templating" : {
                 "type" : [ "boolean", "null" ],
                 "description" : "When true, template all files ending in .ftl within the repo"
               },
+              "type" : {
+                "anyOf" : [ {
+                  "type" : "null"
+                }, {
+                  "type" : "string",
+                  "enum" : [ "FOLDER_BASED", "COPY", "MIRROR" ]
+                } ],
+                "description" : "Content Repos can either be:\ncopied (only the files, starting on ref, starting at path within the repo. Requires target)\n, mirrored (FORCE pushes ref or the whole git repo if no ref set). Requires target, does not allow path and template.)\nfolderBased (folder structure is interpreted as repos. That is, root folder becomes namespace in SCM, sub folders become repository names in SCM, files are copied. Requires target.)"
+              },
               "url" : {
                 "type" : [ "string", "null" ],
-                "description" : "URL of the content repo"
+                "description" : "URL of the content repo. Mandatory for each type."
               },
               "username" : {
                 "type" : [ "string", "null" ],

src/main/groovy/com/cloudogu/gitops/Application.groovy

@@ -47,6 +47,7 @@ class Application {
                     statics             : new DefaultObjectWrapperBuilder(Configuration.VERSION_2_3_32).build().getStaticModels()
             ]))
         }
+        config.content.namespaces = namespaces.toList()
 
         //iterates over all FeatureWithImages and gets their namespaces
         namespaces.addAll(this.features

src/main/groovy/com/cloudogu/gitops/cli/GitopsPlaygroundCliMainScripted.groovy

@@ -85,21 +85,23 @@ class GitopsPlaygroundCliMainScripted {
 
                 def airGappedUtils = new AirGappedUtils(config, scmmRepoProvider, scmmApiClient, fileSystemUtils, helmClient)
                 def networkingUtils = new NetworkingUtils()
-                
+
+                def jenkins = new Jenkins(config, executor, fileSystemUtils, new GlobalPropertyManager(jenkinsApiClient),
+                        new JobManager(jenkinsApiClient), new UserManager(jenkinsApiClient),
+                        new PrometheusConfigurator(jenkinsApiClient), helmStrategy, k8sClient, networkingUtils)
+
                 context.registerSingleton(new Application(config, [
                         new Registry(config, fileSystemUtils, k8sClient, helmStrategy),
                         new ScmManager(config, executor, fileSystemUtils, helmStrategy, k8sClient, networkingUtils),
-                        new Jenkins(config, executor, fileSystemUtils, new GlobalPropertyManager(jenkinsApiClient),
-                                new JobManager(jenkinsApiClient), new UserManager(jenkinsApiClient),
-                                new PrometheusConfigurator(jenkinsApiClient), helmStrategy, k8sClient, networkingUtils),
+                        jenkins,
                         new ArgoCD(config, k8sClient, helmClient, fileSystemUtils, scmmRepoProvider),
                         new IngressNginx(config, fileSystemUtils, deployer, k8sClient, airGappedUtils),
                         new CertManager(config, fileSystemUtils, deployer, k8sClient, airGappedUtils),
                         new Mailhog(config, fileSystemUtils, deployer, k8sClient, airGappedUtils),
                         new PrometheusStack(config, fileSystemUtils, deployer, k8sClient, airGappedUtils, scmmRepoProvider),
                         new ExternalSecretsOperator(config, fileSystemUtils, deployer, k8sClient, airGappedUtils),
                         new Vault(config, fileSystemUtils, k8sClient, deployer, airGappedUtils),
-                        new Content(config, k8sClient, scmmRepoProvider, scmmApiClient),
+                        new Content(config, k8sClient, scmmRepoProvider, scmmApiClient, jenkins),
                 ]))
             }
         }

src/main/groovy/com/cloudogu/gitops/config/ApplicationConfigurator.groovy

@@ -3,6 +3,9 @@ package com.cloudogu.gitops.config
 import com.cloudogu.gitops.utils.FileSystemUtils
 import groovy.util.logging.Slf4j
 
+import static com.cloudogu.gitops.config.Config.ContentRepoType
+import static com.cloudogu.gitops.config.Config.ContentSchema.ContentRepositorySchema
+
 @Slf4j
 class ApplicationConfigurator {
 
@@ -268,18 +271,41 @@ class ApplicationConfigurator {
 
     static void validateContent(Config config) {
         config.content.repos.each { repo ->
+            
             if (!repo.url) {
                 throw new RuntimeException("content.repos requires a url parameter.")
             }
-            if (!repo.folderBased && !repo.target) {
-                throw new RuntimeException("content.repos.folderBased: false requires folder content.repos.target to be set. ${repo.url}")
-            }
             if (repo.target) {
                 if (repo.target.count('/') == 0) {
-                    throw new RuntimeException("content.target needs / to separate namespace/group from repo name. ${repo.url}")
+                    throw new RuntimeException("content.target needs / to separate namespace/group from repo name. Repo: ${repo.url}")
                 }
+            }
 
-
+            switch (repo.type) {
+                case ContentRepoType.COPY:
+                    if (!repo.target) {
+                        throw new RuntimeException("content.repos.type ${ContentRepoType.COPY} requires content.repos.target to be set. Repo: ${repo.url}")
+                    }
+                    break
+                case ContentRepoType.FOLDER_BASED:
+                    if (repo.target) {
+                        throw new RuntimeException("content.repos.type ${ContentRepoType.FOLDER_BASED} does not support target parameter. Repo: ${repo.url}")
+                    }
+                    if (repo.targetRef) {
+                        throw new RuntimeException("content.repos.type ${ContentRepoType.FOLDER_BASED} does not support targetRef parameter. Repo: ${repo.url}")
+                    }
+                    break
+                case ContentRepoType.MIRROR:
+                    if (!repo.target) {
+                        throw new RuntimeException("content.repos.type ${ContentRepoType.MIRROR} requires content.repos.target to be set. Repo: ${repo.url}")
+                    }
+                    if (repo.path != ContentRepositorySchema.DEFAULT_PATH) {
+                        throw new RuntimeException("content.repos.type ${ContentRepoType.MIRROR} does not support path. Current path: ${repo.path}. Repo: ${repo.url}")
+                    }
+                    if (repo.templating) {
+                        throw new RuntimeException("content.repos.type ${ContentRepoType.MIRROR} does not support templating. Repo: ${repo.url}")
+                    }
+                    break
             }
         }
     }

src/main/groovy/com/cloudogu/gitops/config/Config.groovy

@@ -102,14 +102,23 @@ class Config {
         Map<String, Object> variables = [:]
 
         static class ContentRepositorySchema {
+            static final String DEFAULT_PATH = '.'
+            // This is controversial. Forcing users to explicitly choose a type requires them to understand the concept
+            // of types. What would be a good default? The simplest use case ist MIRROR from url to target.
+            // COPY and FOLDER_BASED are more advanced use cases. So we choose MIRROR as the default.
+            static final ContentRepoType DEFAULT_TYPE = ContentRepoType.MIRROR
+
             @JsonPropertyDescription(CONTENT_REPO_URL_DESCRIPTION)
             String url = ''
 
             @JsonPropertyDescription(CONTENT_REPO_PATH_DESCRIPTION)
-            String path = '.'
+            String path = DEFAULT_PATH
 
             @JsonPropertyDescription(CONTENT_REPO_REF_DESCRIPTION)
             String ref = ''
+            
+            @JsonPropertyDescription(CONTENT_REPO_TARGET_REF_DESCRIPTION)
+            String targetRef = ''
 
             @JsonPropertyDescription(CONTENT_REPO_USERNAME_DESCRIPTION)
             String username = ''
@@ -120,14 +129,17 @@ class Config {
             @JsonPropertyDescription(CONTENT_REPO_TEMPLATING_DESCRIPTION)
             Boolean templating = false
 
-            @JsonPropertyDescription(CONTENT_REPO_FOLDER_BASED_REPOS_DESCRIPTION)
-            Boolean folderBased = false
+            @JsonPropertyDescription(CONTENT_REPO_TYPE_DESCRIPTION)
+            ContentRepoType type = DEFAULT_TYPE
 
             @JsonPropertyDescription(CONTENT_REPO_TARGET_DESCRIPTION)
             String target = ''
 
-            @JsonPropertyDescription(CONTENT_REPO_TARGET_OVERRIDE_MODE)
-            OverrideMode overrideMode = OverrideMode.INIT // Defensively use init to not override existing files by default
+            @JsonPropertyDescription(CONTENT_REPO_TARGET_OVERWRITE_MODE_DESCRIPTION)
+            OverwriteMode overwriteMode = OverwriteMode.INIT // Defensively use init to not override existing files by default
+
+            @JsonPropertyDescription(CONTENT_REPO_CREATE_JENKINS_JOB_DESCRIPTION)
+            Boolean createJenkinsJob = false
         }
     }
 
@@ -784,16 +796,19 @@ class Config {
         }
     }
 
+    static enum ContentRepoType {
+        FOLDER_BASED, COPY, MIRROR
+    }
 
     static enum VaultMode {
         dev, prod
     }
 
     /**
      * This defines, how customer repos will be updated.
-     * See {@link ConfigConstants#CONTENT_REPO_TARGET_OVERRIDE_MODE}
+     * See {@link ConfigConstants#CONTENT_REPO_TARGET_OVERWRITE_MODE_DESCRIPTION}
      */
-    static enum OverrideMode {
+    static enum OverwriteMode {
         INIT, RESET, UPGRADE
     }
 

src/main/groovy/com/cloudogu/gitops/config/ConfigConstants.groovy

@@ -5,7 +5,7 @@ interface ConfigConstants {
     public static final String BINARY_NAME = 'apply-ng'
     public static final String APP_NAME = 'gitops-playground (GOP)'
     public static final String APP_DESCRIPTION = 'CLI-tool to deploy gitops-playground.'
-    
+
     // group registry
     String REGISTRY_ENABLE_DESCRIPTION = 'Installs a simple cluster-local registry for demonstration purposes. Warning: Registry does not provide authentication!'
     String REGISTRY_DESCRIPTION = 'Config parameters for Registry'
@@ -18,7 +18,7 @@ interface ConfigConstants {
     String REGISTRY_PROXY_URL_DESCRIPTION = 'The url of your proxy-registry. Used in pipelines to authorize pull base images. Use in conjunction with petclinic base image. Used in helm charts when create-image-pull-secrets is set. Use in conjunction with helm.*image fields.'
     String REGISTRY_PROXY_USERNAME_DESCRIPTION = 'Use with registry-proxy-url, added to Jenkins as credentials and created as pull secrets, when create-image-pull-secrets is set.'
     String REGISTRY_PROXY_PASSWORD_DESCRIPTION = 'Use with registry-proxy-url, added to Jenkins as credentials and created as pull secrets, when create-image-pull-secrets is set.'
-    
+
     String REGISTRY_USERNAME_RO_DESCRIPTION = 'Optional alternative username for registry-url with read-only permissions that is used when create-image-pull-secrets is set.'
     String REGISTRY_PASSWORD_RO_DESCRIPTION = 'Optional alternative password for registry-url with read-only permissions that is used when create-image-pull-secrets is set.'
     String REGISTRY_CREATE_IMAGE_PULL_SECRETS_DESCRIPTION = 'Create image pull secrets for registry and proxy-registry for all GOP namespaces and helm charts. Uses proxy-username, read-only-username or registry-username (in this order).  Use this if your cluster is not auto-provisioned with credentials for your private registries or if you configure individual helm images to be pulled from the proxy-registry that requires authentication.'
@@ -31,15 +31,17 @@ interface ConfigConstants {
     String CONTENT_EXAMPLES_DESCRIPTION = 'Deploy example content: source repos, GitOps repos, Jenkins Job, Argo CD apps/project'
     String CONTENT_NAMESPACES_DESCRIPTION = 'Additional kubernetes namespaces. These are authorized to Argo CD, supplied with image pull secrets, monitored by prometheus, etc. Namespaces can be templates, e.g. ${config.application.namePrefix}staging'
     String CONTENT_REPO_DESCRIPTION = "Content repos to push into target environment"
-    String CONTENT_REPO_URL_DESCRIPTION = "URL of the content repo"
+    String CONTENT_REPO_URL_DESCRIPTION = "URL of the content repo. Mandatory for each type."
     String CONTENT_REPO_PATH_DESCRIPTION = "Path within the content repo to process"
-    String CONTENT_REPO_REF_DESCRIPTION = "Reference for a specific branch, tag, or commit. Emtpy defaults to default branch of the repo"
+    String CONTENT_REPO_REF_DESCRIPTION = "Reference for a specific branch, tag, or commit. Emtpy defaults to default branch of the repo. With type MIRROR: ref must not be a commit hash; Choosing a ref only mirrors the ref but does not delete other branches/tags!"
+    String CONTENT_REPO_TARGET_REF_DESCRIPTION = "Reference for a specific branch or tag in the target repo of a MIRROR or COPY repo. If ref is a tag, targetRef is treated as tag as well. Except: targetRef is full ref like refs/heads/my-branch or refs/tags/my-tag. Empty defaults to the source ref."
     String CONTENT_REPO_USERNAME_DESCRIPTION = "Username to authenticate against content repo"
     String CONTENT_REPO_PASSWORD_DESCRIPTION = "Password to authenticate against content repo"
     String CONTENT_REPO_TEMPLATING_DESCRIPTION = "When true, template all files ending in .ftl within the repo"
-    String CONTENT_REPO_FOLDER_BASED_REPOS_DESCRIPTION = "When true, interpret the folder structure of each repo as repos. That is, root folder becomes namespace in SCM, sub folders become repository names in SCM"
-    String CONTENT_REPO_TARGET_DESCRIPTION = "Target path for the repository"
-    String CONTENT_REPO_TARGET_OVERRIDE_MODE = "This defines, how customer repos will be updated.\nINIT - push only if repo does not exist.\nRESET - delete all files after cloning source - files not in content are deleted\nUPGRADE - clone and copy - existing files will be overwritten, files not in content are kept"
+    String CONTENT_REPO_TYPE_DESCRIPTION = "Content Repos can either be:\ncopied (only the files, starting on ref, starting at path within the repo. Requires target)\n, mirrored (FORCE pushes ref or the whole git repo if no ref set). Requires target, does not allow path and template.)\nfolderBased (folder structure is interpreted as repos. That is, root folder becomes namespace in SCM, sub folders become repository names in SCM, files are copied. Requires target.)"
+    String CONTENT_REPO_TARGET_DESCRIPTION = "Target repo for the repository in the for of namespace/name. Must contain one slash to separate namespace from name."
+    String CONTENT_REPO_TARGET_OVERWRITE_MODE_DESCRIPTION = "This defines, how customer repos will be updated.\nINIT - push only if repo does not exist.\nRESET - delete all files after cloning source - files not in content are deleted\nUPGRADE - clone and copy - existing files will be overwritten, files not in content are kept. For type: MIRROR reset and upgrade have same result: in both cases source repo will be force pushed to target repo."
+    String CONTENT_REPO_CREATE_JENKINS_JOB_DESCRIPTION = "If true, creates a Jenkins job, if jenkinsfile exists in one of the content repo's branches."
     String CONTENT_VARIABLES_DESCRIPTION = "Additional variables to use in custom templates."
 
     // group jenkins