Merge pull request #281 from cloudogu/feature/contentRepos

Implement MVP for content repos

Author: ThomasMichael1811

Dockerfile

@@ -13,7 +13,7 @@ WORKDIR /app
 COPY .mvn/ /app/.mvn/
 COPY mvnw /app/
 COPY pom.xml /app/
-RUN ./mvnw dependency:resolve-plugins dependency:go-offline -B 
+RUN ./mvnw dependency:resolve-plugins dependency:go-offline -B
 
 FROM graal AS maven-build
 ENV MAVEN_OPTS='-Dmaven.repo.local=/mvn'
@@ -25,7 +25,7 @@ COPY compiler.groovy /app
 COPY .git /app/.git
 
 WORKDIR /app
-# Exclude code not needed in productive image 
+# Exclude code not needed in productive image
 RUN cd /app/src/main/groovy/com/cloudogu/gitops/cli/ \
     && rm GenerateJsonSchema.groovy \
     && rm GitopsPlaygroundCliMainScripted.groovy
@@ -37,7 +37,7 @@ RUN mv $(ls -S target/*.jar | head -n 1) /app/gitops-playground.jar
 
 FROM alpine AS downloader
 RUN apk add curl grep
-# When updating, 
+# When updating,
 # * also update the checksum found at https://dl.k8s.io/release/v${K8S_VERSION}/bin/linux/amd64/kubectl.sha256
 # * also update in init-cluster.sh. vars.tf, Config.groovy and apply.sh
 # When upgrading to 1.26 we can verify the kubectl signature with cosign!
@@ -61,11 +61,11 @@ ENV HOME=/tmp
 WORKDIR /tmp
 
 # Helm
-RUN curl --location --fail --retry 20 --retry-connrefused --retry-all-errors --output helm.tar.gz https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz 
+RUN curl --location --fail --retry 20 --retry-connrefused --retry-all-errors --output helm.tar.gz https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz
 RUN curl --location --fail --retry 20 --retry-connrefused --retry-all-errors --output helm.tar.gz.asc https://github.com/helm/helm/releases/download/v${HELM_VERSION}/helm-v${HELM_VERSION}-linux-amd64.tar.gz.asc
 RUN tar -xf helm.tar.gz
 RUN set -o pipefail && curl --location --fail --retry 20 --retry-connrefused --retry-all-errors \
-  https://raw.githubusercontent.com/helm/helm/main/KEYS | gpg --import --batch --no-default-keyring --keyring /tmp/keyring.gpg 
+  https://raw.githubusercontent.com/helm/helm/main/KEYS | gpg --import --batch --no-default-keyring --keyring /tmp/keyring.gpg
 RUN gpgv --keyring /tmp/keyring.gpg helm.tar.gz.asc helm.tar.gz
 RUN mv linux-amd64/helm /dist/usr/local/bin
 ENV PATH=$PATH:/dist/usr/local/bin
@@ -79,7 +79,7 @@ RUN mv /tmp/kubectl /dist/usr/local/bin/kubectl
 
 # External Repos used in GOP
 WORKDIR /dist/gitops/repos
-RUN git clone --bare https://github.com/cloudogu/spring-petclinic.git 
+RUN git clone --bare https://github.com/cloudogu/spring-petclinic.git
 RUN git clone --bare https://github.com/cloudogu/spring-boot-helm-chart.git
 RUN git clone --bare https://github.com/cloudogu/gitops-build-lib.git
 RUN git clone --bare https://github.com/cloudogu/ces-build-lib.git
@@ -118,7 +118,7 @@ RUN mv /dist/app/src /dist-dev/src && \
     chmod a=rwx -R /dist-dev/src && \
     rm -r /dist-dev/src/main/groovy/com/cloudogu/gitops/graal
 COPY --from=maven-build /app/gitops-playground.jar /dist-dev/gitops-playground.jar
-# Remove compiled GOP code from jar to avoid duplicate in dev image, allowing for scripting. 
+# Remove compiled GOP code from jar to avoid duplicate in dev image, allowing for scripting.
 # Keep generated class Version, to avoid ClassNotFoundException.
 RUN zip -d /dist-dev/gitops-playground.jar 'com/cloudogu/gitops/*' -x com/cloudogu/gitops/cli/Version.class
 
@@ -132,7 +132,7 @@ FROM graal AS native-image
 ENV MAVEN_OPTS='-Dmaven.repo.local=/mvn'
 RUN microdnf install gnupg
 
-# Provide binaries used by apply-ng, so our runs with native-image-agent dont fail 
+# Provide binaries used by apply-ng, so our runs with native-image-agent dont fail
 # with "java.io.IOException: Cannot run program "kubectl"..." etc.
 RUN microdnf install iproute
 

docs/configuration.schema.json

@@ -150,6 +150,64 @@
         "examples" : {
           "type" : [ "boolean", "null" ],
           "description" : "Deploy example content: source repos, GitOps repos, Jenkins Job, Argo CD apps/project"
+        },
+        "namespaces" : {
+          "description" : "Additional kubernetes namespaces. These are authorized to Argo CD, supplied with image pull secrets, monitored by prometheus, etc. Namespaces can be templates, e.g. ${config.application.namePrefix}staging",
+          "type" : [ "array", "null" ],
+          "items" : {
+            "type" : "string"
+          }
+        },
+        "repos" : {
+          "description" : "Content repos to push into target environment",
+          "type" : [ "array", "null" ],
+          "items" : {
+            "type" : "object",
+            "properties" : {
+              "folderBased" : {
+                "type" : [ "boolean", "null" ],
+                "description" : "When true, interpret the folder structure of each repo as repos. That is, root folder becomes namespace in SCM, sub folders become repository names in SCM"
+              },
+              "overrideMode" : {
+                "anyOf" : [ {
+                  "type" : "null"
+                }, {
+                  "type" : "string",
+                  "enum" : [ "INIT", "RESET", "UPGRADE" ]
+                } ],
+                "description" : "This defines, how customer repos will be updated.\nINIT - push only if repo does not exist (SCMM Return value 409).\nRESET - delete alle files after cloning source - new files are deleted\nUPGRADE - clone and copy - existing files will be overwritten, new files are kept"
+              },
+              "password" : {
+                "type" : [ "string", "null" ],
+                "description" : "Password to authenticate against content repo"
+              },
+              "path" : {
+                "type" : [ "string", "null" ],
+                "description" : "Path within the content repo to process"
+              },
+              "ref" : {
+                "type" : [ "string", "null" ],
+                "description" : "Reference for a specific branch, tag, or commit"
+              },
+              "target" : {
+                "type" : [ "string", "null" ],
+                "description" : "Target path for the repository"
+              },
+              "templating" : {
+                "type" : [ "boolean", "null" ],
+                "description" : "When true, template all files ending in .ftl within the repo"
+              },
+              "url" : {
+                "type" : [ "string", "null" ],
+                "description" : "URL of the content repo"
+              },
+              "username" : {
+                "type" : [ "string", "null" ],
+                "description" : "Username to authenticate against content repo"
+              }
+            },
+            "additionalProperties" : false
+          }
         }
       },
       "additionalProperties" : false,

docs/developers.md

@@ -13,7 +13,7 @@ The versions are also specified in the `Config.groovy` file, so it is recommende
 
 ## Table of contents
 
-<!-- Update with ` doctoc --notitle docs/developers.md --maxlevel 4      `. See https://github.com/thlorenz/doctoc -->
+<!-- Update with `doctoc --notitle docs/developers.md --maxlevel 4`. See https://github.com/thlorenz/doctoc -->
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 

src/main/groovy/com/cloudogu/gitops/Application.groovy

@@ -1,6 +1,9 @@
 package com.cloudogu.gitops
 
 import com.cloudogu.gitops.config.Config
+import com.cloudogu.gitops.utils.TemplatingEngine
+import freemarker.template.Configuration
+import freemarker.template.DefaultObjectWrapperBuilder
 import groovy.util.logging.Slf4j
 import jakarta.inject.Singleton
 
@@ -35,13 +38,14 @@ class Application {
 
     void setNamespaceListToConfig(Config config) {
         Set<String> namespaces = new HashSet<>()
-        String namePrefix = config.application.namePrefix
-
-        if (config.content.examples) {
-            namespaces.addAll(Arrays.asList(
-                    namePrefix + "example-apps-staging",
-                    namePrefix + "example-apps-production"
-            ))
+        def engine = new TemplatingEngine()
+
+        config.content.namespaces.each { String ns ->
+            namespaces.add(engine.template(ns, [
+                    config              : config,
+                    // Allow for using static classes inside the templates
+                    statics             : new DefaultObjectWrapperBuilder(Configuration.VERSION_2_3_32).build().getStaticModels()
+            ]))
         }
 
         //iterates over all FeatureWithImages and gets their namespaces

src/main/groovy/com/cloudogu/gitops/cli/GitopsPlaygroundCliMainScripted.groovy

@@ -92,7 +92,7 @@ class GitopsPlaygroundCliMainScripted {
                         new Jenkins(config, executor, fileSystemUtils, new GlobalPropertyManager(jenkinsApiClient),
                                 new JobManager(jenkinsApiClient), new UserManager(jenkinsApiClient),
                                 new PrometheusConfigurator(jenkinsApiClient), helmStrategy, k8sClient, networkingUtils),
-                        new Content(config, k8sClient),
+                        new Content(config, k8sClient, scmmRepoProvider, scmmApiClient),
                         new ArgoCD(config, k8sClient, helmClient, fileSystemUtils, scmmRepoProvider),
                         new IngressNginx(config, fileSystemUtils, deployer, k8sClient, airGappedUtils),
                         new CertManager(config, fileSystemUtils, deployer, k8sClient, airGappedUtils),

src/main/groovy/com/cloudogu/gitops/config/ApplicationConfigurator.groovy

@@ -52,9 +52,12 @@ class ApplicationConfigurator {
         if (newConfig.features.ingressNginx.active && !newConfig.application.baseUrl) {
             log.warn("Ingress-controller is activated without baseUrl parameter. Services will not be accessible by hostnames. To avoid this use baseUrl with ingress. ")
         }
-        
-        if (newConfig.content.examples && !newConfig.registry.active) {
-            throw new RuntimeException("content.examples requires either registry.active or registry.url")
+        if (newConfig.content.examples) {
+            if  (!newConfig.registry.active) {
+                throw new RuntimeException("content.examples requires either registry.active or registry.url")
+            }
+            String prefix = newConfig.application.namePrefix
+            newConfig.content.namespaces += [ prefix+"example-apps-staging", prefix+"example-apps-production"]
         }
     }
 
@@ -251,6 +254,7 @@ class ApplicationConfigurator {
     void validateConfig(Config configToSet) {
         validateScmmAndJenkinsAreBothSet(configToSet)
         validateMirrorReposHelmChartFolderSet(configToSet)
+        validateContent(configToSet)
     }
 
     private void validateMirrorReposHelmChartFolderSet(Config configToSet) {
@@ -261,6 +265,17 @@ class ApplicationConfigurator {
                     "LOCAL_HELM_CHART_FOLDER='charts' after running 'scripts/downloadHelmCharts.sh' from the repo")
         }
     }
+    
+    static void validateContent(Config config) {
+        config.content.repos.each { repo ->
+            if (!repo.url) {
+                throw new RuntimeException('content.repos requires a url parameter')
+            }
+            if (!repo.folderBased && !repo.target) {
+                throw new RuntimeException('content.repos.folderBased: false requires folder content.repos.target to be set')
+            }
+        }
+    }
 
     private void validateScmmAndJenkinsAreBothSet(Config configToSet) {
         if (configToSet.jenkins.active && 

src/main/groovy/com/cloudogu/gitops/config/Config.groovy

@@ -85,13 +85,50 @@ class Config {
     @JsonPropertyDescription(CONTENT_DESCRIPTION)
     @Mixin
     ContentSchema content = new ContentSchema()
-
+    
     class ContentSchema {
+
         @Option(names = ['--content-examples'], description = CONTENT_EXAMPLES_DESCRIPTION)
         @JsonPropertyDescription(CONTENT_EXAMPLES_DESCRIPTION)
         Boolean examples = false
+        
+        @JsonPropertyDescription(CONTENT_NAMESPACES_DESCRIPTION)
+        List<String> namespaces = []
+
+        @JsonPropertyDescription(CONTENT_REPO_DESCRIPTION)
+        List<ContentRepositorySchema> repos = []
+
+        static class ContentRepositorySchema {
+            @JsonPropertyDescription(CONTENT_REPO_URL_DESCRIPTION)
+            String url = ''
+
+            @JsonPropertyDescription(CONTENT_REPO_PATH_DESCRIPTION)
+            String path = '.'
+
+            @JsonPropertyDescription(CONTENT_REPO_REF_DESCRIPTION)
+            String ref = 'main'
+
+            @JsonPropertyDescription(CONTENT_REPO_USERNAME_DESCRIPTION)
+            String username = ''
+
+            @JsonPropertyDescription(CONTENT_REPO_PASSWORD_DESCRIPTION)
+            String password = ''
+
+            @JsonPropertyDescription(CONTENT_REPO_TEMPLATING_DESCRIPTION)
+            Boolean templating = false
+
+            @JsonPropertyDescription(CONTENT_REPO_FOLDER_BASED_REPOS_DESCRIPTION)
+            Boolean folderBased = false
+
+            @JsonPropertyDescription(CONTENT_REPO_TARGET_DESCRIPTION)
+            String target = ''
+
+            @JsonPropertyDescription(CONTENT_REPO_TARGET_OVERRIDE_MODE)
+            OverrideMode overrideMode = OverrideMode.INIT // default is init a new repository
+        }
     }
 
+
     static class HelmConfig {
         @JsonPropertyDescription(HELM_CONFIG_CHART_DESCRIPTION)
         String chart = ''
@@ -749,6 +786,14 @@ class Config {
         dev, prod
     }
 
+    /**
+     * This defines, how customer repos will be updated.
+     */
+    static enum OverrideMode {
+        INIT, RESET, UPGRADE
+    }
+
+
     private static final ObjectMapper objectMapper = new ObjectMapper()
             .registerModule(new SimpleModule().addSerializer(GString, new JsonSerializer<GString>() {
                 @Override

src/main/groovy/com/cloudogu/gitops/config/ConfigConstants.groovy

@@ -29,6 +29,17 @@ interface ConfigConstants {
 
     // Content
     String CONTENT_EXAMPLES_DESCRIPTION = 'Deploy example content: source repos, GitOps repos, Jenkins Job, Argo CD apps/project'
+    String CONTENT_NAMESPACES_DESCRIPTION = 'Additional kubernetes namespaces. These are authorized to Argo CD, supplied with image pull secrets, monitored by prometheus, etc. Namespaces can be templates, e.g. ${config.application.namePrefix}staging'
+    String CONTENT_REPO_DESCRIPTION = "Content repos to push into target environment"
+    String CONTENT_REPO_URL_DESCRIPTION = "URL of the content repo"
+    String CONTENT_REPO_PATH_DESCRIPTION = "Path within the content repo to process"
+    String CONTENT_REPO_REF_DESCRIPTION = "Reference for a specific branch, tag, or commit"
+    String CONTENT_REPO_USERNAME_DESCRIPTION = "Username to authenticate against content repo"
+    String CONTENT_REPO_PASSWORD_DESCRIPTION = "Password to authenticate against content repo"
+    String CONTENT_REPO_TEMPLATING_DESCRIPTION = "When true, template all files ending in .ftl within the repo"
+    String CONTENT_REPO_FOLDER_BASED_REPOS_DESCRIPTION = "When true, interpret the folder structure of each repo as repos. That is, root folder becomes namespace in SCM, sub folders become repository names in SCM"
+    String CONTENT_REPO_TARGET_DESCRIPTION = "Target path for the repository"
+    String CONTENT_REPO_TARGET_OVERRIDE_MODE = "This defines, how customer repos will be updated.\nINIT - push only if repo does not exist (SCMM Return value 409).\nRESET - delete alle files after cloning source - new files are deleted\nUPGRADE - clone and copy - existing files will be overwritten, new files are kept"
 
     // group jenkins
     String JENKINS_ENABLE_DESCRIPTION = 'Installs Jenkins as CI server'