Merge pull request #350 from arangodb/feature/advanced-syncer-tests

ArangoSync tests

Author: neunhoef

Makefile

@@ -104,6 +104,22 @@ endif
 SOURCES := $(shell find $(SRCDIR) -name '*.go' -not -path './test/*')
 DASHBOARDSOURCES := $(shell find $(DASHBOARDDIR)/src -name '*.js' -not -path './test/*') $(DASHBOARDDIR)/package.json
 
+ifndef ARANGOSYNCSRCDIR
+	ARANGOSYNCSRCDIR := $(SCRIPTDIR)/arangosync
+endif
+DOCKERARANGOSYNCCTRLFILE=tests/sync/Dockerfile
+ifndef ARANGOSYNCTESTCTRLIMAGE
+	ARANGOSYNCTESTCTRLIMAGE := $(DOCKERNAMESPACE)/kube-arangodb-sync-test-ctrl$(IMAGESUFFIX)
+endif
+ifndef ARANGOSYNCTESTIMAGE
+	ARANGOSYNCTESTIMAGE := $(DOCKERNAMESPACE)/kube-arangodb-sync-test$(IMAGESUFFIX)
+endif
+ifndef ARANGOSYNCIMAGE
+	ARANGOSYNCIMAGE := $(DOCKERNAMESPACE)/kube-arangodb-sync$(IMAGESUFFIX)
+endif
+ARANGOSYNCTESTCTRLBINNAME := $(PROJECT)_sync_test_ctrl
+ARANGOSYNCTESTCTRLBIN := $(BINDIR)/$(ARANGOSYNCTESTCTRLBINNAME)
+
 .PHONY: all
 all: verify-generated build
 
@@ -298,6 +314,23 @@ docker-test: $(TESTBIN)
 run-upgrade-tests:
 	TESTOPTIONS="-test.run=TestUpgrade" make run-tests
 
+.PHONY: prepare-run-tests
+prepare-run-tests:
+ifdef PUSHIMAGES
+	docker push $(OPERATORIMAGE)
+endif
+ifneq ($(DEPLOYMENTNAMESPACE), default)
+	$(ROOTDIR)/scripts/kube_delete_namespace.sh $(DEPLOYMENTNAMESPACE)
+	kubectl create namespace $(DEPLOYMENTNAMESPACE)
+endif
+	kubectl apply -f $(MANIFESTPATHCRD)
+	kubectl apply -f $(MANIFESTPATHSTORAGE)
+	kubectl apply -f $(MANIFESTPATHDEPLOYMENT)
+	kubectl apply -f $(MANIFESTPATHDEPLOYMENTREPLICATION)
+	kubectl apply -f $(MANIFESTPATHTEST)
+	$(ROOTDIR)/scripts/kube_create_storage.sh $(DEPLOYMENTNAMESPACE)
+	$(ROOTDIR)/scripts/kube_create_license_key_secret.sh "$(DEPLOYMENTNAMESPACE)" '$(ENTERPRISELICENSE)'
+
 .PHONY: run-tests
 run-tests: docker-test
 ifdef PUSHIMAGES
@@ -424,3 +457,54 @@ redeploy-operator: delete-operator manifests
 	kubectl apply -f $(MANIFESTPATHDEPLOYMENTREPLICATION)
 	kubectl apply -f $(MANIFESTPATHTEST)
 	kubectl get pods 
+
+## ArangoSync Tests
+
+$(ARANGOSYNCTESTCTRLBIN): $(GOBUILDDIR) $(SOURCES)
+	@mkdir -p $(BINDIR)
+	docker run \
+		--rm \
+		-v $(SRCDIR):/usr/code \
+		-v $(CACHEVOL):/usr/gocache \
+		-e GOCACHE=/usr/gocache \
+		-e GOPATH=/usr/code/.gobuild \
+		-e GOOS=linux \
+		-e GOARCH=amd64 \
+		-e CGO_ENABLED=0 \
+		-w /usr/code/ \
+		golang:$(GOVERSION) \
+		go build -installsuffix cgo -ldflags "-X main.projectVersion=$(VERSION) -X main.projectBuild=$(COMMIT)" -o /usr/code/bin/$(ARANGOSYNCTESTCTRLBINNAME) $(REPOPATH)/tests/sync
+
+.PHONY: check-sync-vars
+check-sync-vars:
+ifndef ARANGOSYNCSRCDIR
+	@echo ARANGOSYNCSRCDIR must point to the arangosync source directory
+	@exit 1
+endif
+ifndef ARANGODIMAGE
+	@echo ARANGODIMAGE must point to the usable arangodb enterprise image
+	@exit 1
+endif
+ifndef ENTERPRISELICENSE
+	@echo For tests using ArangoSync you most likely need the license key. Please set ENTERPRISELICENSE.
+	@exit 1
+endif
+	@echo Using ArangoSync source at $(ARANGOSYNCSRCDIR)
+	@echo Using ArangoDB image $(ARANGODIMAGE)
+
+.PHONY: docker-sync
+docker-sync: check-sync-vars
+	SYNCIMAGE=$(ARANGOSYNCIMAGE) TESTIMAGE=$(ARANGOSYNCTESTIMAGE) $(MAKE) -C $(ARANGOSYNCSRCDIR) docker docker-test
+
+.PHONY:
+docker-sync-test-ctrl: $(ARANGOSYNCTESTCTRLBIN)
+	docker build --quiet -f $(DOCKERARANGOSYNCCTRLFILE) -t $(ARANGOSYNCTESTCTRLIMAGE) .
+
+.PHONY:
+run-sync-tests: check-vars docker-sync docker-sync-test-ctrl prepare-run-tests
+ifdef PUSHIMAGES
+	docker push $(ARANGOSYNCTESTCTRLIMAGE)
+	docker push $(ARANGOSYNCTESTIMAGE)
+	docker push $(ARANGOSYNCIMAGE)
+endif
+	$(ROOTDIR)/scripts/kube_run_sync_tests.sh $(DEPLOYMENTNAMESPACE) '$(ARANGODIMAGE)' '$(ARANGOSYNCIMAGE)' '$(ARANGOSYNCTESTIMAGE)' '$(ARANGOSYNCTESTCTRLIMAGE)' '$(TESTOPTIONS)'

manifests/templates/test/rbac.yaml

@@ -10,7 +10,7 @@ rules:
   resources: ["nodes"]
   verbs: ["list"]
 - apiGroups: [""]
-  resources: ["pods", "services", "persistentvolumes", "persistentvolumeclaims", "secrets", "serviceaccounts"]
+  resources: ["pods", "services", "persistentvolumes", "persistentvolumeclaims", "secrets", "serviceaccounts", "pods/log"]
   verbs: ["*"]
 - apiGroups: ["apps"]
   resources: ["daemonsets", "deployments"]

pkg/apis/deployment/v1alpha/deployment_spec.go

@@ -97,6 +97,14 @@ func (s DeploymentSpec) GetImage() string {
 	return util.StringOrDefault(s.Image)
 }
 
+// GetSyncImage returns, if set, Sync.Image or the default image.
+func (s DeploymentSpec) GetSyncImage() string {
+	if s.Sync.HasSyncImage() {
+		return s.Sync.GetSyncImage()
+	}
+	return s.GetImage()
+}
+
 // GetImagePullPolicy returns the value of imagePullPolicy.
 func (s DeploymentSpec) GetImagePullPolicy() v1.PullPolicy {
 	return util.PullPolicyOrDefault(s.ImagePullPolicy)

pkg/apis/deployment/v1alpha/sync_spec.go

@@ -36,13 +36,24 @@ type SyncSpec struct {
 	Authentication SyncAuthenticationSpec `json:"auth"`
 	TLS            TLSSpec                `json:"tls"`
 	Monitoring     MonitoringSpec         `json:"monitoring"`
+	Image          *string                `json:"image"`
 }
 
 // IsEnabled returns the value of enabled.
 func (s SyncSpec) IsEnabled() bool {
 	return util.BoolOrDefault(s.Enabled)
 }
 
+// GetSyncImage returns the syncer image or empty string
+func (s SyncSpec) GetSyncImage() string {
+	return util.StringOrDefault(s.Image)
+}
+
+// HasSyncImage returns whether a special sync image is set
+func (s SyncSpec) HasSyncImage() bool {
+	return s.GetSyncImage() != ""
+}
+
 // Validate the given spec
 func (s SyncSpec) Validate(mode DeploymentMode) error {
 	if s.IsEnabled() && !mode.SupportsSync() {
@@ -78,6 +89,9 @@ func (s *SyncSpec) SetDefaultsFrom(source SyncSpec) {
 	if s.Enabled == nil {
 		s.Enabled = util.NewBoolOrNil(source.Enabled)
 	}
+	if s.Image == nil {
+		s.Image = util.NewStringOrNil(source.Image)
+	}
 	s.ExternalAccess.SetDefaultsFrom(source.ExternalAccess)
 	s.Authentication.SetDefaultsFrom(source.Authentication)
 	s.TLS.SetDefaultsFrom(source.TLS)
@@ -95,5 +109,8 @@ func (s SyncSpec) ResetImmutableFields(fieldPrefix string, target *SyncSpec) []s
 	if list := s.Authentication.ResetImmutableFields(fieldPrefix+".auth", &target.Authentication); len(list) > 0 {
 		resetFields = append(resetFields, list...)
 	}
+	if s.GetSyncImage() != target.GetSyncImage() {
+		resetFields = append(resetFields, fieldPrefix+".image")
+	}
 	return resetFields
 }

pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go

@@ -196,6 +196,8 @@ func (d *Deployment) ensureAccessPackage(apSecretName string) error {
 		},
 		Data: map[string][]byte{
 			constants.SecretAccessPackageYaml: []byte(allYaml),
+			constants.SecretCACertificate:     []byte(tlsCACert),
+			constants.SecretTLSKeyfile:        []byte(keyfile),
 		},
 	}
 	// Attach secret to owner

pkg/deployment/access_package.go

@@ -598,6 +598,11 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string,
 			log.Debug().Str("image", spec.GetImage()).Msg("Image is not an enterprise image")
 			return maskAny(fmt.Errorf("Image '%s' does not contain an Enterprise version of ArangoDB", spec.GetImage()))
 		}
+		// Check if the sync image is overwritten by the SyncSpec
+		imageID := imageInfo.ImageID
+		if spec.Sync.HasSyncImage() {
+			imageID = spec.Sync.GetSyncImage()
+		}
 		var tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName string
 		// Check master JWT secret
 		masterJWTSecretName = spec.Sync.Authentication.GetJWTSecretName()
@@ -664,7 +669,7 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string,
 		if group == api.ServerGroupSyncWorkers {
 			affinityWithRole = api.ServerGroupDBServers.AsRole()
 		}
-		if err := k8sutil.CreateArangoSyncPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, imageInfo.ImageID, lifecycleImage, spec.GetImagePullPolicy(), terminationGracePeriod, args, env,
+		if err := k8sutil.CreateArangoSyncPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, imageID, lifecycleImage, spec.GetImagePullPolicy(), terminationGracePeriod, args, env,
 			livenessProbe, tolerations, serviceAccountName, tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName, affinityWithRole, groupSpec.GetNodeSelector()); err != nil {
 			return maskAny(err)
 		}

pkg/deployment/resources/pod_creator.go

@@ -44,6 +44,15 @@ func NewKubeClient() (kubernetes.Interface, error) {
 	return c, nil
 }
 
+// MustNewKubeClient calls NewKubeClient an panics if it fails
+func MustNewKubeClient() kubernetes.Interface {
+	i, err := NewKubeClient()
+	if err != nil {
+		panic(err)
+	}
+	return i
+}
+
 // NewKubeExtClient creates a new k8s api extensions client
 func NewKubeExtClient() (apiextensionsclient.Interface, error) {
 	cfg, err := InClusterConfig()

pkg/util/k8sutil/client.go

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Run kubectl run to run the integration tests.
+
+DEPLOYMENTNAMESPACE=$1
+ARANGODIMAGE=$2
+ARANGOSYNCIMAGE=$3
+ARANOSYNCTESTIMAGE=$4
+ARANOSYNCTESTCTRLIMAGE=$5
+TESTARGS=$6
+
+ARANGOSYNCIMAGEID=$(docker inspect ${ARANGOSYNCIMAGE} '--format={{index .RepoDigests 0}}')
+ARANOSYNCTESTIMAGEID=$(docker inspect ${ARANOSYNCTESTIMAGE} '--format={{index .RepoDigests 0}}')
+ARANOSYNCTESTCTRLIMAGEID=$(docker inspect ${ARANOSYNCTESTCTRLIMAGE} '--format={{index .RepoDigests 0}}')
+
+kubectl --namespace ${DEPLOYMENTNAMESPACE} \
+    run kube-arangosync-test-controller -i --rm --quiet --restart=Never \
+    --image=${ARANOSYNCTESTCTRLIMAGEID} \
+    -- \
+    --arango-image=${ARANGODIMAGE} \
+    --arango-sync-image=${ARANGOSYNCIMAGEID} \
+    --arango-sync-test-image=${ARANOSYNCTESTIMAGEID} \
+    --license-key-secret-name=arangodb-jenkins-license-key \
+    --namespace=${DEPLOYMENTNAMESPACE} \
+    --test-args="${TESTARGS}"

scripts/kube_run_sync_tests.sh

@@ -0,0 +1,5 @@
+FROM scratch 
+
+ADD bin/arangodb_operator_sync_test_ctrl /usr/bin/
+
+ENTRYPOINT [ "/usr/bin/arangodb_operator_sync_test_ctrl" ]