Added support for internal build levels

* Dummy change to see if things are working

* Initial internal build support

* Add SSH key support

* Add SSH key support pt 2

* Add SSH key support pt 3

* Add Push support

* Full package support

* Correct typo

* Correct typo 2

* Disable some tests as per master

* Change rsync variable names

* Fix build issues

* Remove the -full package support for now

Co-authored-by: Jason Edmeades <jason_edmeades@uk.ibm.com>

Author: 2 people

Makefile

@@ -38,7 +38,7 @@ MQ_ARCHIVE_DEV ?= $(MQ_VERSION)-IBM-MQ-Advanced-for-Developers-Non-Install-$(MQ_
 # MQ_SDK_ARCHIVE specifies the archive to use for building the golang programs.  Defaults vary on developer or advanced.
 MQ_SDK_ARCHIVE ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION))
 # Options to `go test` for the Docker tests
-TEST_OPTS_DOCKER ?= 
+TEST_OPTS_DOCKER ?=
 # Timeout for the Docker tests
 TEST_TIMEOUT_DOCKER ?= 30m
 # MQ_IMAGE_ADVANCEDSERVER is the name of the built MQ Advanced image
@@ -113,7 +113,7 @@ endif
 
 # LTS_TAG is the tag modifier for an LTS container build
 LTS_TAG=
-ifeq "$(LTS)" "true" 
+ifeq "$(LTS)" "true"
 ifneq "$(LTS_TAG_OVERRIDE)" "$(EMPTY)"
 	LTS_TAG=$(LTS_TAG_OVERRIDE)
 else
@@ -201,18 +201,37 @@ incubating: build-explorer
 downloads/$(MQ_ARCHIVE_DEV):
 	$(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced for Developers "$(MQ_VERSION)$(END)))
 	mkdir -p downloads
+ifneq "$(BUILD_RSYNC_SERVER)" "$(EMPTY)"
+# Use key which is not stored in the repository to fetch the files from the fileserver
+	curl -L $(BUILD_RSYNC_ENCRYPTED_KEY_URL) -o ./host.key.gpg
+	@echo $(BUILD_RSYNC_ENCRYPTION_PASSWORD)|gpg --passphrase-fd 0 ./host.key.gpg
+	chmod 600 ./host.key
+	rsync -rv -e "ssh -o BatchMode=yes -q -o StrictHostKeyChecking=no -i ./host.key" --include="*/" --include="*.tar.gz" --exclude="*" $(BUILD_RSYNC_USER)@$(BUILD_RSYNC_SERVER):"$(BUILD_RSYNC_PATH)" downloads/$(MQ_ARCHIVE_DEV)
+	-@rm host.key.gpg host.key
+else
 ifneq "$(MQ_ARCHIVE_REPOSITORY_DEV)" "$(EMPTY)"
 	curl -u $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -X GET "$(MQ_ARCHIVE_REPOSITORY_DEV)" -o downloads/$(MQ_ARCHIVE_DEV)
 else
 	curl -L https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/$(MQ_ARCHIVE_DEV) -o downloads/$(MQ_ARCHIVE_DEV)
 endif
+endif
 
 downloads/$(MQ_ARCHIVE):
 	$(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced "$(MQ_VERSION)$(END)))
 	mkdir -p downloads
+ifneq "$(BUILD_RSYNC_SERVER)" "$(EMPTY)"
+# Use key which is not stored in the repository to fetch the files from the fileserver
+	-@rm host.key.gpg host.key
+	curl -L $(BUILD_RSYNC_ENCRYPTED_KEY_URL) -o ./host.key.gpg
+	@echo $(BUILD_RSYNC_ENCRYPTION_PASSWORD)|gpg --passphrase-fd 0 ./host.key.gpg
+	chmod 600 ./host.key
+	rsync -rv -e "ssh -o BatchMode=yes -q -o StrictHostKeyChecking=no -i ./host.key" --include="*/" --include="*.tar.gz" --exclude="*" $(BUILD_RSYNC_USER)@$(BUILD_RSYNC_SERVER):"$(BUILD_RSYNC_PATH)" downloads/$(MQ_ARCHIVE)
+	-@rm host.key.gpg host.key
+else
 ifneq "$(MQ_ARCHIVE_REPOSITORY)" "$(EMPTY)"
 	curl -u $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -X GET "$(MQ_ARCHIVE_REPOSITORY)" -o downloads/$(MQ_ARCHIVE)
 endif
+endif
 
 .PHONY: downloads
 downloads: downloads/$(MQ_ARCHIVE_DEV) downloads/$(MQ_SDK_ARCHIVE)
@@ -283,7 +302,7 @@ define build-mq
 	$(COMMAND) build \
 	  --tag $1:$2 \
 	  --file $3 \
-		$(EXTRA_ARGS) \
+	  $(EXTRA_ARGS) \
 	  --build-arg IMAGE_REVISION="$(IMAGE_REVISION)" \
 	  --build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" \
 	  --build-arg IMAGE_TAG="$1:$2" \
@@ -415,15 +434,15 @@ ifneq "$(LTS)" "true"
 	$(eval MQ_IMAGE_DEVSERVER_S390X_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_DEVSERVER_S390X) | jq -r .Digest))
 	$(info $(shell printf "** Determined the built $(MQ_IMAGE_DEVSERVER_AMD64) has a digest of $(MQ_IMAGE_DEVSERVER_AMD64_DIGEST)**"$(END)))
 	$(info $(shell printf "** Determined the built $(MQ_IMAGE_DEVSERVER_S390X) has a digest of $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)**"$(END)))
-endif	
+endif
 	$(eval MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_ADVANCEDSERVER_AMD64) | jq -r .Digest))
 	$(eval MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_ADVANCEDSERVER_S390X) | jq -r .Digest))
 	$(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_AMD64) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST)**"$(END)))
 	$(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_S390X) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)**"$(END)))
-ifneq "$(LTS)" "true"	
+ifneq "$(LTS)" "true"
 	$(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_DEVSERVER_MANIFEST)**"$(END)))
 	echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_DEVSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL)  -d "$(MQ_IMAGE_DEVSERVER_AMD64_DIGEST) $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)" $(END))
-endif	
+endif
 	$(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_ADVANCEDSERVER_MANIFEST)**"$(END)))
 	echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_ADVANCEDSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST) $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)" $(END))
 

travis-build-scripts/build.sh

@@ -20,20 +20,36 @@ if [ "$TRAVIS_BRANCH" = "$MAIN_BRANCH" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]
   echo 'Retrieving global tagcache' && echo -en 'travis_fold:start:tag-cache-retrieve\\r'
   ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --check
   ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --get
-  echo -en 'travis_fold:end:tag-cache-retrieve\\r' 
+  echo -en 'travis_fold:end:tag-cache-retrieve\\r'
 fi
-if [ "$LTS" != true ] ; then
+if [ -z "$BUILD_INTERNAL_LEVEL" ] ; then
+  if [ "$LTS" != true ] ; then
+    echo 'Building Developer JMS test image...' && echo -en 'travis_fold:start:build-devjmstest\\r'
+    make build-devjmstest
+    echo -en 'travis_fold:end:build-devjmstest\\r'
+    echo 'Building Developer image...' && echo -en 'travis_fold:start:build-devserver\\r'
+    make build-devserver
+    echo -en 'travis_fold:end:build-devserver\\r'
+  fi
+  if [ "$BUILD_ALL" = true ] || [ "$LTS" = true ] ; then
+      if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then
+          echo 'Building Production image...' && echo -en 'travis_fold:start:build-advancedserver\\r'
+          make build-advancedserver
+          echo -en 'travis_fold:end:build-advancedserver\\r'
+      fi
+  fi
+else
   echo 'Building Developer JMS test image...' && echo -en 'travis_fold:start:build-devjmstest\\r'
   make build-devjmstest
   echo -en 'travis_fold:end:build-devjmstest\\r'
-  echo 'Building Developer image...' && echo -en 'travis_fold:start:build-devserver\\r'
-  make build-devserver
-  echo -en 'travis_fold:end:build-devserver\\r'
-fi
-if [ "$BUILD_ALL" = true ] || [ "$LTS" = true ] ; then
-    if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then
-        echo 'Building Production image...' && echo -en 'travis_fold:start:build-advancedserver\\r'
-        make build-advancedserver
-        echo -en 'travis_fold:end:build-advancedserver\\r'
-    fi
+
+  if [[ "$BUILD_INTERNAL_LEVEL" == *".DE"* ]]; then
+    echo 'Building Developer image...' && echo -en 'travis_fold:start:build-devserver\\r'
+    make build-devserver
+    echo -en 'travis_fold:end:build-devserver\\r'
+  else
+    echo 'Building Production image...' && echo -en 'travis_fold:start:build-advancedserver\\r'
+    make build-advancedserver
+    echo -en 'travis_fold:end:build-advancedserver\\r'
+  fi
 fi

travis-build-scripts/push.sh

@@ -21,7 +21,7 @@ if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
     exit 0
 fi
 
-if [ ! -z $2 ]; then 
+if [ ! -z $2 ]; then
     export ARCH=$2
 fi
 
@@ -50,7 +50,7 @@ if [ ! -z $1 ]; then
         exit 1
         ;;
     esac
-else 
+else
     echo "ERROR: Type ( developer | production ) must be passed to push.sh"
     exit 1
 fi

travis-build-scripts/run.sh

@@ -21,7 +21,7 @@ if [ "$(uname -m)" = "x86_64" ] ; then export ARCH="amd64" ; else export ARCH=$(
 if [ "$PUSH_MANIFEST_ONLY" = true ] ; then
   echo 'Retrieving remote tagcache' && echo -en 'travis_fold:start:retrieve-tag-cache\\r'
   ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --get
-  echo -en 'travis_fold:end:retrieve-tag-cache\\r' 
+  echo -en 'travis_fold:end:retrieve-tag-cache\\r'
   make push-manifest
   ./travis-build-scripts/cleanup-cache.sh
   exit 0
@@ -38,9 +38,17 @@ echo -en 'travis_fold:end:docker-downgrade\\r'
 ./travis-build-scripts/test.sh
 
 ## Push images
-if [ "$BUILD_ALL" = true ] ; then
+if [ -z "$BUILD_INTERNAL_LEVEL" ] ; then
+  if [ "$BUILD_ALL" = true ] ; then
     ./travis-build-scripts/push.sh developer
     ./travis-build-scripts/push.sh production
+  fi
+else
+  if [[ "$BUILD_INTERNAL_LEVEL" == *".DE"* ]]; then
+    ./travis-build-scripts/push.sh developer
+  else
+    ./travis-build-scripts/push.sh production
+  fi
 fi
 
 if [ "$LTS" = true ] ; then

travis-build-scripts/test.sh

@@ -16,17 +16,29 @@
 
 set -e
 
-if [ "$LTS" != true ] ; then
-  echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r'
-  make test-devserver
-  echo -en 'travis_fold:end:test-devserver\\r'
-fi
-if [ "$BUILD_ALL" = true ] || [ "$LTS" = true ] ; then
-    if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then
-        echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r'
-        make test-advancedserver
-        echo -en 'travis_fold:end:test-advancedserver\\r'
-    fi
+if [ -z "$BUILD_INTERNAL_LEVEL" ] ; then
+  if [ "$LTS" != true ] ; then
+    echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r'
+    make test-devserver
+    echo -en 'travis_fold:end:test-devserver\\r'
+  fi
+  if [ "$BUILD_ALL" = true ] || [ "$LTS" = true ] ; then
+      if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then
+          echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r'
+          make test-advancedserver
+          echo -en 'travis_fold:end:test-advancedserver\\r'
+      fi
+  fi
+else
+  if [[ "$BUILD_INTERNAL_LEVEL" == *".DE"* ]]; then
+    echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r'
+    make test-devserver
+    echo -en 'travis_fold:end:test-devserver\\r'
+  else
+    echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r'
+    make test-advancedserver
+    echo -en 'travis_fold:end:test-advancedserver\\r'
+  fi
 fi
 echo 'Running gosec scan...' && echo -en 'travis_fold:start:gosec-scan\\r'
 if [ "$ARCH" = "amd64" ] ; then