Merge pull request #16 from inloco/feature/elastic-auth

Feature/elastic auth

Author: racevedoo

.circleci/config.yml

@@ -1,12 +1,12 @@
-version: 2
+version: 2.1
+defaults: &defaults
+  environment:
+    DOCKER_REPO: inlocomedia/kafka-elasticsearch-injector-go
 jobs:
-  build:
-    environment:
-      - PACKAGE_PATH: "github.com/inloco/kafka-elasticsearch-injector"
+  test:
+    <<: *defaults
     docker:
       - image: circleci/golang:1.12
-        environment:
-          DOCKER_REPO: inlocomedia/kafka-elasticsearch-injector-go
       - image: confluentinc/cp-zookeeper:3.1.1
         environment:
           - ZOOKEEPER_CLIENT_PORT=2181
@@ -27,9 +27,6 @@ jobs:
           transport.host: 127.0.0.1
     steps:
       - checkout
-      - run:
-          name: Generate helper files
-          command: echo $(git rev-parse --short HEAD) > HEAD
       - setup_remote_docker:
           reusable: true
       - run:
@@ -47,34 +44,125 @@ jobs:
               until nc -z localhost 9200 || ((count ++ >= 10)); \
               do echo "Retrying: Verify if Elasticsearch is ready"; sleep 5; done
       - run:
-          name: "Run project tests (excluding vendor)"
-          command: go test $(go list ./... | grep -v /vendor/)
+          name: "Run tests"
+          command: make test
+      - save_cache:
+          key: go-mod-v4-{{ checksum "go.sum" }}
+          paths:
+            - "/go/pkg/mod"
+  build-image:
+    <<: *defaults
+    docker:
+      - image: circleci/golang:1.12
+    steps:
+      - checkout
+      - setup_remote_docker:
+          reusable: true
+          docker_layer_caching: true
+      - run:
+          name: Make `golang get` play nice with bitbucket and github repositories
+          command: |
+            git config --global url."git@bitbucket.org:".insteadOf "https://bitbucket.org/"
+            git config --global url."git@github.com:".insteadOf "https://github.com/"
+            cat ~/.gitconfig
+            ssh-keyscan bitbucket.org >> ~/.ssh/known_hosts
+            ssh-keyscan github.com >> ~/.ssh/known_hosts
+      - restore_cache:
+          keys:
+            - go-mod-v4-{{ checksum "go.sum" }}
       - run:
-          name: Build injector
-          command: GOOS=linux GOARCH=386 go build -a --ldflags="-s" -o bin/injector cmd/injector.go
+          name: Compile binary
+          command: make compile-binary
       - run:
-          name: Build injector docker image
+          name: Build docker images
           command: |
             docker build --rm=false -t "${DOCKER_REPO}:ci" -f cmd/Dockerfile .
-      - save_cache:
-          key: go-mod-v4-{{ checksum "go.sum" }}
+      - run:
+          name: Save docker images and version description files
+          command: |
+            mkdir -p /tmp/workspace
+            docker save -o /tmp/workspace/image.tar $DOCKER_REPO:ci
+            (git describe --abbrev=0 2>/dev/null | sed 's/[^0-9\\.]*//g' || echo -n "0.0.0") > /tmp/workspace/TAG
+            (git rev-parse --short HEAD) > /tmp/workspace/HEAD
+      - persist_to_workspace:
+          root: /tmp/workspace
           paths:
-            - "/go/pkg/mod"
-      - deploy:
-          name: Push to Docker Hub
+            - image.tar
+            - TAG
+            - HEAD
+
+  deploy-image:
+    <<: *defaults
+    docker:
+      - image: circleci/python:3.7
+    steps:
+      - setup_remote_docker:
+          reusable: true
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: Restore docker images from build
+          command: |
+            docker load -i /tmp/workspace/image.tar
+      - run:
+          name: "docker login"
+          command: docker login --username $DOCKER_USER --password $DOCKER_PASSWORD
+      - run:
+          name: Push images
           command: |
-            if [ "${CIRCLE_BRANCH}" == "master" ]; then
-              docker tag "${DOCKER_REPO}:ci" "${DOCKER_REPO}:$(cat VERSION)"
-              docker push "${DOCKER_REPO}:$(cat VERSION)"
-              docker tag "${DOCKER_REPO}:ci" "${DOCKER_REPO}:master"
-              docker push "${DOCKER_REPO}:master"
-            elif [[ $(echo "${CIRCLE_BRANCH}"| grep -c -e "hotfix\/.*") -gt 0 ]]; then
-              docker tag "${DOCKER_REPO}:ci" "${DOCKER_REPO}:$(cat VERSION)-pre"
-              docker push "${DOCKER_REPO}:$(cat VERSION)-pre"
-            elif [[ $(echo "${CIRCLE_BRANCH}"| grep -c -e "release\/.*") -gt 0 ]]; then
-              docker tag "${DOCKER_REPO}:ci" "${DOCKER_REPO}:$(cat VERSION)-pre"
-              docker push "${DOCKER_REPO}:$(cat VERSION)-pre"
+            export TAG=$(cat /tmp/workspace/TAG)
+            export HEAD=$(cat /tmp/workspace/HEAD)
+            export VERSION=$(echo -n "$TAG-$HEAD")
+
+            if [ -z $CIRCLE_TAG ]; then
+              docker tag $DOCKER_REPO:ci $DOCKER_REPO:$VERSION
+              docker push $DOCKER_REPO:$VERSION
+              echo "Pushed $DOCKER_REPO:$VERSION"
             else
-              docker tag "${DOCKER_REPO}:ci" "${DOCKER_REPO}:$(cat VERSION)-$(cat HEAD)"
-              docker push "${DOCKER_REPO}:$(cat VERSION)-$(cat HEAD)"
+              docker tag $DOCKER_REPO:ci $DOCKER_REPO:$TAG
+              docker push $DOCKER_REPO:$TAG
+              echo "Pushed $DOCKER_REPO:$TAG"
             fi
+workflows:
+  version: 2.1
+  test-and-deploy-image-for-commit:
+    jobs:
+      - test:
+        filters:
+          branches:
+            ignore: /master/
+      - build-image:
+        filters:
+          branches:
+            ignore: /master/
+      - deploy-image:
+          context: media
+          requires:
+            - build-image
+          filters:
+            branches:
+              ignore: /master/
+  test-and-deploy-image-for-tag:
+    jobs:
+      - test:
+          filters:
+            tags:
+              only: /.+/
+            branches:
+              ignore: /.*/
+      - build-image:
+          filters:
+            tags:
+              only: /.+/
+            branches:
+              ignore: /.*/
+      - deploy-image:
+          context: media
+          requires:
+            - test
+            - build-image
+          filters:
+            tags:
+              only: /.+/
+            branches:
+              ignore: /.*/

Makefile

@@ -1,11 +1,10 @@
-go/deps:
-	dep ensure -v
-
 test:
 	go test $$(go list ./... | grep -v /vendor/)
 
-docker/build:
+compile-binary:
 	GOOS=linux GOARCH=386 go build -o bin/injector cmd/injector.go
+
+docker/build: compile-binary
 	GOOS=linux GOARCH=386 go build -o bin/producer util/producer/producer.go
 	docker build --rm=false -t "inlocomedia/kafka-elasticsearch-injector:local" -f cmd/Dockerfile .
 	docker build --rm=false -t "inlocomedia/kafka-elasticsearch-injector:producer-local" -f util/producer/Dockerfile .

README.md

@@ -21,6 +21,8 @@ To create new injectors for your topics, you should create a new kubernetes depl
 - `PROBES_PORT` Kubernetes probes port. Set to any available port. **REQUIRED**
 - `K8S_LIVENESS_ROUTE` Kubernetes route for liveness check. **REQUIRED**
 - `K8S_READINESS_ROUTE`Kubernetes route for readiness check. **REQUIRED**
+- `ELASTICSEARCH_USER` Elasticsearch user. **OPTIONAL**
+- `ELASTICSEARCH_PASSWORD` Elasticsearch password. **OPTIONAL**
 - `KAFKA_CONSUMER_CONCURRENCY` Number of parallel goroutines working as a consumer. Default value is 1 **OPTIONAL**
 - `KAFKA_CONSUMER_BATCH_SIZE` Number of records to accumulate before sending them to elasticsearch(for each goroutine). Default value is 100 **OPTIONAL**
 - `ES_INDEX_COLUMN` Record field to append to index name. Ex: to create one ES index per campaign, use "campaign_id" here **OPTIONAL**

VERSION

@@ -9,10 +9,10 @@ require (
 	github.com/datamountaineer/schema-registry v0.0.0-20170721142813-6240b64c5baa
 	github.com/go-kit/kit v0.8.0
 	github.com/inloco/goavro v0.0.0-20180215164239-eba86de05ad3
+	github.com/kr/pretty v0.1.0 // indirect
 	github.com/olivere/elastic/v7 v7.0.1
 	github.com/onsi/ginkgo v1.8.0 // indirect
 	github.com/onsi/gomega v1.5.0 // indirect
-	github.com/pierrec/xxHash v0.1.1 // indirect
 	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
 	github.com/stretchr/testify v1.2.2