Skip to content

Commit 0760b85

Browse files
authored
Merge pull request #43 from splunk/codeship
Merge codeship changes to develop
2 parents a4d7f3c + b186990 commit 0760b85

File tree

11 files changed

+401
-0
lines changed

11 files changed

+401
-0
lines changed

codeship-services.yml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
test:
2+
build:
3+
dockerfile_path: codeship/Dockerfile.build
4+
add_docker: true
5+
cached: true
6+
depends_on:
7+
- splunk-hec
8+
9+
splunk-hec:
10+
build:
11+
context: ./codeship/
12+
dockerfile_path: Dockerfile.splunk
13+
hostname: splunk-hec
14+
environment:
15+
- SPLUNK_START_ARGS=--accept-license
16+
- SPLUNK_USER=root
17+
cached: true

codeship-steps.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
- name: compile
2+
service: test
3+
command: bash ./codeship/compile.sh
4+
- name: testing
5+
type: parallel
6+
steps:
7+
- name: unit tests
8+
service: test
9+
command: bash ./codeship/unit_tests.sh
10+
- name: integration tests
11+
service: test
12+
command: bash ./codeship/integration_tests.sh
13+

codeship/Dockerfile.build

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
FROM golang:1.9.2
2+
3+
RUN \
4+
apt-get update && \
5+
apt-get install -y --no-install-recommends \
6+
apt-transport-https \
7+
ca-certificates \
8+
curl \
9+
gnupg2 \
10+
software-properties-common
11+
12+
RUN \
13+
# Docker APT repository
14+
curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg | apt-key add - && \
15+
add-apt-repository \
16+
"deb [arch=amd64] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") \
17+
$(lsb_release -cs) \
18+
stable" && \
19+
\
20+
# Node.js APT repository
21+
curl -sL https://deb.nodesource.com/setup_9.x | bash - && \
22+
\
23+
# Basic dependencies
24+
apt-get update && \
25+
apt-get install -y --no-install-recommends \
26+
build-essential \
27+
docker-ce \
28+
gcc \
29+
git \
30+
nodejs \
31+
npm \
32+
python3 \
33+
python3-dev \
34+
python3-pip \
35+
psmisc && \
36+
apt-get autoclean && \
37+
apt-get clean && \
38+
\
39+
# Node.js dependencies
40+
npm install --no-save \
41+
protocol-buffers \
42+
request && \
43+
\
44+
# Python dependencies
45+
pip3 install virtualenv
46+
47+
RUN mkdir -p /go/src/github.com/splunk/docker-logging-plugin/splunk-logging-plugin/rootfs/bin
48+
49+
WORKDIR /go/src/github.com/splunk/docker-logging-plugin/
50+
51+
COPY . /go/src/github.com/splunk/docker-logging-plugin/
52+
53+
# install go dep
54+
RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
55+
RUN cd /go/src/github.com/splunk/docker-logging-plugin && dep ensure
56+
57+
#Build plugin
58+
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o /go/src/github.com/splunk/docker-logging-plugin/splunk-logging-plugin/rootfs/bin/splunk-logging-plugin .

codeship/Dockerfile.splunk

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
FROM mlaccetti/docker-oracle-java8-ubuntu-16.04
2+
3+
ENV SPLUNK_PRODUCT=splunk \
4+
SPLUNK_VERSION=7.0.2 \
5+
SPLUNK_BUILD=03bbabbd5c0f \
6+
SPLUNK_HOME=/opt/splunk \
7+
SPLUNK_GROUP=splunk \
8+
SPLUNK_USER=splunk
9+
10+
ENV SPLUNK_FILENAME=splunk-${SPLUNK_VERSION}-${SPLUNK_BUILD}-Linux-x86_64.tgz
11+
12+
# Set WORKDIR to Splunks home directory
13+
RUN mkdir -p ${SPLUNK_HOME}
14+
WORKDIR ${SPLUNK_HOME}
15+
VOLUME [ "/opt/splunk/var" ]
16+
17+
# Ports Splunk Web, Splunk Daemon, KVStore, Splunk Indexing Port, Network Input, HTTP Event Collector
18+
EXPOSE 8000/tcp \
19+
8089/tcp \
20+
8191/tcp \
21+
9997/tcp \
22+
1514 \
23+
8088/tcp
24+
25+
# add splunk:splunk user
26+
RUN groupadd -r ${SPLUNK_GROUP} \
27+
&& useradd -r -m -g ${SPLUNK_GROUP} ${SPLUNK_USER}
28+
29+
# make the "en_US.UTF-8" locale so splunk will be utf-8 enabled by default
30+
RUN apt-get update && \
31+
apt-get install -y locales && \
32+
localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
33+
ENV LANG en_US.utf8
34+
35+
RUN rm -rf /etc/service/sshd /etc/my_init.d/00_regen_ssh_host_keys.sh
36+
37+
# install very base dependencies
38+
RUN apt-get update && \
39+
apt-get install -y --no-install-recommends \
40+
bzip2 \
41+
ca-certificates \
42+
curl \
43+
git \
44+
libgssapi-krb5-2 \
45+
procps \
46+
pstack \
47+
python-software-properties \
48+
software-properties-common \
49+
sudo \
50+
unzip \
51+
wget \
52+
xz-utils && \
53+
apt-get --purge remove openjdk* && \
54+
apt-get purge -y --auto-remove && \
55+
rm -rf /var/lib/apt/lists/*
56+
57+
# Download official Splunk release, verify checksum and unzip in /opt/splunk
58+
# Also backup etc folder, so it will be later copied to the linked volume
59+
RUN \
60+
wget -qO /tmp/${SPLUNK_FILENAME} \
61+
https://download.splunk.com/products/${SPLUNK_PRODUCT}/releases/${SPLUNK_VERSION}/linux/${SPLUNK_FILENAME} && \
62+
wget -qO /tmp/${SPLUNK_FILENAME}.md5 \
63+
https://download.splunk.com/products/${SPLUNK_PRODUCT}/releases/${SPLUNK_VERSION}/linux/${SPLUNK_FILENAME}.md5 && \
64+
(cd /tmp && md5sum -c ${SPLUNK_FILENAME}.md5) && \
65+
tar xzf /tmp/${SPLUNK_FILENAME} --strip 1 -C ${SPLUNK_HOME} && \
66+
rm /tmp/${SPLUNK_FILENAME} && \
67+
rm /tmp/${SPLUNK_FILENAME}.md5 && \
68+
# Accept the splunk license on first run \
69+
./bin/splunk version --accept-license
70+
71+
# Create HEC input
72+
RUN mkdir -p ./etc/apps/splunk_httpinput/local
73+
COPY ./splunk/inputs.conf ./etc/apps/splunk_httpinput/local/inputs.conf
74+
75+
# Allow remote logging and reduce the minFreeSpace size to 1GB
76+
RUN mkdir -p ./etc/system/local
77+
COPY ./splunk/server.conf /opt/splunk/etc/system/local/server.conf
78+
79+
HEALTHCHECK --interval=10s --timeout=60s --start-period=20s --retries=10 \
80+
CMD curl -ksSL "https://localhost:8088/services/collector/health" || exit 1
81+
82+
ENTRYPOINT ["./bin/splunk"]
83+
CMD ["start", "--nodaemon"]

codeship/compile.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/usr/bin/env bash
2+
export SHELL=/bin/bash
3+
4+
set -e
5+
6+
echo "Building Docker logging plugin binary.."
7+
make

codeship/functional_tests.sh

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/usr/bin/env bash
2+
export SHELL=/bin/bash
3+
4+
set -e
5+
6+
echo "Running functional tests..."
7+
8+
# Start the plugin
9+
/go/src/github.com/splunk/docker-logging-plugin/splunk-logging-plugin/rootfs/bin/splunk-logging-plugin &
10+
11+
echo "Creating virtual env to run functional tests..."
12+
cd /go/src/github.com/splunk/docker-logging-plugin/test
13+
pip3 install virtualenv
14+
virtualenv --python=python3.5 venv
15+
source venv/bin/activate
16+
pip install -r requirements.txt
17+
18+
19+
python -m pytest \
20+
--splunkd-url https://splunk-hec:8089 \
21+
--splunk-user admin \
22+
--splunk-password changeme \
23+
--splunk-hec-url https://splunk-hec:8088 \
24+
--splunk-hec-token 00000000-0000-0000-0000-000000000000 \
25+
--docker-plugin-path /go/src/github.com/splunk/docker-logging-plugin/splunk-logging-plugin/rootfs/bin/splunk-logging-plugin \
26+
--fifo-path /go/src/github.com/splunk/docker-logging-plugin/pipe

codeship/integration/run.js

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
var protobuf = require('protocol-buffers')
2+
var fs = require('fs')
3+
var net = require('net');
4+
var client = new net.Socket();
5+
var request = require('request');
6+
7+
const stringToWrite = 'helloworld123'
8+
const pipe_file = '/tmp/pipe'
9+
var messages = protobuf(`
10+
message LogEntry {
11+
string source = 1;
12+
int64 time_nano = 2;
13+
bytes line = 3;
14+
bool partial = 4;
15+
}
16+
`)
17+
18+
const writeToPipe = () => {
19+
// create a streaming encoder
20+
var buffer = messages.LogEntry.encode({
21+
source: 'test',
22+
time_nano: Date.now() * 1000000,
23+
line: stringToWrite,
24+
partial: false
25+
});
26+
27+
var buffer2 = Buffer.alloc(4);
28+
buffer2.writeInt32BE(buffer.length, 0);
29+
30+
var writeStream = fs.createWriteStream(pipe_file)
31+
32+
writeStream.write(buffer2);
33+
writeStream.write(buffer);
34+
35+
// close the stream
36+
writeStream.end();
37+
}
38+
writeToPipe();
39+
40+
41+
let reqObj = {
42+
"File": pipe_file,
43+
"Info": {
44+
"ContainerID": "dfjaksfdaslgsaf1213",
45+
"Config": {
46+
"splunk-url": "https://splunk-hec:8088",
47+
"splunk-token": "00000000-0000-0000-0000-000000000000",
48+
"splunk-insecureskipverify": "true",
49+
"splunk-format": "raw"
50+
},
51+
"LogPath": "/go/docker-logging-plugin/plugin/rootfs/test.txt"
52+
}
53+
};
54+
55+
request.post({
56+
headers: {
57+
'Content-Type' : 'application/json',
58+
'Host': 'localhost'
59+
},
60+
url: 'http://unix:/run/docker/plugins/splunklog.sock:/LogDriver.StartLogging',
61+
json: reqObj
62+
}, (error, response, body) => {
63+
console.log(body)
64+
if (body.Err === '') {
65+
console.log('Waiting for 10 sec till events get to splunk')
66+
setTimeout(()=> {
67+
verifyInSplunk();
68+
}, 10000);
69+
}
70+
})
71+
72+
73+
// For the follow http requests
74+
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
75+
76+
const url = 'https://splunk-hec:8089'
77+
const verifyInSplunk = () => {
78+
request.post({
79+
url: url + '/services/search/jobs?output_mode=json',
80+
form: {
81+
earliest_time: '-15m',
82+
latest_time: 'now',
83+
search: `search index=main ${stringToWrite}`
84+
},
85+
auth: {
86+
username: 'admin',
87+
password: 'changeme'
88+
},
89+
agentOptions: 'SSL_OP_NO_SSLv3'
90+
}, (error, res, body) => {
91+
if (error) {
92+
console.log("FAILED", error);
93+
return;
94+
}
95+
let result = JSON.parse(body);
96+
let job_id = result['sid'];
97+
console.log('sid', job_id)
98+
console.log('waiting for 2 sec')
99+
setTimeout(()=> {
100+
waitToGetResult(job_id)
101+
}, 2000);
102+
})
103+
}
104+
105+
const waitToGetResult = (job) => {
106+
const job_url = `${url}/services/search/jobs/${job}?output_mode=json`;
107+
108+
request.get({
109+
url: job_url,
110+
auth: {
111+
username: 'admin',
112+
password: 'changeme'
113+
},
114+
agentOptions: 'SSL_OP_NO_SSLv3'
115+
}, (err, res, body) => {
116+
if (err) {
117+
console.log("FAILED", err);
118+
return;
119+
}
120+
let result = JSON.parse(body);
121+
if (result['entry'][0]['content']['dispatchState'] !== 'DONE') {
122+
console.log('FAIL');
123+
return;
124+
}
125+
console.log('dispatchState=DONE')
126+
getEvents(job);
127+
})
128+
}
129+
130+
const getEvents = (job) => {
131+
const event_url = `${url}/services/search/jobs/${job}/events?output_mode=json`;
132+
request.get({
133+
url: event_url,
134+
auth: {
135+
username: 'admin',
136+
password: 'changeme'
137+
},
138+
agentOptions: 'SSL_OP_NO_SSLv3'
139+
}, (err, res, body) => {
140+
if (err) {
141+
console.log("FAILED", err);
142+
return;
143+
}
144+
let result = JSON.parse(body);
145+
if (result['results'] && result['results'].length > 0 ) {
146+
console.log('PASS')
147+
} else {
148+
console.log('FAIL: 0 result')
149+
}
150+
})
151+
}

codeship/integration_tests.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/usr/bin/env bash
2+
export SHELL=/bin/bash
3+
4+
set -e
5+
6+
echo "Running integration tests..."
7+
8+
# Start the Docker plugin
9+
/go/src/github.com/splunk/docker-logging-plugin/splunk-logging-plugin/rootfs/bin/splunk-logging-plugin &
10+
11+
nodejs /go/src/github.com/splunk/docker-logging-plugin/codeship/integration/run.js

codeship/splunk/inputs.conf

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[http]
2+
disabled=0
3+
port=8088
4+
5+
6+
[http://codeship]
7+
disabled = 0
8+
index = main
9+
indexes = main,history
10+
token = 00000000-0000-0000-0000-000000000000

0 commit comments

Comments
 (0)