Skip to content

Commit c6d37f9

Browse files
Gunasekar-KGunasekar-K
committed
producer and consumer intgration
1 parent 7233e1c commit c6d37f9

24 files changed

+2130
-1
lines changed

‎Delta Migration from Informix to Postgres.docx.pdf‎

159 KB
Binary file not shown.

‎Dockerfile‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
FROM appiriodevops/tc-database-scripts:base
2+
3+
ARG servername=informix
4+
5+
USER root
6+
WORKDIR /home/informix
7+
RUN sed -i '/jessie-updates/d' /etc/apt/sources.list
8+
RUN apt-get -qq update && apt-get -qq install -y \
9+
wget gcc g++ make xz-utils python2.7 git curl
10+
11+
RUN wget -q -O node8.tar.xz https://nodejs.org/dist/v8.11.3/node-v8.11.3-linux-x64.tar.xz \
12+
&& tar xfJ node8.tar.xz && rm -rf node8.tar.xz
13+
14+
ENV SERVERNAME=$servername
15+
ENV INFORMIXDIR /opt/IBM/informix
16+
ENV INFORMIX_HOME /home/informix
17+
ENV INFORMIXSERVER informixoltp_tcp
18+
ENV INFORMIXTERM terminfo
19+
ENV CLIENT_LOCALE=en_US.utf8
20+
ENV DB_LOCALE=en_US.utf8
21+
ENV DBDATE Y4MD-
22+
ENV DBDELIMITER "|"
23+
24+
COPY docker/esql ${INFORMIXDIR}/bin/
25+
26+
RUN chmod +x ${INFORMIXDIR}/bin/esql
27+
RUN echo "informixoltp_tcp onsoctcp ${SERVERNAME:-informix} sqlexec" \
28+
> ${INFORMIXDIR}/etc/sqlhosts.informixoltp_tcp
29+
30+
31+
ENV PATH /home/informix/node-v8.11.3-linux-x64/bin:${INFORMIXDIR}/bin:${INFORMIXDIR}/lib:${INFORMIXDIR}/lib/esql:${PATH}
32+
ENV LD_LIBRARY_PATH ${INFORMIXDIR}/lib:${INFORMIXDIR}/lib/esql:${INFORMIXDIR}/lib/cli
33+
ENV INFORMIXSQLHOSTS ${INFORMIXDIR}/etc/sqlhosts.informixoltp_tcp
34+
ENV USER root
35+
ENV LICENSE accept
36+
37+
RUN rm /usr/bin/python && ln -s /usr/bin/python2.7 /usr/bin/python
38+
RUN echo "sqlexec 2021/tcp" >> /etc/services
39+
40+
RUN mkdir /app
41+
COPY . /app
42+
43+
WORKDIR /app
44+
RUN rm -rf node_modules && npm install --unsafe-perm
45+
46+
# ENTRYPOINT [ "/bin/bash" ]
47+
ENTRYPOINT [ "npm" ]

‎Dockerfile-kaf‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
FROM ubuntu:latest
2+
RUN apt update
3+
RUN apt install openjdk-8-jdk net-tools vim wget -y
4+
WORKDIR /opt
5+
RUN wget http://apachemirror.wuchna.com/kafka/2.2.0/kafka_2.12-2.2.0.tgz
6+
RUN tar -xvzf kafka_2.12-2.2.0.tgz
7+
RUN mv kafka_2.12-2.2.0 kafka
8+
WORKDIR /opt/kafka
9+
RUN cp config/server.properties config/serverorg.properties
10+
RUN sed -i '/^#listeners=*/s/^#//' config/server.properties
11+
RUN sed -i '/^#advertised.listeners=*/s/^#//' config/server.properties
12+
RUN sed -i '/^advertised.listeners=*/s/your.host.name/host.docker.internal/' config/server.properties
13+
RUN touch nohup.out
14+
RUN echo "#!/bin/bash" >start.sh
15+
RUN echo "nohup bin/zookeeper-server-start.sh config/zookeeper.properties &" >>start.sh
16+
RUN echo "nohup bin/kafka-server-start.sh config/server.properties &" >>start.sh
17+
RUN echo "tail -f nohup.out" >>start.sh
18+
RUN chmod +x start.sh
19+
EXPOSE 2181 9092
20+
# CMD ./start.sh
21+
22+
ENTRYPOINT ./start.sh

‎README.md‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,41 @@
1-
# ifx-pg-sync-kafka
1+
#Data Migration
2+
## Prerequisites
3+
4+
- NodeJS (v12)
5+
- Docker, Docker Compose
6+
7+
## Configuration
8+
Configuration for the application is at `config/default.js` The following parameters can be set in config files
9+
10+
- PORT: the server port
11+
- topic: kafka topic NAME&PARTITION
12+
- topic_error: kafka topic NAME&PARTITION, which handle some errors
13+
- RETRY_COUNTER: kafka max retry counter
14+
- SOURCE: SOURCE database information. This will be in `INFORMIX` now
15+
- DESTINATION: DESTINATION database information. This will be in `POSTGRESQL` now
16+
- db: the DBs URI and OPTIONS
17+
18+
##Environment setup
19+
###Informix/Kafka
20+
open the pdf for more Details `Migration from Informix to Postgres.docx.pdf`
21+
22+
###Postgres
23+
run `docker-compose up -d`
24+
25+
###Producer and Consume
26+
- open a terminal
27+
- run `npm i`
28+
- run `npm run producer`
29+
- open new terminal
30+
- run `npm run consumer`
31+
32+
## Verification
33+
- create testdb schema via `http://localhost:8081`
34+
- Connect with Informix server which we are running already
35+
- run cd /home/informix/trunk/dev2
36+
- run dbaccess -e testdb saveAuditOnFile.sql
37+
- This will generate a json file in /tmp. Also it will post the json file to Producer
38+
- We can see the db detail at `http://localhost:8081`
39+
- delete line 54:`DELETE FROM test where name = 'name'; `
40+
- run dbaccess -e testdb saveAuditOnFile.sql again
41+
- see the db detail at `http://localhost:8081`

‎config/default.js‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
const path = require('path');
2+
module.exports = {
3+
LOG_LEVEL: process.env.LOG_LEVEL || 'debug',
4+
LOG_FILE: path.join(__dirname, '../app.log'),
5+
PORT: process.env.PORT || 8080,
6+
topic: {
7+
NAME: 'db.ifxpgmigrate.sync',
8+
PARTITION: 0
9+
},
10+
RETRY_COUNTER: 3,
11+
topic_error: {
12+
NAME: 'db.ifxpgmigrate.error',
13+
PARTITION: 0,
14+
EMAIL: 'admin@abc.com'
15+
},
16+
SOURCE: 'INFORMIX',
17+
DESTINATION: 'POSTGRESQL',
18+
db: {
19+
// URL: "postgres://postgres:password@host.docker.internal:54321/",
20+
OPTIONS: {
21+
logging: true,
22+
operatorsAliases: false,
23+
dialectOptions: { encrypt: true }
24+
},
25+
DB_NAME:['auditlog']
26+
},
27+
POSTGRES: {
28+
user: process.env.PG_USER || 'postgres',
29+
host: process.env.PG_HOST || 'host.docker.internal',
30+
database: process.env.PG_DATABASE || 'testdb', // database must exists before run tool
31+
password: process.env.PG_PASSWORD || 'password',
32+
port: parseInt(process.env.PG_PORT || 54321, 10),
33+
},
34+
INFORMIX: {
35+
SERVER: process.env.IFX_SERVER || 'informixoltp_tcp',
36+
DATABASE: process.env.IFX_DATABASE || 'tcs_catalog',
37+
HOST: process.env.INFORMIX_HOST || 'host.docker.internal',
38+
PROTOCOL: process.env.IFX_PROTOCOL || 'onsoctcp',
39+
PORT: process.env.IFX_PORT || '2021',
40+
DB_LOCALE: process.env.IFX_DB_LOCALE || 'en_US.utf8',
41+
USER: process.env.IFX_USER || 'informix',
42+
PASSWORD: process.env.IFX_PASSWORD || '1nf0rm1x',
43+
POOL_MAX_SIZE: parseInt(process.env.IFX_POOL_MAX_SIZE || '10')
44+
}
45+
}

‎docker-compose.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
version: '3.1'
2+
3+
services:
4+
5+
pgdb:
6+
image: postgres
7+
restart: always
8+
ports:
9+
- "54321:5432"
10+
environment:
11+
POSTGRES_DB: auditlog
12+
POSTGRES_PASSWORD: password
13+
14+
adminer:
15+
image: adminer
16+
restart: always
17+
ports:
18+
- 8082:8080
19+
20+
kafka:
21+
image: kafka-ifxpg-local:latest
22+
build:
23+
context: .
24+
dockerfile: Dockerfile-kaf
25+
ports:
26+
- "9092:9092"
27+
28+
producer:
29+
image: node-ifxpg:latest
30+
build:
31+
context: .
32+
ports:
33+
- "8080:8080"
34+
command: run producer
35+
depends_on:
36+
- "kafka"
37+
38+
producerwithoutkafka:
39+
image: node-ifxpg:latest
40+
build:
41+
context: .
42+
ports:
43+
- "8080:8080"
44+
command: run producerwithoutkafka
45+
46+
47+
consumer:
48+
image: node-ifxpg:latest
49+
build:
50+
context: .
51+
command: run consumer
52+
depends_on:
53+
- "kafka"
54+
- "pgdb"
55+
56+
57+

0 commit comments

Comments
 (0)