Skip to content

Commit 03e9ceb

Browse files
viragtripathiviragtripathi
committed
Oracle demo/docs initial update.
1 parent d40d83e commit 03e9ceb

29 files changed

+1633
-42
lines changed

‎connectors/oracle/README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# redis-connect-oracle
2+
3+
redis-connect-oracle is a Redis Connect connector for capturing changes (INSERT, UPDATE and DELETE) from MySQL (source) and writing them to a Redis Enterprise database (Target). The connector uses [Oracle LogMiner](https://docs.oracle.com/cd/B19306_01/server.102/b14215/logminer.htm#i1010243) to read the database redo log.
4+
5+
<p>
6+
The first time redis-connect-oracle connects to a Oracle database, it reads a consistent snapshot of all of the schemas.
7+
When that snapshot is complete, the connector continuously streams the changes that were committed to Oracle and generates a corresponding insert, update or delete event.
8+
All of the events for each tables are recorded in a separate Redis data structure or module of your choice, where they can be easily consumed by applications and services.

‎connectors/oracle/demo/README.md‎

Lines changed: 582 additions & 0 deletions
Large diffs are not rendered by default.

‎connectors/oracle/demo/config/logback.xml‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
<configuration debug="true" scan="true" scanPeriod="15 seconds">
2+
3+
<property name="START_UP_PATH" value="logs/redis-connect-startup.log"/>
4+
<property name="LOG_PATH" value="logs/redis-connect.log"/>
5+
6+
<appender name="STARTUP" class="ch.qos.logback.core.rolling.RollingFileAppender">
7+
<file>${START_UP_PATH}</file>
8+
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
9+
<fileNamePattern>logs/archived/startup.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
10+
<!-- each archived file, size max 10MB -->
11+
<maxFileSize>10MB</maxFileSize>
12+
<!-- total size of all archive files, if total size > 20GB, it will delete old archived file -->
13+
<totalSizeCap>20GB</totalSizeCap>
14+
<!-- 60 days to keep -->
15+
<maxHistory>60</maxHistory>
16+
</rollingPolicy>
17+
<encoder>
18+
<pattern>%d %p %c{1.} [%t] %m%n</pattern>
19+
</encoder>
20+
</appender>
21+
22+
<appender name="REDISCONNECT" class="ch.qos.logback.core.rolling.RollingFileAppender">
23+
<file>${LOG_PATH}</file>
24+
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
25+
<fileNamePattern>logs/archived/app.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
26+
<!-- each archived file, size max 10MB -->
27+
<maxFileSize>10MB</maxFileSize>
28+
<!-- total size of all archive files, if total size > 20GB, it will delete old archived file -->
29+
<totalSizeCap>20GB</totalSizeCap>
30+
<!-- 60 days to keep -->
31+
<maxHistory>60</maxHistory>
32+
</rollingPolicy>
33+
<encoder>
34+
<pattern>%d %p %c{1.} [%t] %m%n</pattern>
35+
</encoder>
36+
</appender>
37+
38+
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
39+
<encoder>
40+
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
41+
</encoder>
42+
</appender>
43+
44+
<logger name="startup" level="INFO" additivity="false">
45+
<appender-ref ref="STARTUP"/>
46+
<appender-ref ref="CONSOLE" />
47+
</logger>
48+
49+
<logger name="redisconnect" level="INFO" additivity="false">
50+
<appender-ref ref="REDISCONNECT"/>
51+
<appender-ref ref="CONSOLE" />
52+
</logger>
53+
54+
55+
<logger name="com.redislabs" level="INFO" additivity="false">
56+
<appender-ref ref="REDISCONNECT"/>
57+
<appender-ref ref="CONSOLE" />
58+
</logger>
59+
<logger name="io.netty" level="OFF" additivity="false">
60+
<appender-ref ref="REDISCONNECT"/>
61+
<appender-ref ref="CONSOLE" />
62+
</logger>
63+
<logger name="io.lettuce" level="OFF" additivity="false">
64+
<appender-ref ref="REDISCONNECT"/>
65+
<appender-ref ref="CONSOLE" />
66+
</logger>
67+
<logger name="com.zaxxer" level="OFF" additivity="false">
68+
<appender-ref ref="REDISCONNECT"/>
69+
<appender-ref ref="CONSOLE"/>
70+
</logger>
71+
<logger name="io.debezium" level="INFO" additivity="false">
72+
<appender-ref ref="REDISCONNECT"/>
73+
<appender-ref ref="CONSOLE"/>
74+
</logger>
75+
<logger name="org.apache.kafka" level="OFF" additivity="false">
76+
<appender-ref ref="REDISCONNECT"/>
77+
<appender-ref ref="CONSOLE"/>
78+
</logger>
79+
<logger name="org.springframework" level="OFF" additivity="false">
80+
<appender-ref ref="REDISCONNECT"/>
81+
<appender-ref ref="CONSOLE"/>
82+
</logger>
83+
84+
<root>
85+
<appender-ref ref="STARTUP"/>
86+
<appender-ref ref="REDISCONNECT"/>
87+
</root>
88+
89+
</configuration>

‎connectors/oracle/demo/config/samples/loader/FormatterConfig.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
default:
2+
formatter: FREEMARKER_FORMATTER
3+
template: name-value.ftl
4+
dbo.emp:
5+
template: pipe-delimited.ftl

‎connectors/oracle/demo/config/samples/loader/JobConfig.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
jobId: ${jobId}
2+
producerConfig:
3+
producerId: RDB_INITIAL_LOAD_PRODUCER
4+
connectionId: RDBConnection
5+
batchSize: 5000
6+
startRecord: ${startRecord}
7+
endRecord: ${endRecord}
8+
loadQuery: ${loadQuery}
9+
table: ${table}
10+
pipelineConfig:
11+
eventTranslator: "RDB_RS_TO_OP_TRANSLATOR"
12+
checkpointConfig:
13+
providerId: REDIS_HASH_CHECKPOINT_READER
14+
connectionId: targetConnection
15+
checkpoint: "${jobId}"
16+
stages:
17+
HashWriteStage:
18+
handlerId: REDIS_HASH_WRITER
19+
connectionId: targetConnection
20+
metricsEnabled: false
21+
prependTableNameToKeys: true
22+
deleteOnKeyUpdate: true
23+
async: true
24+
CheckpointStage:
25+
handlerId: REDIS_HASH_CHECKPOINT_WRITER
26+
connectionId: targetConnection
27+
metricEnabled: false
28+
async: true
29+
checkpoint: "${jobId}"
30+
31+

‎connectors/oracle/demo/config/samples/loader/JobManager.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
connectionId: jobConfigConnection
2+
metricsReporter:
3+
- REDIS_TS_METRICS_REPORTER

‎connectors/oracle/demo/config/samples/loader/Setup.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
connectionId: jobConfigConnection
2+
job:
3+
jobConfig:
4+
- name: initial_load
5+
config: TaskCreator.yml

‎connectors/oracle/demo/config/samples/loader/TaskCreator.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
jobId: ${jobId}
2+
jobType: TASK_CREATOR
3+
producerConfig:
4+
producerId: RDB_TASK_CREATOR # internal id
5+
connectionId: RDBConnection # source connectionId same as env.yml
6+
jobs:
7+
- table: HR.EMPLOYEES # schema.table
8+
numJobs: 5
9+
config: JobConfig.yml
10+
loadQuery: "SELECT * FROM (SELECT e.*, ROW_NUMBER() OVER (ORDER BY EMPLOYEE_ID) RowIndex FROM HR.EMPLOYEES e) WHERE RowIndex BETWEEN ${start} and ${end}"

‎connectors/oracle/demo/config/samples/loader/env.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
connections:
2+
- id: jobConfigConnection
3+
type: Redis
4+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
5+
- id: targetConnection
6+
type: Redis
7+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14000
8+
- id: metricsConnection
9+
type: Redis
10+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
11+
- id: RDBConnection
12+
type: RDB
13+
name: RedisConnect #database pool name
14+
database: ORCLPDB1 #database
15+
url: "jdbc:oracle:thin:@//127.0.0.1:1521/ORCLPDB1" # 12c
16+
#url: "jdbc:oracle:thin:@//127.0.0.1:1522/ORCLPDB1" # 19c
17+
host: 127.0.0.1
18+
port: 1521
19+
username: ${REDISCONNECT_SOURCE_USERNAME}
20+
password: ${REDISCONNECT_SOURCE_PASSWORD}

‎connectors/oracle/demo/config/samples/loader/mappers/mapper1.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
schema: HR
2+
tables:
3+
- table: EMPLOYEES
4+
mapper:
5+
id: employees
6+
processorID: employees
7+
publishBefore: false
8+
columns:
9+
- src: EMPLOYEE_ID
10+
target: EmpId
11+
type: INT
12+
publishBefore: false
13+
- src: FIRST_NAME
14+
target: FName
15+
- src: LAST_NAME
16+
target: LName
17+
- src: EMAIL
18+
target: Email
19+
- src: JOB_ID
20+
target: JobId
21+
- src: SALARY
22+
target: Salary
23+
type: DOUBLE
24+
- table: JOBS
25+
mapper:
26+
id: Jobs
27+
processorID: Jobs
28+
publishBefore: false
29+
passThrough: true
30+
columns:
31+
- src: JOB_ID
32+
target: JobId
33+
type: INT
34+
- table: EMP
35+
mapper:
36+
id: emp
37+
processorID: emp
38+
publishBefore: false
39+
columns:
40+
- src: EMPNO
41+
target: EmployeeNumber
42+
type: INT
43+
publishBefore: false
44+
- src: FNAME
45+
target: FirstName
46+
- src: LNAME
47+
target: LastName
48+
- src: JOB
49+
target: Job
50+
- src: MGR
51+
target: Manager
52+
- src: HIREDATE
53+
target: HireDate
54+
- src: SAL
55+
target: Salary
56+
- src: COMM
57+
target: Commission
58+
- src: DEPT
59+
target: Department

0 commit comments

Comments
 (0)