Skip to content

Commit b054880

Browse files
viragtripathiviragtripathi
committed
Updated mssql and postgres demo artifacts
1 parent 1019f06 commit b054880

File tree

24 files changed

+645
-139
lines changed

24 files changed

+645
-139
lines changed

‎connectors/db2/demo/setup_re.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ sudo docker exec -it re-node1 bash -c "/opt/redislabs/bin/rladmin info cluster"
1919
echo "Creating databases..."
2020
rm create_demodb.sh
2121
tee -a create_demodb.sh <<EOF
22-
curl -v -k -L -u demo@redis.com:redislabs --location-trusted -H "Content-type:application/json" -d '{ "name": "RedisConnect-Target-db", "port": 12000, "memory_size": 1000000000, "type" : "redis", "replication": false, "module_list": [ {"module_args": "PARTITIONS AUTO", "module_id": "9f8e4b44fbd28838190dbee7935e964d", "module_name": "search", "semantic_version": "2.0.11"} ] }' https://localhost:9443/v1/bdbs
22+
curl -v -k -L -u demo@redis.com:redislabs --location-trusted -H "Content-type:application/json" -d '{ "name": "RedisConnect-Target-db", "port": 12000, "memory_size": 1000000000, "type" : "redis", "replication": false, "module_list": [ {"module_args": "PARTITIONS AUTO", "module_id": "9f8e4b44fbd28838190dbee7935e964d", "module_name": "search", "semantic_version": "2.0.11"}, {"module_args": "", "module_id": "4942f870bcd96678dd92cd55ae5d2801", "module_name": "ReJSON", "semantic_version": "1.0.8"} ] }' https://localhost:9443/v1/bdbs
2323
curl -v -k -L -u demo@redis.com:redislabs --location-trusted -H "Content-type:application/json" -d '{"name": "RedisConnect-JobConfig-Metrics-db", "type":"redis", "replication": false, "memory_size":1000000000, "port":12001, "module_list": [{"module_args": "", "module_id": "6715acd18978c330bb2fb3f4193f070c", "module_name": "timeseries", "semantic_version": "1.4.10"}]}' https://localhost:9443/v1/bdbs
2424
EOF
2525
sleep 20

‎connectors/mssql/demo/README.md‎

Lines changed: 242 additions & 127 deletions
Large diffs are not rendered by default.

‎connectors/mssql/demo/config/logback.xml‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
<configuration debug="true" scan="true" scanPeriod="15 seconds">
2+
3+
<property name="START_UP_PATH" value="logs/redis-connect-startup.log"/>
4+
<property name="LOG_PATH" value="logs/redis-connect.log"/>
5+
6+
<appender name="STARTUP" class="ch.qos.logback.core.rolling.RollingFileAppender">
7+
<file>${START_UP_PATH}</file>
8+
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
9+
<fileNamePattern>logs/archived/startup.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
10+
<!-- each archived file, size max 10MB -->
11+
<maxFileSize>10MB</maxFileSize>
12+
<!-- total size of all archive files, if total size > 20GB, it will delete old archived file -->
13+
<totalSizeCap>20GB</totalSizeCap>
14+
<!-- 60 days to keep -->
15+
<maxHistory>60</maxHistory>
16+
</rollingPolicy>
17+
<encoder>
18+
<pattern>%d %p %c{1.} [%t] %m%n</pattern>
19+
</encoder>
20+
</appender>
21+
22+
<appender name="REDISCONNECT" class="ch.qos.logback.core.rolling.RollingFileAppender">
23+
<file>${LOG_PATH}</file>
24+
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
25+
<fileNamePattern>logs/archived/app.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
26+
<!-- each archived file, size max 10MB -->
27+
<maxFileSize>10MB</maxFileSize>
28+
<!-- total size of all archive files, if total size > 20GB, it will delete old archived file -->
29+
<totalSizeCap>20GB</totalSizeCap>
30+
<!-- 60 days to keep -->
31+
<maxHistory>60</maxHistory>
32+
</rollingPolicy>
33+
<encoder>
34+
<pattern>%d %p %c{1.} [%t] %m%n</pattern>
35+
</encoder>
36+
</appender>
37+
38+
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
39+
<encoder>
40+
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
41+
</encoder>
42+
</appender>
43+
44+
<logger name="startup" level="INFO" additivity="false">
45+
<appender-ref ref="STARTUP"/>
46+
<appender-ref ref="CONSOLE" />
47+
</logger>
48+
49+
<logger name="redisconnect" level="INFO" additivity="false">
50+
<appender-ref ref="REDISCONNECT"/>
51+
<appender-ref ref="CONSOLE" />
52+
</logger>
53+
54+
55+
<logger name="com.redislabs" level="INFO" additivity="false">
56+
<appender-ref ref="REDISCONNECT"/>
57+
<appender-ref ref="CONSOLE" />
58+
</logger>
59+
<logger name="io.netty" level="OFF" additivity="false">
60+
<appender-ref ref="REDISCONNECT"/>
61+
<appender-ref ref="CONSOLE" />
62+
</logger>
63+
<logger name="io.lettuce" level="OFF" additivity="false">
64+
<appender-ref ref="REDISCONNECT"/>
65+
<appender-ref ref="CONSOLE" />
66+
</logger>
67+
<logger name="com.zaxxer" level="OFF" additivity="false">
68+
<appender-ref ref="REDISCONNECT"/>
69+
<appender-ref ref="CONSOLE"/>
70+
</logger>
71+
<logger name="io.debezium" level="INFO" additivity="false">
72+
<appender-ref ref="REDISCONNECT"/>
73+
<appender-ref ref="CONSOLE"/>
74+
</logger>
75+
<logger name="org.apache.kafka" level="OFF" additivity="false">
76+
<appender-ref ref="REDISCONNECT"/>
77+
<appender-ref ref="CONSOLE"/>
78+
</logger>
79+
<logger name="org.springframework" level="OFF" additivity="false">
80+
<appender-ref ref="REDISCONNECT"/>
81+
<appender-ref ref="CONSOLE"/>
82+
</logger>
83+
84+
<root>
85+
<appender-ref ref="STARTUP"/>
86+
<appender-ref ref="REDISCONNECT"/>
87+
</root>
88+
89+
</configuration>

‎connectors/mssql/demo/config/samples/loader/FormatterConfig.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
default:
2+
formatter: FREEMARKER_FORMATTER
3+
template: name-value.ftl
4+
dbo.emp:
5+
template: pipe-delimited.ftl

‎connectors/mssql/demo/config/samples/loader/JobConfig.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
jobId: ${jobId}
2+
producerConfig:
3+
producerId: RDB_INITIAL_LOAD_PRODUCER
4+
connectionId: RDBConnection
5+
batchSize: 5000
6+
startRecord: ${startRecord}
7+
endRecord: ${endRecord}
8+
loadQuery: ${loadQuery}
9+
table: ${table}
10+
pipelineConfig:
11+
eventTranslator: "RDB_RS_TO_OP_TRANSLATOR"
12+
checkpointConfig:
13+
providerId: REDIS_HASH_CHECKPOINT_READER
14+
connectionId: targetConnection
15+
checkpoint: "${jobId}"
16+
stages:
17+
HashWriteStage:
18+
handlerId: REDIS_HASH_WRITER
19+
connectionId: targetConnection
20+
metricsEnabled: false
21+
prependTableNameToKeys: true
22+
deleteOnKeyUpdate: true
23+
async: true
24+
CheckpointStage:
25+
handlerId: REDIS_HASH_CHECKPOINT_WRITER
26+
connectionId: targetConnection
27+
metricEnabled: false
28+
async: true
29+
checkpoint: "${jobId}"
30+
31+

‎connectors/mssql/demo/config/samples/loader/JobManager.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
connectionId: jobConfigConnection
2+
metricsReporter:
3+
- REDIS_TS_METRICS_REPORTER

‎connectors/mssql/demo/config/samples/loader/Setup.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
connectionId: jobConfigConnection
2+
job:
3+
jobConfig:
4+
- name: initial_load
5+
config: TaskCreator.yml
6+
7+

‎connectors/mssql/demo/config/samples/loader/TaskCreator.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
jobId: ${jobId}
2+
jobType: TASK_CREATOR
3+
producerConfig:
4+
producerId: RDB_TASK_CREATOR # internal id
5+
connectionId: RDBConnection # source connectionId same as env.yml
6+
jobs:
7+
- table: public.emp # schema.table
8+
numJobs: 5
9+
config: JobConfig.yml
10+
loadQuery: "select * from (select Row_Number() over (order by (select 0)) as RowIndex, * from public.emp) as emps Where emps.RowIndex >= ${start} and emps.RowIndex <= ${end}"

‎connectors/mssql/demo/config/samples/loader/env.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
connections:
2+
- id: jobConfigConnection
3+
type: Redis
4+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
5+
- id: targetConnection
6+
type: Redis
7+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14000
8+
- id: metricsConnection
9+
type: Redis
10+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
11+
- id: RDBConnection
12+
type: RDB
13+
name: RedisConnect #database pool name
14+
database: RedisConnect #database
15+
url: "jdbc:sqlserver://127.0.0.1:1433;database=RedisConnect"
16+
host: 127.0.0.1
17+
port: 1433
18+
username: ${REDISCONNECT_SOURCE_USERNAME}
19+
password: ${REDISCONNECT_SOURCE_PASSWORD}

‎connectors/mssql/demo/config/samples/loader/mappers/mapper1.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
schema: public
2+
tables:
3+
- table: emp
4+
mapper:
5+
id: Test
6+
processorID: Test
7+
publishBefore: false
8+
passThrough: false
9+
columns:
10+
- src: empno
11+
target: EmployeeNumber
12+
- src: fname
13+
target: FirstName
14+
- src: lname
15+
target: LastName
16+
- src: job
17+
target: Job
18+
- src: mgr
19+
target: Manager
20+
- src: hiredate
21+
target: HireDate
22+
- src: sal
23+
target: Salary
24+
- src: comm
25+
target: Commission
26+
- src: dept
27+
target: Department

0 commit comments

Comments
 (0)