MySQL demo initial updates

Author: viragtripathi

connectors/db2/demo/config/samples/loader/JobConfig.yml

@@ -1,28 +1,28 @@
 jobId: ${jobId}
 producerConfig:
-  producerId: RDB_BATCH_EVENT_PRODUCER
+  producerId: RDB_INITIAL_LOAD_PRODUCER
   connectionId: RDBConnection
   batchSize: 5000
   startRecord: ${startRecord}
   endRecord: ${endRecord}
   loadQuery: ${loadQuery}
   table: ${table}
 pipelineConfig:
-  bufferSize: 1024
   eventTranslator: "RDB_RS_TO_OP_TRANSLATOR"
   checkpointConfig:
-    providerId: HASH_CHECKPOINT_READER
+    providerId: REDIS_HASH_CHECKPOINT_READER
     connectionId: targetConnection
     checkpoint: "${jobId}"
   stages:
     HashWriteStage:
       handlerId: REDIS_HASH_WRITER
       connectionId: targetConnection
+      metricsEnabled: false
       prependTableNameToKeys: true
       deleteOnKeyUpdate: true
       async: true
     CheckpointStage:
-      handlerId: REDIS_OP_CP_WRITER
+      handlerId: REDIS_HASH_CHECKPOINT_WRITER
       connectionId: targetConnection
       metricEnabled: false
       async: true

connectors/db2/demo/config/samples/loader/JobManager.yml

@@ -1,21 +1,3 @@
 connectionId: jobConfigConnection
-jobTypeId: empLoader
-jobStream: loaderJobStream
-jobConfigSet: loaderJobConfigs
-completedJobsSet: loaderCompletedJobs
-initialDelay: 10000
-numManagementThreads: 2
-heartBeatConfig:
-  key: hb-loaderJobManager
-  expiry: 30000
-jobHeartBeatKeyPrefix: "hb-job:"
-jobHeartbeatCheckInterval: 45000
-jobClaimerConfig:
-  initialDelay: 10000
-  claimInterval: 30000
-  heartBeatConfig:
-    key: "hb-job:"
-    expiry: 30000
-  maxNumberOfJobs: 5
-  consumerGroup: loaderJobGroup
-  batchSize: 3
+metricsReporter:
+  - REDIS_TS_METRICS_REPORTER

connectors/db2/demo/config/samples/loader/Setup.yml

@@ -1,9 +1,5 @@
 connectionId: jobConfigConnection
 job:
-  stream: loaderJobStream
-  configSet: loaderJobConfigs
-  completedJobsSet: loadercompletedJobs
-  consumerGroup: loaderJobGroup
   jobConfig:
-    - name: batchtaskcreator
+    - name: initial_load
       config: TaskCreator.yml

connectors/db2/demo/config/samples/loader/TaskCreator.yml

@@ -5,7 +5,6 @@ producerConfig:
   connectionId: RDBConnection
   jobs:
     - table: jjd47182.emp # schema.table
-      jobPrefix: empLoader-
       numJobs: 5
       config: JobConfig.yml
       loadQuery: "select * from (select ROW_NUMBER() OVER (ORDER BY empno) AS RowIndex, * from jjd47182.emp) AS emps Where emps.RowIndex >= ${start} and emps.RowIndex <= ${end}"

connectors/db2/demo/config/samples/loader/env.yml

@@ -1,18 +1,19 @@
 connections:
-  jobConfigConnection:
-    redisUrl: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
-  targetConnection:
-    redisUrl: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14000
-  metricsConnection:
-    redisUrl: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
-  RDBConnection:
-    database:
-      name: loaderdb #database pool name
-      db: BLUDB #database
-      hostname: dashdb-txn-sbox-yp-dal09-11.services.dal.bluemix.net
-      port: 50000
-      username: ${REDISCONNECT_SOURCE_USERNAME}
-      password: ${REDISCONNECT_SOURCE_PASSWORD}
-      jdbcUrl: "jdbc:db2://dashdb-txn-sbox-yp-dal09-11.services.dal.bluemix.net:50000/BLUDB"
-      maximumPoolSize: 10
-      minimumIdle: 2
+  - id: jobConfigConnection
+    type: Redis
+    url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
+  - id: targetConnection
+    type: Redis
+    url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14000
+  - id: metricsConnection
+    type: Redis
+    url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
+  - id: RDBConnection
+    type: RDB
+    name: RedisConnect #database pool name
+    database: BLUDB #database
+    url: "jdbc:db2://dashdb-txn-sbox-yp-dal09-11.services.dal.bluemix.net:50000/BLUDB"
+    host: dashdb-txn-sbox-yp-dal09-11.services.dal.bluemix.net
+    port: 50000
+    username: ${REDISCONNECT_SOURCE_USERNAME}
+    password: ${REDISCONNECT_SOURCE_PASSWORD}

connectors/mssql/README.md

@@ -418,8 +418,8 @@ cli: starts redis-connect-cli.
 stage: clean and stage redis database with cdc or initial loader job configurations.
 start: start Redis Connect instance with provided cdc or initial loader job configurations.
 -------------------------------
-    
 ```
+
 </p>
 </details>
 

connectors/mssql/demo/README.md

@@ -97,9 +97,6 @@ The above script will create a 1-node Redis Enterprise cluster in a docker conta
 
 ---
 
-## Setup RedisInsight
-Open a web browser and navigate to http://127.0.0.1:18001/ and [add both job config & metrics and target Redis databbases](https://docs.redislabs.com/latest/ri/using-redisinsight/add-instance/) (use redisUrl's from env.yml) to RedisInsight UI. Use Redis database endpoints for job management and target databasees, use the `Internal IP` instead of `127.0.0.1` on cloud machines.
-
 ## Start Redis Connect SQL Server Connector
 
 <details><summary>Run Redis Connect SQL Server Connector docker container to see all the options</summary>

connectors/mssql/demo/config/samples/loader/Setup.yml

@@ -3,5 +3,3 @@ job:
   jobConfig:
     - name: initial_load
       config: TaskCreator.yml
-
-

connectors/mssql/demo/config/samples/sqlserver/JobConfig.yml

@@ -4,7 +4,6 @@ producerConfig:
   connectionId: RDBConnection
   tables:
     - dbo.emp #schema.table
-  metricsKey: RedisConnect-emp
   metricsEnabled: false
 pipelineConfig:
   eventTranslator: "${sourceValueTranslator}"

connectors/mssql/demo/config/samples/sqlserver/Setup.yml

@@ -9,7 +9,7 @@ job:
         labels:
           schema: dbo
           table: emp
-          op: I
+          op: C
       - key: "dbo:emp:U:Throughput"
         retentionInHours: 4
         labels: