Merge pull request #1157 from PapaPiya/fix_elastic_magic

1.es修复增量采集相关问题；2.修复根据魔法变量切换索引无法继续采集问题

Author: liukai2012

reader/config/config.go

@@ -945,7 +945,7 @@ var ModeKeyOptions = map[string][]Option{
 			Required:     true,
 			DefaultNoUse: false,
 			Description:  "offset保存时间(es_keepalive)",
-			CheckRegex:   "\\d+[dms]",
+			CheckRegex:   "\\d+[dhms]",
 			Advance:      true,
 			ToolTip:      "logkit重启后可以继续读取ES数据的Offset记录在es服务端保存的时长，默认1d",
 		},
@@ -962,22 +962,63 @@ var ModeKeyOptions = map[string][]Option{
 		{
 			KeyName:      KeyESCron,
 			ChooseOnly:   false,
-			Default:      "loop 3s",
+			Default:      "",
 			Placeholder:  "00 00 04 * * *",
 			DefaultNoUse: false,
 			Description:  "定时任务(es_cron)",
 			Advance:      true,
-			ToolTip:      `定时任务触发周期，直接写"loop"循环执行，crontab的写法，类似于* * * * * *，对应的是秒(0~59)，分(0~59)，时(0~23)，日(1~31)，月(1-12)，星期(0~6)，填*号表示所有遍历都执行,loop模式为全量读取，cron模式为增量读取`,
+			ToolTip:      `定时任务触发周期，循环周期不宜太小，会有es中scrollId被快速消耗完的风险，直接写"loop"循环执行，crontab的写法，类似于* * * * * *，对应的是秒(0~59)，分(0~59)，时(0~23)，日(1~31)，月(1-12)，星期(0~6)，填*号表示所有遍历都执行`,
+		},
+		{
+			KeyName:      KeyESOffsetKey,
+			ChooseOnly:   false,
+			Default:      "",
+			Placeholder:  "递增字段的字段名",
+			DefaultNoUse: false,
+			Description:  "递增字段的字段名(es_cron_offset)",
+			Advance:      true,
+			ToolTip:      `定时读取时作为offset字段的字段名，只支持对整型字段进行增量采集`,
+		},
+		{
+			KeyName:       KeyESOffsetKeyType,
+			ChooseOnly:    true,
+			ChooseOptions: []interface{}{"long", "date"},
+			Default:       "long",
+			Placeholder:   "递增字段的类型",
+			DefaultNoUse:  false,
+			Description:   "递增字段的类型(es_offset_key_type)",
+			Advance:       true,
+			ToolTip:       `目前只支持对整型和时间类型的字段进行增量采集`,
 		},
 		{
-			KeyName:      KeyESCronOffset,
+			KeyName:      KeyESOffsetStartTime,
 			ChooseOnly:   false,
 			Default:      "",
-			Placeholder:  "定时读取时作为offset字段的字段名",
+			Placeholder:  "递增字段的起始值",
 			DefaultNoUse: false,
-			Description:  "定时任务offset字段字段名(es_cron_offset)",
+			Description:  "递增字段的起始值(es_offset_start_time)",
 			Advance:      true,
-			ToolTip:      `定时读取时作为offset字段的字段名，该字段只在es_cron中使用的是定时任务时生效,且此字段应该具有作为offset字段的特性(递增，例如:timestamp字段)`,
+			ToolTip:      `定时读取时作为offset字段的起始值，默认为空`,
+		},
+		{
+			KeyName:      KeyESDelayTime,
+			ChooseOnly:   false,
+			Default:      "",
+			DefaultNoUse: false,
+			Description:  "采集延迟时间(" + KeyESDelayTime + ")",
+			Advance:      true,
+			ToolTip:      `只采集比当前时间延迟一段时间的数据，以保证数据完全写入，如果递增列为整型，该字段也需要为整型，为整型时需要填写时间戳单位，延迟时间与递增列单位一致；如果是递增的时间字段，请按照'now-1d'的格式填写；为整型时，默认查询到最新位置，为时间类型时，默认查询到now+1y时间`,
+		},
+		{
+			KeyName:       KeyESDelayTimeUnit,
+			ChooseOnly:    true,
+			ChooseOptions: []interface{}{"second", "millisecond"},
+			Default:       "second",
+			Placeholder:   "",
+			DefaultNoUse:  false,
+			Description:   "延迟时间单位(" + KeyESDelayTimeUnit + ")",
+			Advance:       true,
+			ToolTip:       `延迟时间单位，默认为秒`,
 		},
 		{
 			KeyName:       KeyESDateShift,

reader/config/models.go

@@ -94,17 +94,21 @@ const (
 	KeyPGsqlCron        = "postgres_cron"
 	KeyPGsqlExecOnStart = "postgres_exec_onstart"
 
-	KeyESReadBatch   = "es_limit_batch"
-	KeyESIndex       = "es_index"
-	KeyESType        = "es_type"
-	KeyESHost        = "es_host"
-	KeyESKeepAlive   = "es_keepalive"
-	KeyESVersion     = "es_version"
-	KeyESCron        = "es_cron"
-	KeyESExecOnstart = "es_exec_onstart"
-	KeyESCronOffset  = "es_cron_offset"
-	KeyESDateShift   = "es_date_shift"
-	KeyESDateOffset  = "es_date_offset"
+	KeyESReadBatch       = "es_limit_batch"
+	KeyESIndex           = "es_index"
+	KeyESType            = "es_type"
+	KeyESHost            = "es_host"
+	KeyESKeepAlive       = "es_keepalive"
+	KeyESVersion         = "es_version"
+	KeyESCron            = "es_cron"
+	KeyESExecOnstart     = "es_exec_onstart"
+	KeyESOffsetKey       = "es_cron_offset"
+	KeyESOffsetKeyType   = "es_offset_key_type"
+	KeyESOffsetStartTime = "es_offset_start_time"
+	KeyESDateShift       = "es_date_shift"
+	KeyESDateOffset      = "es_date_offset"
+	KeyESDelayTime       = "es_delay_time"
+	KeyESDelayTimeUnit   = "es_delay_time_unit"
 
 	KeyMongoHost        = "mongo_host"
 	KeyMongoDatabase    = "mongo_database"
@@ -126,7 +130,7 @@ const (
 
 	KeyScriptParams        = "script_params"
 	KeyScriptParamsSpliter = "script_params_spliter"
-	KeyScriptTimeout      = "script_timeout"
+	KeyScriptTimeout       = "script_timeout"
 	KeyScriptContent       = "script_content"
 	KeyExecInterpreter     = "script_exec_interprepter"
 	KeyScriptCron          = "script_cron"