From 31a04ece61d723a739f8427f7420cfc41066a8f1 Mon Sep 17 00:00:00 2001 From: Liqi Geng Date: Wed, 13 May 2020 22:56:09 +0800 Subject: [PATCH 1/8] add write stress Signed-off-by: Liqi Geng --- Makefile | 5 +- cmd/write-stress/main.go | 61 +++++++++ tests/write-stress/write_stress.go | 207 +++++++++++++++++++++++++++++ 3 files changed, 272 insertions(+), 1 deletion(-) create mode 100644 cmd/write-stress/main.go create mode 100644 tests/write-stress/write_stress.go diff --git a/Makefile b/Makefile index dee65ce0..fa4a7a3f 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ default: tidy fmt lint build build: tidb pocket tpcc ledger txn-rand-pessimistic on-dup sqllogic block-writer \ region-available deadlock-detector crud bank bank2 abtest cdc-pocket tiflash-pocket vbank \ - read-stress rawkv-linearizability tiflash-abtest tiflash-cdc follower-read + read-stress rawkv-linearizability tiflash-abtest tiflash-cdc follower-read write-stress tidb: $(GOBUILD) $(GOMOD) -o bin/chaos-tidb cmd/tidb/main.go @@ -97,6 +97,9 @@ tiflash-cdc: follower-read: $(GOBUILD) $(GOMOD) -o bin/follower-read cmd/follower-read/*.go +write-stress: + $(GOBUILD) $(GOMOD) -o bin/write-stress cmd/write-stress/*.go + fmt: groupimports go fmt ./... diff --git a/cmd/write-stress/main.go b/cmd/write-stress/main.go new file mode 100644 index 00000000..da7cac18 --- /dev/null +++ b/cmd/write-stress/main.go @@ -0,0 +1,61 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "flag" + + // use mysql + _ "github.com/go-sql-driver/mysql" + + test_infra "github.com/pingcap/tipocket/pkg/test-infra" + writestress "github.com/pingcap/tipocket/tests/write-stress" + + "github.com/pingcap/tipocket/cmd/util" + "github.com/pingcap/tipocket/pkg/cluster" + "github.com/pingcap/tipocket/pkg/control" + "github.com/pingcap/tipocket/pkg/test-infra/fixture" +) + +var ( + dataNum = flag.Int("dataNum", 2000, "the number of data（the unit is 10 thoudstand)") + concurrency = flag.Int("concurrency", 400, "concurrency of worker") + batch = flag.Int("batch", 100, "batch of insert sql") +) + +func main() { + flag.Parse() + cfg := control.Config{ + Mode: control.ModeSelfScheduled, + ClientCount: 1, + RunTime: fixture.Context.RunTime, + RunRound: 1, + } + kvs := []string{"127.0.0.1:20160", "127.0.0.1:20162", "127.0.0.1:20161"} + suit := util.Suit{ + Config: &cfg, + //Provisioner: cluster.NewK8sProvisioner(), + Provisioner: cluster.NewLocalClusterProvisioner([]string{"127.0.0.1:4000"}, []string{"127.0.0.1:2379"}, kvs), + ClientCreator: writestress.ClientCreator{Cfg: &writestress.Config{ + DataNum: *dataNum, + Concurrency: *concurrency, + Batch: *batch, + }}, + NemesisGens: util.ParseNemesisGenerators(fixture.Context.Nemesis), + ClusterDefs: test_infra.NewDefaultCluster(fixture.Context.Namespace, fixture.Context.Namespace, + fixture.Context.TiDBClusterConfig), + } + suit.Run(context.Background()) +} diff --git a/tests/write-stress/write_stress.go b/tests/write-stress/write_stress.go new file mode 100644 index 00000000..0db83d5c --- /dev/null +++ b/tests/write-stress/write_stress.go @@ -0,0 +1,207 @@ +package writestress + +import ( + "context" + "database/sql" + "encoding/binary" + "fmt" + "math/rand" + "sync" + "time" + + "github.com/juju/errors" + "github.com/ngaut/log" + + "github.com/pingcap/tipocket/pkg/cluster/types" + "github.com/pingcap/tipocket/pkg/core" + "github.com/pingcap/tipocket/util" +) + +// Table schema comes from the bank of pufa `tmp_jieb_instmnt_daily` +// CREATE TABLE `tmp_jieb_instmnt_daily` ( +// `ID` bigint(20) DEFAULT NULL COMMENT '主键ID', +// `TABLE_ID` int(11) NOT NULL COMMENT '分库ID', +// `FILE_DATE` char(8) NOT NULL COMMENT '文件日期', +// `CONTRACT_NO` varchar(128) NOT NULL COMMENT '借据号', +// `SETTLE_DATE` char(8) NOT NULL COMMENT '减免会计日期', +// `TERM_NO` int(11) NOT NULL COMMENT '期次号', +// +// `INPT_DATE` char(8) DEFAULT NULL COMMENT '录入日期', +// `INPT_TIME` varchar(20) DEFAULT NULL COMMENT '录入时间', +// `RCRD_ST_CODE` varchar(1) DEFAULT NULL COMMENT '记录状态代码', +// UNIQUE KEY `TMP_JIEB_INSTMNT_DAILY_IDX1` (`CONTRACT_NO`,`TERM_NO`), +// KEY `TMP_JIEB_INSTMNT_DAILY_IDX2` (`TABLE_ID`,`CONTRACT_NO`) +// ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin/*!90000 SHARD_ROW_ID_BITS=5 PRE_SPLIT_REGIONS=5 */ COMMENT='借呗日终（分期）信息临时表'; +const ( + stmtDrop = `DROP TABLE IF EXISTS write_stress` + stmtCreate = ` + CREATE TABLE write_stress ( + TABLE_ID int(11) NOT NULL COMMENT '分库ID', + CONTRACT_NO varchar(128) NOT NULL COMMENT '借据号', + TERM_NO int(11) NOT NULL COMMENT '期次号', + NOUSE char(60) NOT NULL COMMENT '填充位', + + UNIQUE KEY TMP_JIEB_INSTMNT_DAILY_IDX1 (CONTRACT_NO, TERM_NO), + KEY TMP_JIEB_INSTMNT_DAILY_IDX2 (TABLE_ID, CONTRACT_NO) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +` +) + +// Config is for writestressClient +type Config struct { + DataNum int `toml:"dataNum"` + Concurrency int `toml:"concurrency"` + Batch int `toml:"batch"` +} + +// ClientCreator creates writestressClient +type ClientCreator struct { + Cfg *Config +} + +// Create ... +func (l ClientCreator) Create(node types.ClientNode) core.Client { + return &writestressClient{ + Config: l.Cfg, + } +} + +// ledgerClient simulates a complete record of financial transactions over the +// life of a bank (or other company). +type writestressClient struct { + *Config + db *sql.DB + contract_ids [][]byte +} + +func (c *writestressClient) SetUp(ctx context.Context, nodes []types.ClientNode, idx int) error { + if idx != 0 { + return nil + } + + var err error + node := nodes[idx] + dsn := fmt.Sprintf("root@tcp(%s:%d)/test", node.IP, node.Port) + + log.Infof("start to init...") + c.db, err = util.OpenDB(dsn, c.Concurrency) + if err != nil { + return err + } + defer func() { + log.Infof("init end...") + }() + + if _, err := c.db.Exec(stmtDrop); err != nil { + log.Fatalf("execute statement %s error %v", stmtDrop, err) + } + + if _, err := c.db.Exec(stmtCreate); err != nil { + log.Fatalf("execute statement %s error %v", stmtCreate, err) + } + + return nil +} + +func (c *writestressClient) TearDown(ctx context.Context, nodes []types.ClientNode, idx int) error { + return nil +} + +func (c *writestressClient) Invoke(ctx context.Context, node types.ClientNode, r interface{}) core.UnknownResponse { + panic("implement me") +} + +func (c *writestressClient) NextRequest() interface{} { + panic("implement me") +} + +func (c *writestressClient) DumpState(ctx context.Context) (interface{}, error) { + panic("implement me") +} + +func (c *writestressClient) Start(ctx context.Context, cfg interface{}, clientNodes []types.ClientNode) error { + log.Infof("start to test...") + defer func() { + log.Infof("test end...") + }() + c.contract_ids = make([][]byte, c.DataNum) + timeUnix := time.Now().Unix() + count := uint8(0) + b := make([]byte, 8) + for i := 0; i < c.DataNum; i++ { + // "abcd" + timestamp(8 bit) + count(8 bit) + c.contract_ids[i] = append(c.contract_ids[i], []byte("abcd")...) + binary.LittleEndian.PutUint64(b, uint64(timeUnix)) + c.contract_ids[i] = append(c.contract_ids[i], b...) + binary.LittleEndian.PutUint64(b, uint64(count)) + c.contract_ids[i] = append(c.contract_ids[i], b...) + + count++ + if count == 0 { + timeUnix++ + } + } + + var wg sync.WaitGroup + for i := 0; i < c.Concurrency; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + for { + select { + case <-ctx.Done(): + return + default: + } + if err := c.ExecuteInsert(c.db, i); err != nil { + log.Fatalf("exec failed %v", err) + } + } + }(i) + } + + wg.Wait() + return nil +} + +// ExecuteInsert is run case +func (c *writestressClient) ExecuteInsert(db *sql.DB, pos int) error { + num := c.Config.DataNum * 10000 / c.Config.Concurrency + + tx, err := db.Begin() + if err != nil { + return errors.Trace(err) + } + defer tx.Rollback() + str := make([]byte, 50) + rnd := rand.New(rand.NewSource(time.Now().Unix())) + for i := 0; i < num/c.Config.Batch; i++ { + n := num*pos + i*c.Config.Batch + if n >= c.DataNum { + break + } + query := fmt.Sprintf(`INSERT INTO write_stress (TABLE_ID, CONTRACT_NO, TERM_NO, NOUSE) VALUES `) + for j := 0; j < c.Config.Batch; j++ { + n := num*pos + i*c.Config.Batch + j + if n >= c.DataNum { + break + } + contract_id := c.contract_ids[n] + util.RandString(str, rnd) + if j != 0 { + query += "," + } + query += fmt.Sprintf(`(%v, %v, %v, %v)`, rnd.Uint32()%960+1, string(contract_id[:]), rnd.Uint32()%36+1, string(str[:])) + } + fmt.Println(query) + if _, err := tx.Exec(query); err != nil { + return errors.Trace(err) + } + } + + if err := tx.Commit(); err != nil { + return errors.Trace(err) + } + + return nil +} From dfa2ab7a4e754b3b4d14f48089b71ac5f092b039 Mon Sep 17 00:00:00 2001 From: Liqi Geng Date: Wed, 13 May 2020 23:37:05 +0800 Subject: [PATCH 2/8] change sql Signed-off-by: Liqi Geng --- cmd/write-stress/main.go | 8 ++-- tests/write-stress/write_stress.go | 69 +++++++++++++----------------- 2 files changed, 34 insertions(+), 43 deletions(-) diff --git a/cmd/write-stress/main.go b/cmd/write-stress/main.go index da7cac18..1563da68 100644 --- a/cmd/write-stress/main.go +++ b/cmd/write-stress/main.go @@ -43,11 +43,11 @@ func main() { RunTime: fixture.Context.RunTime, RunRound: 1, } - kvs := []string{"127.0.0.1:20160", "127.0.0.1:20162", "127.0.0.1:20161"} + //kvs := []string{"127.0.0.1:20160", "127.0.0.1:20162", "127.0.0.1:20161"} suit := util.Suit{ - Config: &cfg, - //Provisioner: cluster.NewK8sProvisioner(), - Provisioner: cluster.NewLocalClusterProvisioner([]string{"127.0.0.1:4000"}, []string{"127.0.0.1:2379"}, kvs), + Config: &cfg, + Provisioner: cluster.NewK8sProvisioner(), + //Provisioner: cluster.NewLocalClusterProvisioner([]string{"127.0.0.1:4000"}, []string{"127.0.0.1:2379"}, kvs), ClientCreator: writestress.ClientCreator{Cfg: &writestress.Config{ DataNum: *dataNum, Concurrency: *concurrency, diff --git a/tests/write-stress/write_stress.go b/tests/write-stress/write_stress.go index 0db83d5c..c12fb35f 100644 --- a/tests/write-stress/write_stress.go +++ b/tests/write-stress/write_stress.go @@ -3,9 +3,9 @@ package writestress import ( "context" "database/sql" - "encoding/binary" "fmt" "math/rand" + "strconv" "sync" "time" @@ -124,21 +124,21 @@ func (c *writestressClient) Start(ctx context.Context, cfg interface{}, clientNo defer func() { log.Infof("test end...") }() - c.contract_ids = make([][]byte, c.DataNum) + totalNum := c.DataNum * 10000 + c.contract_ids = make([][]byte, totalNum) timeUnix := time.Now().Unix() - count := uint8(0) - b := make([]byte, 8) - for i := 0; i < c.DataNum; i++ { - // "abcd" + timestamp(8 bit) + count(8 bit) + count := 0 + for i := 0; i < totalNum; i++ { + // "abcd" + timestamp + count c.contract_ids[i] = append(c.contract_ids[i], []byte("abcd")...) - binary.LittleEndian.PutUint64(b, uint64(timeUnix)) - c.contract_ids[i] = append(c.contract_ids[i], b...) - binary.LittleEndian.PutUint64(b, uint64(count)) - c.contract_ids[i] = append(c.contract_ids[i], b...) + tm := time.Unix(timeUnix, 0) + c.contract_ids[i] = append(c.contract_ids[i], tm.String()...) + c.contract_ids[i] = append(c.contract_ids[i], strconv.Itoa(count)...) count++ - if count == 0 { + if count%200 == 0 { timeUnix++ + count = 0 } } @@ -147,15 +147,8 @@ func (c *writestressClient) Start(ctx context.Context, cfg interface{}, clientNo wg.Add(1) go func(i int) { defer wg.Done() - for { - select { - case <-ctx.Done(): - return - default: - } - if err := c.ExecuteInsert(c.db, i); err != nil { - log.Fatalf("exec failed %v", err) - } + if err := c.ExecuteInsert(c.db, i); err != nil { + log.Fatalf("exec failed %v", err) } }(i) } @@ -166,24 +159,23 @@ func (c *writestressClient) Start(ctx context.Context, cfg interface{}, clientNo // ExecuteInsert is run case func (c *writestressClient) ExecuteInsert(db *sql.DB, pos int) error { - num := c.Config.DataNum * 10000 / c.Config.Concurrency - - tx, err := db.Begin() - if err != nil { - return errors.Trace(err) - } - defer tx.Rollback() + totalNum := c.DataNum * 10000 + num := totalNum / c.Concurrency str := make([]byte, 50) rnd := rand.New(rand.NewSource(time.Now().Unix())) - for i := 0; i < num/c.Config.Batch; i++ { - n := num*pos + i*c.Config.Batch - if n >= c.DataNum { + for i := 0; i < num/c.Batch; i++ { + tx, err := db.Begin() + if err != nil { + return errors.Trace(err) + } + n := num*pos + i*c.Batch + if n >= totalNum { break } query := fmt.Sprintf(`INSERT INTO write_stress (TABLE_ID, CONTRACT_NO, TERM_NO, NOUSE) VALUES `) - for j := 0; j < c.Config.Batch; j++ { - n := num*pos + i*c.Config.Batch + j - if n >= c.DataNum { + for j := 0; j < c.Batch; j++ { + n := num*pos + i*c.Batch + j + if n >= totalNum { break } contract_id := c.contract_ids[n] @@ -191,16 +183,15 @@ func (c *writestressClient) ExecuteInsert(db *sql.DB, pos int) error { if j != 0 { query += "," } - query += fmt.Sprintf(`(%v, %v, %v, %v)`, rnd.Uint32()%960+1, string(contract_id[:]), rnd.Uint32()%36+1, string(str[:])) + query += fmt.Sprintf(`(%v, "%v", %v, "%v")`, rnd.Uint32()%960+1, string(contract_id[:]), rnd.Uint32()%36+1, string(str[:])) } - fmt.Println(query) + //fmt.Println(query) if _, err := tx.Exec(query); err != nil { return errors.Trace(err) } - } - - if err := tx.Commit(); err != nil { - return errors.Trace(err) + if err := tx.Commit(); err != nil { + return errors.Trace(err) + } } return nil From 8bc5f0ca5ca3d3ac0a2e01e9e6050c76aadeab7f Mon Sep 17 00:00:00 2001 From: Liqi Geng Date: Thu, 14 May 2020 11:52:15 +0800 Subject: [PATCH 3/8] make data larger Signed-off-by: Liqi Geng --- cmd/write-stress/main.go | 2 +- tests/write-stress/write_stress.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/write-stress/main.go b/cmd/write-stress/main.go index 1563da68..2a313fcf 100644 --- a/cmd/write-stress/main.go +++ b/cmd/write-stress/main.go @@ -30,7 +30,7 @@ import ( ) var ( - dataNum = flag.Int("dataNum", 2000, "the number of data（the unit is 10 thoudstand)") + dataNum = flag.Int("dataNum", 10000, "the number of data（the unit is 10 thoudstand)") concurrency = flag.Int("concurrency", 400, "concurrency of worker") batch = flag.Int("batch", 100, "batch of insert sql") ) diff --git a/tests/write-stress/write_stress.go b/tests/write-stress/write_stress.go index c12fb35f..82796819 100644 --- a/tests/write-stress/write_stress.go +++ b/tests/write-stress/write_stress.go @@ -39,7 +39,7 @@ const ( TABLE_ID int(11) NOT NULL COMMENT '分库ID', CONTRACT_NO varchar(128) NOT NULL COMMENT '借据号', TERM_NO int(11) NOT NULL COMMENT '期次号', - NOUSE char(60) NOT NULL COMMENT '填充位', + NOUSE char(510) NOT NULL COMMENT '填充位', UNIQUE KEY TMP_JIEB_INSTMNT_DAILY_IDX1 (CONTRACT_NO, TERM_NO), KEY TMP_JIEB_INSTMNT_DAILY_IDX2 (TABLE_ID, CONTRACT_NO) @@ -161,7 +161,7 @@ func (c *writestressClient) Start(ctx context.Context, cfg interface{}, clientNo func (c *writestressClient) ExecuteInsert(db *sql.DB, pos int) error { totalNum := c.DataNum * 10000 num := totalNum / c.Concurrency - str := make([]byte, 50) + str := make([]byte, 500) rnd := rand.New(rand.NewSource(time.Now().Unix())) for i := 0; i < num/c.Batch; i++ { tx, err := db.Begin() From f1b2dde2d949ab9e376f76f6e73609ff255c5367 Mon Sep 17 00:00:00 2001 From: Liqi Geng Date: Thu, 14 May 2020 13:04:19 +0800 Subject: [PATCH 4/8] make data larger Signed-off-by: Liqi Geng --- tests/write-stress/write_stress.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/write-stress/write_stress.go b/tests/write-stress/write_stress.go index 82796819..9f959b11 100644 --- a/tests/write-stress/write_stress.go +++ b/tests/write-stress/write_stress.go @@ -39,7 +39,7 @@ const ( TABLE_ID int(11) NOT NULL COMMENT '分库ID', CONTRACT_NO varchar(128) NOT NULL COMMENT '借据号', TERM_NO int(11) NOT NULL COMMENT '期次号', - NOUSE char(510) NOT NULL COMMENT '填充位', + NOUSE char(255) NOT NULL COMMENT '填充位', UNIQUE KEY TMP_JIEB_INSTMNT_DAILY_IDX1 (CONTRACT_NO, TERM_NO), KEY TMP_JIEB_INSTMNT_DAILY_IDX2 (TABLE_ID, CONTRACT_NO) @@ -161,7 +161,7 @@ func (c *writestressClient) Start(ctx context.Context, cfg interface{}, clientNo func (c *writestressClient) ExecuteInsert(db *sql.DB, pos int) error { totalNum := c.DataNum * 10000 num := totalNum / c.Concurrency - str := make([]byte, 500) + str := make([]byte, 250) rnd := rand.New(rand.NewSource(time.Now().Unix())) for i := 0; i < num/c.Batch; i++ { tx, err := db.Begin() From 3b6245448b297d9be900c63b4221a720e2a8fbeb Mon Sep 17 00:00:00 2001 From: Liqi Geng Date: Thu, 14 May 2020 20:49:12 +0800 Subject: [PATCH 5/8] change data generater Signed-off-by: Liqi Geng --- tests/write-stress/write_stress.go | 56 ++++++++++++++++++------------ 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/tests/write-stress/write_stress.go b/tests/write-stress/write_stress.go index 9f959b11..4aab6f43 100644 --- a/tests/write-stress/write_stress.go +++ b/tests/write-stress/write_stress.go @@ -70,8 +70,9 @@ func (l ClientCreator) Create(node types.ClientNode) core.Client { // life of a bank (or other company). type writestressClient struct { *Config - db *sql.DB - contract_ids [][]byte + db *sql.DB + timeUnix int64 + rnd *rand.Rand } func (c *writestressClient) SetUp(ctx context.Context, nodes []types.ClientNode, idx int) error { @@ -124,23 +125,8 @@ func (c *writestressClient) Start(ctx context.Context, cfg interface{}, clientNo defer func() { log.Infof("test end...") }() - totalNum := c.DataNum * 10000 - c.contract_ids = make([][]byte, totalNum) - timeUnix := time.Now().Unix() - count := 0 - for i := 0; i < totalNum; i++ { - // "abcd" + timestamp + count - c.contract_ids[i] = append(c.contract_ids[i], []byte("abcd")...) - tm := time.Unix(timeUnix, 0) - c.contract_ids[i] = append(c.contract_ids[i], tm.String()...) - c.contract_ids[i] = append(c.contract_ids[i], strconv.Itoa(count)...) - - count++ - if count%200 == 0 { - timeUnix++ - count = 0 - } - } + + c.rnd = rand.New(rand.NewSource(time.Now().Unix())) var wg sync.WaitGroup for i := 0; i < c.Concurrency; i++ { @@ -162,7 +148,16 @@ func (c *writestressClient) ExecuteInsert(db *sql.DB, pos int) error { totalNum := c.DataNum * 10000 num := totalNum / c.Concurrency str := make([]byte, 250) - rnd := rand.New(rand.NewSource(time.Now().Unix())) + + limit := 1000 + if num < 100 { + limit = 1 + } else if num < 1000 { + limit = 100 + } + timeUnix := c.timeUnix + int64(pos*num/limit) + nextTimeUnix := c.timeUnix + int64((pos+1)*num/limit) + count := 0 for i := 0; i < num/c.Batch; i++ { tx, err := db.Begin() if err != nil { @@ -178,12 +173,27 @@ func (c *writestressClient) ExecuteInsert(db *sql.DB, pos int) error { if n >= totalNum { break } - contract_id := c.contract_ids[n] - util.RandString(str, rnd) + // "abcd" + timestamp + count + contract_id := []byte("abcd") + tm := time.Unix(timeUnix, 0) + contract_id = append(contract_id, tm.String()...) + contract_id = append(contract_id, strconv.Itoa(count)...) + util.RandString(str, c.rnd) if j != 0 { query += "," } - query += fmt.Sprintf(`(%v, "%v", %v, "%v")`, rnd.Uint32()%960+1, string(contract_id[:]), rnd.Uint32()%36+1, string(str[:])) + + query += fmt.Sprintf(`(%v, "%v", %v, "%v")`, c.rnd.Uint32()%960+1, string(contract_id[:]), c.rnd.Uint32()%36+1, string(str[:])) + + count++ + if count%limit == 0 { + if timeUnix+1 == nextTimeUnix { + count++ + } else { + timeUnix++ + count = 0 + } + } } //fmt.Println(query) if _, err := tx.Exec(query); err != nil { From cfcdba743c11de9b6ba19ab546f20e196044d526 Mon Sep 17 00:00:00 2001 From: Liqi Geng Date: Thu, 14 May 2020 21:45:22 +0800 Subject: [PATCH 6/8] add config Signed-off-by: Liqi Geng --- config/tikv/write-stress.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 config/tikv/write-stress.toml diff --git a/config/tikv/write-stress.toml b/config/tikv/write-stress.toml new file mode 100644 index 00000000..c94b8445 --- /dev/null +++ b/config/tikv/write-stress.toml @@ -0,0 +1,3 @@ +[raftstore] +apply-pool-size = 6 +store-pool-size = 6 \ No newline at end of file From fcfd3f2945bce2e74d3d4bee1a92db0945f1d9f8 Mon Sep 17 00:00:00 2001 From: Liqi Geng Date: Thu, 14 May 2020 22:34:57 +0800 Subject: [PATCH 7/8] change rand Signed-off-by: Liqi Geng --- tests/write-stress/write_stress.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/write-stress/write_stress.go b/tests/write-stress/write_stress.go index 4aab6f43..1eaada6e 100644 --- a/tests/write-stress/write_stress.go +++ b/tests/write-stress/write_stress.go @@ -72,7 +72,6 @@ type writestressClient struct { *Config db *sql.DB timeUnix int64 - rnd *rand.Rand } func (c *writestressClient) SetUp(ctx context.Context, nodes []types.ClientNode, idx int) error { @@ -126,8 +125,6 @@ func (c *writestressClient) Start(ctx context.Context, cfg interface{}, clientNo log.Infof("test end...") }() - c.rnd = rand.New(rand.NewSource(time.Now().Unix())) - var wg sync.WaitGroup for i := 0; i < c.Concurrency; i++ { wg.Add(1) @@ -145,6 +142,8 @@ func (c *writestressClient) Start(ctx context.Context, cfg interface{}, clientNo // ExecuteInsert is run case func (c *writestressClient) ExecuteInsert(db *sql.DB, pos int) error { + rnd := rand.New(rand.NewSource(rand.Int63())) + totalNum := c.DataNum * 10000 num := totalNum / c.Concurrency str := make([]byte, 250) @@ -178,12 +177,12 @@ func (c *writestressClient) ExecuteInsert(db *sql.DB, pos int) error { tm := time.Unix(timeUnix, 0) contract_id = append(contract_id, tm.String()...) contract_id = append(contract_id, strconv.Itoa(count)...) - util.RandString(str, c.rnd) + util.RandString(str, rnd) if j != 0 { query += "," } - query += fmt.Sprintf(`(%v, "%v", %v, "%v")`, c.rnd.Uint32()%960+1, string(contract_id[:]), c.rnd.Uint32()%36+1, string(str[:])) + query += fmt.Sprintf(`(%v, "%v", %v, "%v")`, rnd.Uint32()%960+1, string(contract_id[:]), rnd.Uint32()%36+1, string(str[:])) count++ if count%limit == 0 { From 37a4fad35152c9237ea8a5637252639f10a77e3a Mon Sep 17 00:00:00 2001 From: Liqi Geng Date: Mon, 18 May 2020 17:45:26 +0800 Subject: [PATCH 8/8] remove some code Signed-off-by: Liqi Geng --- Makefile | 4 +--- tests/write-stress/write_stress.go | 23 ++++------------------- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index fa4a7a3f..e75e656c 100644 --- a/Makefile +++ b/Makefile @@ -15,9 +15,7 @@ DOCKER_REGISTRY_PREFIX := $(if $(DOCKER_REGISTRY),$(DOCKER_REGISTRY)/,) default: tidy fmt lint build -build: tidb pocket tpcc ledger txn-rand-pessimistic on-dup sqllogic block-writer \ - region-available deadlock-detector crud bank bank2 abtest cdc-pocket tiflash-pocket vbank \ - read-stress rawkv-linearizability tiflash-abtest tiflash-cdc follower-read write-stress +build: write-stress tidb: $(GOBUILD) $(GOMOD) -o bin/chaos-tidb cmd/tidb/main.go diff --git a/tests/write-stress/write_stress.go b/tests/write-stress/write_stress.go index 1eaada6e..1f41db75 100644 --- a/tests/write-stress/write_stress.go +++ b/tests/write-stress/write_stress.go @@ -17,29 +17,14 @@ import ( "github.com/pingcap/tipocket/util" ) -// Table schema comes from the bank of pufa `tmp_jieb_instmnt_daily` -// CREATE TABLE `tmp_jieb_instmnt_daily` ( -// `ID` bigint(20) DEFAULT NULL COMMENT '主键ID', -// `TABLE_ID` int(11) NOT NULL COMMENT '分库ID', -// `FILE_DATE` char(8) NOT NULL COMMENT '文件日期', -// `CONTRACT_NO` varchar(128) NOT NULL COMMENT '借据号', -// `SETTLE_DATE` char(8) NOT NULL COMMENT '减免会计日期', -// `TERM_NO` int(11) NOT NULL COMMENT '期次号', -// -// `INPT_DATE` char(8) DEFAULT NULL COMMENT '录入日期', -// `INPT_TIME` varchar(20) DEFAULT NULL COMMENT '录入时间', -// `RCRD_ST_CODE` varchar(1) DEFAULT NULL COMMENT '记录状态代码', -// UNIQUE KEY `TMP_JIEB_INSTMNT_DAILY_IDX1` (`CONTRACT_NO`,`TERM_NO`), -// KEY `TMP_JIEB_INSTMNT_DAILY_IDX2` (`TABLE_ID`,`CONTRACT_NO`) -// ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin/*!90000 SHARD_ROW_ID_BITS=5 PRE_SPLIT_REGIONS=5 */ COMMENT='借呗日终（分期）信息临时表'; const ( stmtDrop = `DROP TABLE IF EXISTS write_stress` stmtCreate = ` CREATE TABLE write_stress ( - TABLE_ID int(11) NOT NULL COMMENT '分库ID', - CONTRACT_NO varchar(128) NOT NULL COMMENT '借据号', - TERM_NO int(11) NOT NULL COMMENT '期次号', - NOUSE char(255) NOT NULL COMMENT '填充位', + TABLE_ID int(11) NOT NULL, + CONTRACT_NO varchar(128) NOT NULL, + TERM_NO int(11) NOT NULL, + NOUSE char(255) NOT NULL, UNIQUE KEY TMP_JIEB_INSTMNT_DAILY_IDX1 (CONTRACT_NO, TERM_NO), KEY TMP_JIEB_INSTMNT_DAILY_IDX2 (TABLE_ID, CONTRACT_NO)