Init commit

Author: willzhen

.gitignore

@@ -0,0 +1 @@
+go.sum

README.md

@@ -0,0 +1,35 @@
+# GO library
+- client：调用第三方服务的客户端。
+  - ti: ti 平台客户端。
+    - ti3.4
+    - ti3.7
+- httpcall：http 调用的封装。
+  - json_post.go: JsonPost。
+  - json_get.go: JsonGet。
+  - json_put.go: JsonPut。
+  - json_delete.go: JsonDelete。
+- misc：常用的公共函数库
+  - time_related：时间相关的函数
+    - time_tick.go: TimeTick -- 计时器。
+    - timed.go: Timed -- 函数计时调用。
+  - goroutine_help.go：
+    - GoroutineHelp.Recoverd：协程 recover。
+    - GoroutineHelp.SafeGo：安全的 go。
+  - retry.go： Retry -- 带重试调用。
+  - id.go：
+    - GenerateRandomString：生成随机字符串。
+    - GenerateUUID：生成UUID。
+    - BighumpToUnderscore：大驼峰参数转换成下划线。
+    - UnderscoreToBighump：下划线参数转换成大驼峰。
+  - log.og
+    - FoldLog: 折叠日志中数据过长的字段，仅支持 json。
+- stl_extension：stl 的扩展
+  - limit_waitgroup.go：LimitWaitGroup -- 对于系统 WaitGroup 的扩展，支持 limit 并发限制并且阻塞。
+  - ordered_map.go：OrderedMap -- 实现了 c++ 中的排序 map，可以按照顺序遍历所有元素。
+- text: 文本处理相关方法
+  - aho_automaton.go：ac 自动机，多模式串快速匹配。在一个文本中找出来，出现过哪些字符串子串以及其定位。可以理解对同一文本 s 多次调用 strings.Contains(s, xxx) 的加速。
+    - BuildAcTrie: 构建自动机。
+    - AcTrie.Search: 匹配。
+  - edit_distance.go：
+    - Levenshtein：文本编辑距离。
+    - TextSim：计算两个文本的相似度。

go.mod

@@ -0,0 +1,10 @@
+module github.com/memory-overflow/go-common-library
+
+go 1.16
+
+require (
+	github.com/eapache/queue v1.1.0
+	github.com/google/uuid v1.3.0
+	github.com/json-iterator/go v1.1.12
+	github.com/tidwall/gjson v1.14.4
+)

httpcall/httpcall_test.go

@@ -0,0 +1,27 @@
+package httpcall_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/memory-overflow/go-common-library/httpcall"
+	service "github.com/memory-overflow/go-common-library/task_scheduler/example/add_service"
+)
+
+func TestJsonPost(t *testing.T) {
+	// 控制请求超时时间
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	uri := "http://127.0.0.1:8000/add"
+	req := &service.AddRequest{
+		A: 1,
+		B: 2,
+	}
+	rsp := &service.AddResponse{}
+	err := httpcall.JsonPost(ctx, uri, nil, req, rsp)
+	if err != nil {
+		t.Fatalf("JsonPost err: %v", err)
+	}
+	t.Logf("add taskId: %d", rsp.TaskId)
+}

httpcall/json_delete.go

@@ -0,0 +1,42 @@
+package httpcall
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+)
+
+// JsonDelete http delete 请求
+func JsonDelete(ctx context.Context, uri string, header map[string]string, req, rsp interface{}) (err error) {
+	body, _ := json.Marshal(req)
+	reqCtx, err := http.NewRequestWithContext(ctx, http.MethodDelete, uri, bytes.NewReader(body))
+	if err != nil {
+		return fmt.Errorf("JsonDelete make NewRequest error: %v", err)
+	}
+
+	for head, value := range header {
+		reqCtx.Header.Set(head, value)
+	}
+
+	response, err := http.DefaultClient.Do(reqCtx)
+	if err != nil {
+		return fmt.Errorf("JsonDelete do request error: %v", err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("JsonDelete response status error, statuscode: %d, reason: %s",
+			response.StatusCode, response.Status)
+	}
+	data, _ := ioutil.ReadAll(response.Body)
+	// log.Printf("JsonDelete req: %v, rsp: %v \n", string(body), string(data))
+	if rsp != nil {
+		err = json.Unmarshal(data, rsp)
+		if err != nil {
+			return fmt.Errorf("JsonDelete response protocol error: %v, data: %s", err, string(data))
+		}
+	}
+	return nil
+}

httpcall/json_get.go

@@ -0,0 +1,39 @@
+package httpcall
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+)
+
+// JsonGet http json get 请求
+func JsonGet(ctx context.Context, uri string, header map[string]string, rsp interface{}) (err error) {
+	reqCtx, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
+	if err != nil {
+		return fmt.Errorf("JsonGet make NewRequest error: %v", err)
+	}
+
+	for head, value := range header {
+		reqCtx.Header.Set(head, value)
+	}
+
+	response, err := http.DefaultClient.Do(reqCtx)
+	if err != nil {
+		return fmt.Errorf("JsonGet do request error: %v", err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("JsonGet response status error, statuscode: %d, reason: %s",
+			response.StatusCode, response.Status)
+	}
+	data, _ := ioutil.ReadAll(response.Body)
+	if rsp != nil {
+		err = json.Unmarshal(data, rsp)
+		if err != nil {
+			return fmt.Errorf("JsonGet response protocol error: %v, data: %s", err, string(data))
+		}
+	}
+	return nil
+}

httpcall/json_post.go

@@ -0,0 +1,44 @@
+package httpcall
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+)
+
+// JsonPost http json post 请求
+func JsonPost(ctx context.Context, uri string, header map[string]string, req, rsp interface{}) (err error) {
+	body, _ := json.Marshal(req)
+	reqCtx, err := http.NewRequestWithContext(ctx, http.MethodPost, uri, bytes.NewReader(body))
+	if err != nil {
+		return fmt.Errorf("HttpPost make NewRequest error: %v", err)
+	}
+	if header == nil {
+		reqCtx.Header.Set("Content-Type", "application/json")
+	} else {
+		for head, value := range header {
+			reqCtx.Header.Set(head, value)
+		}
+	}
+	response, err := http.DefaultClient.Do(reqCtx)
+	if err != nil {
+		return fmt.Errorf("HttpPost do request error: %v", err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("HttpPost response status error, statuscode: %d, reason: %s",
+			response.StatusCode, response.Status)
+	}
+	data, _ := ioutil.ReadAll(response.Body)
+	// log.Printf("JsonPost req: %v, rsp: %v \n", string(body), string(data))
+	if rsp != nil {
+		err = json.Unmarshal(data, rsp)
+		if err != nil {
+			return fmt.Errorf("HttpPost response protocol error: %v, data: %s", err, string(data))
+		}
+	}
+	return nil
+}

httpcall/json_put.go

@@ -0,0 +1,47 @@
+package httpcall
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+)
+
+// JsonPut http put 请求
+func JsonPut(ctx context.Context, uri string, header map[string]string, req, rsp interface{}) (err error) {
+	body, ok := req.([]byte)
+	if !ok {
+		body, _ = json.Marshal(req)
+	}
+	reqCtx, err := http.NewRequestWithContext(ctx, http.MethodPut, uri, bytes.NewReader(body))
+	if err != nil {
+		return fmt.Errorf("JsonPut make NewRequest error: %v", err)
+	}
+	if header == nil {
+		reqCtx.Header.Set("Content-Type", "application/octet-stream")
+	} else {
+		for head, value := range header {
+			reqCtx.Header.Set(head, value)
+		}
+	}
+	response, err := http.DefaultClient.Do(reqCtx)
+	if err != nil {
+		return fmt.Errorf("JsonPut do request error: %v", err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("JsonPut response status error, statuscode: %d, reason: %s",
+			response.StatusCode, response.Status)
+	}
+	data, _ := ioutil.ReadAll(response.Body)
+	// log.Printf("JsonPut req: %v, rsp: %v \n", string(body), string(data))
+	if rsp != nil {
+		err = json.Unmarshal(data, rsp)
+		if err != nil {
+			return fmt.Errorf("JsonPut response protocol error: %v, data: %s", err, string(data))
+		}
+	}
+	return nil
+}

httpcall/readme.md

@@ -0,0 +1,35 @@
+# http 最常用的 json 协议调用封装
+- json_post.go: JsonPost。
+- json_get.go: JsonGet。
+- json_put.go: JsonPut。
+- json_delete.go: JsonDelete。
+
+
+example: https://github.com/memory-overflow/go-common-library/httpcall/httpcall_test.go
+```go
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/memory-overflow/go-common-library/httpcall"
+	service "github.com/memory-overflow/go-common-library/task_scheduler/example/add_service"
+)
+
+func TestJsonPost(t *testing.T) {
+	// 控制请求超时时间
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	uri := "http://127.0.0.1:8000/add"
+	req := &service.AddRequest{
+		A: 1,
+		B: 2,
+	}
+	rsp := &service.AddResponse{}
+	err := httpcall.JsonPost(ctx, uri, nil, req, rsp)
+	if err != nil {
+		t.Fatalf("JsonPost err: %v", err)
+	}
+	t.Logf("add taskId: %d", rsp.TaskId)
+}
+```

misc/donwlod.go

@@ -0,0 +1,88 @@
+package misc
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+)
+
+// DownloadFile 下载网络文件到服务器本地
+func DownloadFile(ctx context.Context, url string, dirs, fileName string) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return errors.New("DownloadFile " + url + " failed! " + err.Error())
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return errors.New("DownloadFile " + url + " failed! " + err.Error())
+	}
+	if res.StatusCode != 200 {
+		return errors.New("DownloadFile " + url + " failed! " + res.Status)
+	}
+	if err := os.MkdirAll(dirs, 0766); err != nil {
+		return errors.New("DownloadFile " + url + " failed! " + err.Error())
+	}
+	filePath := dirs + "/" + fileName
+	file, err := os.Create(filePath)
+	if err != nil {
+		return errors.New("DownloadFile " + url + " create file failed! " + err.Error())
+	}
+	if length, err := io.Copy(file, res.Body); err != nil {
+		os.Remove(filePath)
+		return errors.New("DownloadFile " + url + " copy failed! " + err.Error() +
+			" copied length: " + strconv.Itoa(int(length)))
+	}
+	file.Close()
+	_, err = os.Stat(filePath)
+	if err != nil {
+		os.Remove(filePath)
+		return errors.New("DownloadFile " + url + " failed! " + err.Error())
+	}
+	return nil
+}
+
+// DownloadFileWithLimit 下载网络文件到服务器本地，限制文件大小
+func DownloadFileWithLimit(ctx context.Context, url string, dirs, fileName string, maxBytes int64) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return errors.New("DownloadFile " + url + " failed! " + err.Error())
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return errors.New("DownloadFile " + url + " failed! " + err.Error())
+	}
+	if res.StatusCode != 200 {
+		return errors.New("DownloadFile " + url + " failed! " + res.Status)
+	}
+	if err := os.MkdirAll(dirs, 0766); err != nil {
+		return errors.New("DownloadFile " + url + " failed! " + err.Error())
+	}
+	filePath := dirs + "/" + fileName
+	file, err := os.Create(filePath)
+	if err != nil {
+		return errors.New("DownloadFile " + url + " create file failed! " + err.Error())
+	}
+	if length, err := io.CopyN(file, res.Body, int64(maxBytes)); err != nil && err != io.EOF {
+		os.Remove(filePath)
+		return errors.New("DownloadFile " + url + " copy failed! " + err.Error() +
+			" copied length: " + strconv.Itoa(int(length)))
+	}
+	file.Close()
+	nextByte := make([]byte, 1)
+	nRead, _ := io.ReadFull(res.Body, nextByte)
+	if nRead > 0 {
+		// Yep, there's too much data
+		os.Remove(filePath)
+		return fmt.Errorf("DownloadFile %s exceed the file size limit %d", url, maxBytes)
+	}
+	_, err = os.Stat(filePath)
+	if err != nil {
+		os.Remove(filePath)
+		return errors.New("DownloadFile " + url + " failed! " + err.Error())
+	}
+	return nil
+}