Merge pull request #592 from openset/develop

Add: parallel

Author: Shuo

internal/base/base.go

@@ -10,11 +10,15 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"sync"
 )
 
 const CmdName = "leetcode"
 
-var Commands []*Command
+var (
+	Commands []*Command
+	Mutex    sync.Mutex
+)
 
 type Command struct {
 	Run       func(cmd *Command, args []string)

internal/description/description.go

@@ -1,6 +1,8 @@
 package description
 
 import (
+	"sync"
+
 	"github.com/openset/leetcode/internal/base"
 	"github.com/openset/leetcode/internal/leetcode"
 )
@@ -16,13 +18,22 @@ func runDescription(cmd *base.Command, args []string) {
 	if len(args) != 0 {
 		cmd.Usage()
 	}
+	var wg sync.WaitGroup
+	tokens := make(chan bool, 1<<7)
 	problems := leetcode.ProblemsAll()
 	for _, problem := range problems.StatStatusPairs {
-		titleSlug := problem.Stat.QuestionTitleSlug
-		question := leetcode.QuestionData(titleSlug, false).Data.Question
-		if question.Content == "" && problem.PaidOnly == true && problem.Stat.QuestionArticleLive {
-			question.Content = leetcode.GetDescription(problem.Stat.QuestionArticleSlug)
-		}
-		question.SaveContent()
+		wg.Add(1)
+		tokens <- true
+		go func(problem leetcode.StatStatusPairsType) {
+			titleSlug := problem.Stat.QuestionTitleSlug
+			question := leetcode.QuestionData(titleSlug, false).Data.Question
+			if question.Content == "" && problem.PaidOnly == true && problem.Stat.QuestionArticleLive {
+				question.Content = leetcode.GetDescription(problem.Stat.QuestionArticleSlug)
+			}
+			question.SaveContent()
+			<-tokens
+			wg.Done()
+		}(problem)
 	}
+	wg.Wait()
 }

internal/leetcode/base.go

@@ -23,7 +23,7 @@ var (
 	translationSet  = make(map[int]string)
 )
 
-func graphQLRequest(filename string, days int, jsonStr string, v interface{}) {
+func graphQLRequest(graphQL, jsonStr, filename string, days int, v interface{}) {
 	data := remember(filename, days, func() []byte {
 		return client.PostJson(graphQL, jsonStr)
 	})

internal/leetcode/config.go

@@ -27,5 +27,3 @@ const (
 	questionArticleFile     = "question_article_%s.html"
 	topicTagFile            = "topic_tag_%s.json"
 )
-
-var graphQL = graphQLCnUrl

internal/leetcode/problems_all.go

@@ -27,13 +27,13 @@ type problemsType struct {
 	AcEasy          int                   `json:"ac_easy"`
 	AcMedium        int                   `json:"ac_medium"`
 	AcHard          int                   `json:"ac_hard"`
-	StatStatusPairs []statStatusPairsType `json:"stat_status_pairs"`
+	StatStatusPairs []StatStatusPairsType `json:"stat_status_pairs"`
 	FrequencyHigh   int                   `json:"frequency_high"`
 	FrequencyMid    int                   `json:"frequency_mid"`
 	CategorySlug    string                `json:"category_slug"`
 }
 
-type statStatusPairsType struct {
+type StatStatusPairsType struct {
 	Stat       statType       `json:"stat"`
 	Status     string         `json:"status"`
 	Difficulty difficultyType `json:"difficulty"`
@@ -62,7 +62,7 @@ type difficultyType struct {
 
 type paidType bool
 
-func (problem statStatusPairsType) WriteRow(buf *bytes.Buffer) {
+func (problem StatStatusPairsType) WriteRow(buf *bytes.Buffer) {
 	format := "| <span id=\"%d\">%d</span> | [%s](https://leetcode.com/problems/%s%s)%s | [%s](https://github.com/openset/leetcode/tree/master/problems/%s) | %s |\n"
 	id := problem.Stat.FrontendQuestionId
 	stat := problem.Stat

internal/leetcode/question_data.go

@@ -12,7 +12,7 @@ import (
 	"unicode"
 )
 
-func QuestionData(titleSlug string, isForce bool) (qd questionDataType) {
+func QuestionData(titleSlug string, isForce bool, graphQL ...string) (qd questionDataType) {
 	jsonStr := `{
 		"operationName": "questionData",
 		"variables": {
@@ -24,19 +24,20 @@ func QuestionData(titleSlug string, isForce bool) (qd questionDataType) {
 	if isForce {
 		days = 0
 	}
+	if len(graphQL) == 0 {
+		graphQL = []string{graphQLCnUrl}
+	}
 	filename := fmt.Sprintf(questionDataFile, slugToSnake(titleSlug))
-	graphQLRequest(filename, days, jsonStr, &qd)
+	graphQLRequest(graphQL[0], jsonStr, filename, days, &qd)
 	if qd.Data.Question.TitleSlug == "" {
 		_ = os.Remove(getCachePath(filename))
-		if graphQL == graphQLCnUrl {
-			graphQL = graphQLUrl
-			return QuestionData(titleSlug, isForce)
+		if graphQL[0] == graphQLCnUrl {
+			return QuestionData(titleSlug, isForce, graphQLUrl)
 		}
 		for _, err := range qd.Errors {
 			log.Println(titleSlug, err.Message)
 		}
 	}
-	graphQL = graphQLCnUrl
 	return
 }
 
@@ -68,7 +69,7 @@ type questionType struct {
 	Dislikes           int                `json:"dislikes"`
 	IsLiked            int                `json:"isLiked"`
 	SimilarQuestions   string             `json:"similarQuestions"`
-	TopicTags          []tagType          `json:"topicTags"`
+	TopicTags          []TagType          `json:"topicTags"`
 	CodeSnippets       []codeSnippetsType `json:"codeSnippets"`
 	Hints              []string           `json:"hints"`
 	MysqlSchemas       []string           `json:"mysqlSchemas"`

internal/leetcode/question_translation.go

@@ -12,7 +12,7 @@ func GetQuestionTranslation() (qt questionTranslationType) {
 		"variables": {},
 		"query": "query getQuestionTranslation($lang: String) {\n  translations: allAppliedQuestionTranslations(lang: $lang) {\n    title\n    question {\n      questionId\n      __typename\n    }\n    __typename\n  }\n}\n"
 	}`
-	graphQLRequest(questionTranslationFile, 2, jsonStr, &qt)
+	graphQLRequest(graphQLCnUrl, jsonStr, questionTranslationFile, 2, &qt)
 	if qt.Data.Translations == nil {
 		_ = os.Remove(getCachePath(questionTranslationFile))
 		for _, err := range qt.Errors {

internal/leetcode/topic_tag.go

@@ -10,11 +10,12 @@ import (
 	"sort"
 	"strconv"
 
+	"github.com/openset/leetcode/internal/base"
 	"github.com/openset/leetcode/internal/client"
 )
 
 var (
-	initTags []tagType
+	initTags []TagType
 	tagsFile = path.Join("tag", "tags.json")
 )
 
@@ -25,25 +26,27 @@ func init() {
 	reg := regexp.MustCompile(`href="/tag/(\S+?)/"`)
 	for _, matches := range reg.FindAllStringSubmatch(string(html), -1) {
 		if len(matches) >= 2 {
-			initTags = append(initTags, tagType{Slug: matches[1]})
+			initTags = append(initTags, TagType{Slug: matches[1]})
 		}
 	}
 }
 
-func GetTags() (tags []tagType) {
+func GetTags() (tags []TagType) {
 	cts := fileGetContents(tagsFile)
 	jsonDecode(cts, &tags)
 	tags = tagsUnique(tags)
 	return
 }
 
-func saveTags(tags []tagType) {
+func saveTags(tags []TagType) {
+	base.Mutex.Lock()
 	tags = append(GetTags(), tags...)
 	filePutContents(tagsFile, jsonEncode(tagsUnique(tags)))
+	base.Mutex.Unlock()
 }
 
-func tagsUnique(tags []tagType) []tagType {
-	rs, top := make([]tagType, 0, len(tags)), 1
+func tagsUnique(tags []TagType) []TagType {
+	rs, top := make([]TagType, 0, len(tags)), 1
 	tags = append(initTags, tags...)
 	var flag = make(map[string]int)
 	for _, tag := range tags {
@@ -74,7 +77,7 @@ func GetTopicTag(slug string) (tt topicTagType) {
 		"query": "query getTopicTag($slug: String!) {\n  topicTag(slug: $slug) {\n    name\n    translatedName\n    questions {\n      status\n      questionId\n      questionFrontendId\n      title\n      titleSlug\n      translatedTitle\n      stats\n      difficulty\n      isPaidOnly\n      topicTags {\n        name\n        translatedName\n        slug\n        __typename\n      }\n      __typename\n    }\n    frequencies\n    __typename\n  }\n  favoritesLists {\n    publicFavorites {\n      ...favoriteFields\n      __typename\n    }\n    privateFavorites {\n      ...favoriteFields\n      __typename\n    }\n    __typename\n  }\n}\n\nfragment favoriteFields on FavoriteNode {\n  idHash\n  id\n  name\n  isPublicFavorite\n  viewCount\n  creator\n  isWatched\n  questions {\n    questionId\n    title\n    titleSlug\n    __typename\n  }\n  __typename\n}\n"
 	}`
 	filename := fmt.Sprintf(topicTagFile, slugToSnake(slug))
-	graphQLRequest(filename, 2, jsonStr, &tt)
+	graphQLRequest(graphQLCnUrl, jsonStr, filename, 2, &tt)
 	if tt.Data.TopicTag.Name == "" {
 		_ = os.Remove(getCachePath(filename))
 		for _, err := range tt.Errors {
@@ -84,7 +87,7 @@ func GetTopicTag(slug string) (tt topicTagType) {
 	return
 }
 
-type tagType struct {
+type TagType struct {
 	Name           string
 	Slug           string
 	TranslatedName string
@@ -114,7 +117,7 @@ type ttQuestionType struct {
 	TranslatedContent  string    `json:"translatedContent"`
 	IsPaidOnly         paidType  `json:"isPaidOnly"`
 	Difficulty         string    `json:"difficulty"`
-	TopicTags          []tagType `json:"topicTags"`
+	TopicTags          []TagType `json:"topicTags"`
 }
 
 func (question ttQuestionType) TagsStr() string {
@@ -127,7 +130,7 @@ func (question ttQuestionType) TagsStr() string {
 	return string(buf.Bytes())
 }
 
-func (tag tagType) SaveContents() {
+func (tag TagType) SaveContents() {
 	questions := GetTopicTag(tag.Slug).Data.TopicTag.Questions
 	sort.Slice(questions, func(i, j int) bool {
 		m, _ := strconv.Atoi(questions[i].QuestionFrontendId)
@@ -150,7 +153,7 @@ func (tag tagType) SaveContents() {
 	filePutContents(filename, buf.Bytes())
 }
 
-func (tag tagType) ShowName() string {
+func (tag TagType) ShowName() string {
 	if tag.TranslatedName != "" {
 		return tag.TranslatedName
 	}

internal/tag/tag.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"reflect"
+	"sync"
 
 	"github.com/openset/leetcode/internal/base"
 	"github.com/openset/leetcode/internal/leetcode"
@@ -20,6 +21,7 @@ func runTag(cmd *base.Command, args []string) {
 	if len(args) != 0 {
 		cmd.Usage()
 	}
+	var wg sync.WaitGroup
 	var buf bytes.Buffer
 	buf.WriteString(base.AuthInfo("tag"))
 	buf.WriteString("\n## 话题分类\n\n")
@@ -38,11 +40,16 @@ func runTag(cmd *base.Command, args []string) {
 			if i&1 == 1 {
 				buf.WriteString("\n")
 			}
-			tag.SaveContents()
+			wg.Add(1)
+			go func(tag leetcode.TagType) {
+				tag.SaveContents()
+				wg.Done()
+			}(tag)
 		}
 		if reflect.DeepEqual(tags, leetcode.GetTags()) {
 			break
 		}
 	}
 	base.FilePutContents("tag/README.md", buf.Bytes())
+	wg.Wait()
 }