diff --git a/internal/base/base.go b/internal/base/base.go index 1643016f4..10fa1ed8f 100644 --- a/internal/base/base.go +++ b/internal/base/base.go @@ -10,11 +10,15 @@ import ( "path" "path/filepath" "strings" + "sync" ) const CmdName = "leetcode" -var Commands []*Command +var ( + Commands []*Command + Mutex sync.Mutex +) type Command struct { Run func(cmd *Command, args []string) diff --git a/internal/description/description.go b/internal/description/description.go index bf84716c9..65f6602ed 100644 --- a/internal/description/description.go +++ b/internal/description/description.go @@ -1,6 +1,8 @@ package description import ( + "sync" + "github.com/openset/leetcode/internal/base" "github.com/openset/leetcode/internal/leetcode" ) @@ -16,13 +18,22 @@ func runDescription(cmd *base.Command, args []string) { if len(args) != 0 { cmd.Usage() } + var wg sync.WaitGroup + tokens := make(chan bool, 1<<7) problems := leetcode.ProblemsAll() for _, problem := range problems.StatStatusPairs { - titleSlug := problem.Stat.QuestionTitleSlug - question := leetcode.QuestionData(titleSlug, false).Data.Question - if question.Content == "" && problem.PaidOnly == true && problem.Stat.QuestionArticleLive { - question.Content = leetcode.GetDescription(problem.Stat.QuestionArticleSlug) - } - question.SaveContent() + wg.Add(1) + tokens <- true + go func(problem leetcode.StatStatusPairsType) { + titleSlug := problem.Stat.QuestionTitleSlug + question := leetcode.QuestionData(titleSlug, false).Data.Question + if question.Content == "" && problem.PaidOnly == true && problem.Stat.QuestionArticleLive { + question.Content = leetcode.GetDescription(problem.Stat.QuestionArticleSlug) + } + question.SaveContent() + <-tokens + wg.Done() + }(problem) } + wg.Wait() } diff --git a/internal/leetcode/base.go b/internal/leetcode/base.go index b1e053649..d9c5068d5 100644 --- a/internal/leetcode/base.go +++ b/internal/leetcode/base.go @@ -23,7 +23,7 @@ var ( translationSet = make(map[int]string) ) -func graphQLRequest(filename string, days int, jsonStr string, v interface{}) { +func graphQLRequest(graphQL, jsonStr, filename string, days int, v interface{}) { data := remember(filename, days, func() []byte { return client.PostJson(graphQL, jsonStr) }) diff --git a/internal/leetcode/config.go b/internal/leetcode/config.go index 0fbf09123..80a8aeaef 100644 --- a/internal/leetcode/config.go +++ b/internal/leetcode/config.go @@ -27,5 +27,3 @@ const ( questionArticleFile = "question_article_%s.html" topicTagFile = "topic_tag_%s.json" ) - -var graphQL = graphQLCnUrl diff --git a/internal/leetcode/problems_all.go b/internal/leetcode/problems_all.go index 9a3d6df8a..09f2666a6 100644 --- a/internal/leetcode/problems_all.go +++ b/internal/leetcode/problems_all.go @@ -27,13 +27,13 @@ type problemsType struct { AcEasy int `json:"ac_easy"` AcMedium int `json:"ac_medium"` AcHard int `json:"ac_hard"` - StatStatusPairs []statStatusPairsType `json:"stat_status_pairs"` + StatStatusPairs []StatStatusPairsType `json:"stat_status_pairs"` FrequencyHigh int `json:"frequency_high"` FrequencyMid int `json:"frequency_mid"` CategorySlug string `json:"category_slug"` } -type statStatusPairsType struct { +type StatStatusPairsType struct { Stat statType `json:"stat"` Status string `json:"status"` Difficulty difficultyType `json:"difficulty"` @@ -62,7 +62,7 @@ type difficultyType struct { type paidType bool -func (problem statStatusPairsType) WriteRow(buf *bytes.Buffer) { +func (problem StatStatusPairsType) WriteRow(buf *bytes.Buffer) { format := "| %d | [%s](https://leetcode.com/problems/%s%s)%s | [%s](https://github.com/openset/leetcode/tree/master/problems/%s) | %s |\n" id := problem.Stat.FrontendQuestionId stat := problem.Stat diff --git a/internal/leetcode/question_data.go b/internal/leetcode/question_data.go index 147675a27..6b4d16229 100644 --- a/internal/leetcode/question_data.go +++ b/internal/leetcode/question_data.go @@ -12,7 +12,7 @@ import ( "unicode" ) -func QuestionData(titleSlug string, isForce bool) (qd questionDataType) { +func QuestionData(titleSlug string, isForce bool, graphQL ...string) (qd questionDataType) { jsonStr := `{ "operationName": "questionData", "variables": { @@ -24,19 +24,20 @@ func QuestionData(titleSlug string, isForce bool) (qd questionDataType) { if isForce { days = 0 } + if len(graphQL) == 0 { + graphQL = []string{graphQLCnUrl} + } filename := fmt.Sprintf(questionDataFile, slugToSnake(titleSlug)) - graphQLRequest(filename, days, jsonStr, &qd) + graphQLRequest(graphQL[0], jsonStr, filename, days, &qd) if qd.Data.Question.TitleSlug == "" { _ = os.Remove(getCachePath(filename)) - if graphQL == graphQLCnUrl { - graphQL = graphQLUrl - return QuestionData(titleSlug, isForce) + if graphQL[0] == graphQLCnUrl { + return QuestionData(titleSlug, isForce, graphQLUrl) } for _, err := range qd.Errors { log.Println(titleSlug, err.Message) } } - graphQL = graphQLCnUrl return } @@ -68,7 +69,7 @@ type questionType struct { Dislikes int `json:"dislikes"` IsLiked int `json:"isLiked"` SimilarQuestions string `json:"similarQuestions"` - TopicTags []tagType `json:"topicTags"` + TopicTags []TagType `json:"topicTags"` CodeSnippets []codeSnippetsType `json:"codeSnippets"` Hints []string `json:"hints"` MysqlSchemas []string `json:"mysqlSchemas"` diff --git a/internal/leetcode/question_translation.go b/internal/leetcode/question_translation.go index 98a2edba1..097a272e1 100644 --- a/internal/leetcode/question_translation.go +++ b/internal/leetcode/question_translation.go @@ -12,7 +12,7 @@ func GetQuestionTranslation() (qt questionTranslationType) { "variables": {}, "query": "query getQuestionTranslation($lang: String) {\n translations: allAppliedQuestionTranslations(lang: $lang) {\n title\n question {\n questionId\n __typename\n }\n __typename\n }\n}\n" }` - graphQLRequest(questionTranslationFile, 2, jsonStr, &qt) + graphQLRequest(graphQLCnUrl, jsonStr, questionTranslationFile, 2, &qt) if qt.Data.Translations == nil { _ = os.Remove(getCachePath(questionTranslationFile)) for _, err := range qt.Errors { diff --git a/internal/leetcode/topic_tag.go b/internal/leetcode/topic_tag.go index 3f263671d..4739f0eba 100644 --- a/internal/leetcode/topic_tag.go +++ b/internal/leetcode/topic_tag.go @@ -10,11 +10,12 @@ import ( "sort" "strconv" + "github.com/openset/leetcode/internal/base" "github.com/openset/leetcode/internal/client" ) var ( - initTags []tagType + initTags []TagType tagsFile = path.Join("tag", "tags.json") ) @@ -25,25 +26,27 @@ func init() { reg := regexp.MustCompile(`href="/tag/(\S+?)/"`) for _, matches := range reg.FindAllStringSubmatch(string(html), -1) { if len(matches) >= 2 { - initTags = append(initTags, tagType{Slug: matches[1]}) + initTags = append(initTags, TagType{Slug: matches[1]}) } } } -func GetTags() (tags []tagType) { +func GetTags() (tags []TagType) { cts := fileGetContents(tagsFile) jsonDecode(cts, &tags) tags = tagsUnique(tags) return } -func saveTags(tags []tagType) { +func saveTags(tags []TagType) { + base.Mutex.Lock() tags = append(GetTags(), tags...) filePutContents(tagsFile, jsonEncode(tagsUnique(tags))) + base.Mutex.Unlock() } -func tagsUnique(tags []tagType) []tagType { - rs, top := make([]tagType, 0, len(tags)), 1 +func tagsUnique(tags []TagType) []TagType { + rs, top := make([]TagType, 0, len(tags)), 1 tags = append(initTags, tags...) var flag = make(map[string]int) for _, tag := range tags { @@ -74,7 +77,7 @@ func GetTopicTag(slug string) (tt topicTagType) { "query": "query getTopicTag($slug: String!) {\n topicTag(slug: $slug) {\n name\n translatedName\n questions {\n status\n questionId\n questionFrontendId\n title\n titleSlug\n translatedTitle\n stats\n difficulty\n isPaidOnly\n topicTags {\n name\n translatedName\n slug\n __typename\n }\n __typename\n }\n frequencies\n __typename\n }\n favoritesLists {\n publicFavorites {\n ...favoriteFields\n __typename\n }\n privateFavorites {\n ...favoriteFields\n __typename\n }\n __typename\n }\n}\n\nfragment favoriteFields on FavoriteNode {\n idHash\n id\n name\n isPublicFavorite\n viewCount\n creator\n isWatched\n questions {\n questionId\n title\n titleSlug\n __typename\n }\n __typename\n}\n" }` filename := fmt.Sprintf(topicTagFile, slugToSnake(slug)) - graphQLRequest(filename, 2, jsonStr, &tt) + graphQLRequest(graphQLCnUrl, jsonStr, filename, 2, &tt) if tt.Data.TopicTag.Name == "" { _ = os.Remove(getCachePath(filename)) for _, err := range tt.Errors { @@ -84,7 +87,7 @@ func GetTopicTag(slug string) (tt topicTagType) { return } -type tagType struct { +type TagType struct { Name string Slug string TranslatedName string @@ -114,7 +117,7 @@ type ttQuestionType struct { TranslatedContent string `json:"translatedContent"` IsPaidOnly paidType `json:"isPaidOnly"` Difficulty string `json:"difficulty"` - TopicTags []tagType `json:"topicTags"` + TopicTags []TagType `json:"topicTags"` } func (question ttQuestionType) TagsStr() string { @@ -127,7 +130,7 @@ func (question ttQuestionType) TagsStr() string { return string(buf.Bytes()) } -func (tag tagType) SaveContents() { +func (tag TagType) SaveContents() { questions := GetTopicTag(tag.Slug).Data.TopicTag.Questions sort.Slice(questions, func(i, j int) bool { m, _ := strconv.Atoi(questions[i].QuestionFrontendId) @@ -150,7 +153,7 @@ func (tag tagType) SaveContents() { filePutContents(filename, buf.Bytes()) } -func (tag tagType) ShowName() string { +func (tag TagType) ShowName() string { if tag.TranslatedName != "" { return tag.TranslatedName } diff --git a/internal/tag/tag.go b/internal/tag/tag.go index 039bb24b4..6f8eb6451 100644 --- a/internal/tag/tag.go +++ b/internal/tag/tag.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "reflect" + "sync" "github.com/openset/leetcode/internal/base" "github.com/openset/leetcode/internal/leetcode" @@ -20,6 +21,7 @@ func runTag(cmd *base.Command, args []string) { if len(args) != 0 { cmd.Usage() } + var wg sync.WaitGroup var buf bytes.Buffer buf.WriteString(base.AuthInfo("tag")) buf.WriteString("\n## 话题分类\n\n") @@ -38,11 +40,16 @@ func runTag(cmd *base.Command, args []string) { if i&1 == 1 { buf.WriteString("\n") } - tag.SaveContents() + wg.Add(1) + go func(tag leetcode.TagType) { + tag.SaveContents() + wg.Done() + }(tag) } if reflect.DeepEqual(tags, leetcode.GetTags()) { break } } base.FilePutContents("tag/README.md", buf.Bytes()) + wg.Wait() }