From 0bde35eb4fe42521a6a10b63835226b2bf315668 Mon Sep 17 00:00:00 2001 From: yinyicao Date: Mon, 25 Mar 2024 16:33:05 +0800 Subject: [PATCH 1/5] feat(Enterprise version supporting training camps): support Enterprise version supporting training camps at https://b.geekbang.org/ --- cmd/root.go | 51 +++-- internal/geektime/geektime.go | 121 ++++++++++ .../struct_v1_enterprise_article_info.go | 195 ++++++++++++++++ .../response/struct_v1_enterprise_articles.go | 208 ++++++++++++++++++ .../struct_v1_enterprise_product_info.go | 166 ++++++++++++++ internal/video/video.go | 35 ++- 6 files changed, 757 insertions(+), 19 deletions(-) create mode 100644 internal/geektime/response/struct_v1_enterprise_article_info.go create mode 100644 internal/geektime/response/struct_v1_enterprise_articles.go create mode 100644 internal/geektime/response/struct_v1_enterprise_product_info.go diff --git a/cmd/root.go b/cmd/root.go index 37c9337..92d6e99 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/nicoxiang/geektime-downloader/internal/geektime/response" "io/ioutil" "math" "math/rand" @@ -30,22 +31,24 @@ import ( ) var ( - phone string - gcid string - gcess string - concurrency int - downloadFolder string - sp *spinner.Spinner - selectedProduct geektime.Product - quality string - downloadComments bool - selectedProductType productTypeSelectOption - columnOutputType int - waitSeconds int - productTypeOptions = make([]productTypeSelectOption, 6) - geektimeClient *geektime.Client - accountClient *geektime.Client - universityClient *geektime.Client + phone string + gcid string + gcess string + concurrency int + downloadFolder string + sp *spinner.Spinner + selectedProduct geektime.Product + selectedEnterProduct response.V1EnterpriseArticlesResponse + quality string + downloadComments bool + selectedProductType productTypeSelectOption + columnOutputType int + waitSeconds int + productTypeOptions = make([]productTypeSelectOption, 7) + geektimeClient *geektime.Client + geekEnterpriseClient *geektime.Client + accountClient *geektime.Client + universityClient *geektime.Client ) type productTypeSelectOption struct { @@ -90,6 +93,7 @@ func setProductTypeOptions() { productTypeOptions[3] = productTypeSelectOption{3, "大厂案例", 4, []string{"q"}, false} productTypeOptions[4] = productTypeSelectOption{4, "训练营", 5, []string{""}, true} //custom source type, not use productTypeOptions[5] = productTypeSelectOption{5, "其他", 1, []string{"x", "c6"}, true} + productTypeOptions[6] = productTypeSelectOption{6, "企业版训练营", 6, []string{"c44"}, true} } var rootCmd = &cobra.Command{ @@ -145,6 +149,7 @@ var rootCmd = &cobra.Command{ } geektimeClient = geektime.NewClient(readCookies) universityClient = geektime.NewUniversityClient(readCookies) + geekEnterpriseClient = geektime.NewEnterpriseClient(readCookies) selectProductType(cmd.Context()) }, } @@ -232,6 +237,8 @@ func loadProduct(ctx context.Context, productID int) { p, err = universityClient.MyClassProduct(productID) // university don't need check product type // if input invalid id, access mark is 0 + } else if isEnterpriseUniversity() { + p, err = geekEnterpriseClient.ArticlesInfo(productID) } else { p, err = geektimeClient.ColumnInfo(productID) if err == nil { @@ -426,6 +433,9 @@ func handleDownloadAll(ctx context.Context) { if isUniversity() { err := video.DownloadUniversityVideo(ctx, universityClient, a.AID, selectedProduct, projectDir, quality, concurrency) checkError(err) + } else if isEnterpriseUniversity() { + err := video.DownloadEnterpriseArticleVideo(ctx, geekEnterpriseClient, a.AID, selectedProductType.SourceType, projectDir, quality, concurrency) + checkError(err) } else { err := video.DownloadArticleVideo(ctx, geektimeClient, a.AID, selectedProductType.SourceType, projectDir, quality, concurrency) checkError(err) @@ -441,7 +451,7 @@ func increasePDFCount(total int, i *int) { } func loadArticles() { - if !isUniversity() && len(selectedProduct.Articles) <= 0 { + if !isUniversity() && !isEnterpriseUniversity() && len(selectedProduct.Articles) <= 0 { sp.Prefix = "[ 正在加载文章列表... ]" sp.Start() articles, err := geektimeClient.ColumnArticles(strconv.Itoa(selectedProduct.ID)) @@ -513,6 +523,9 @@ func downloadArticle(ctx context.Context, article geektime.Article, projectDir s if isUniversity() { err := video.DownloadUniversityVideo(ctx, universityClient, article.AID, selectedProduct, projectDir, quality, concurrency) checkError(err) + } else if isEnterpriseUniversity() { + err := video.DownloadEnterpriseArticleVideo(ctx, geekEnterpriseClient, article.AID, selectedProductType.SourceType, projectDir, quality, concurrency) + checkError(err) } else { err := video.DownloadArticleVideo(ctx, geektimeClient, article.AID, selectedProductType.SourceType, projectDir, quality, concurrency) checkError(err) @@ -528,6 +541,10 @@ func isUniversity() bool { return selectedProductType.Index == 4 } +func isEnterpriseUniversity() bool { + return selectedProductType.Index == 6 +} + // Sets the bit at pos in the integer n. func setBit(n int, pos uint) int { n |= (1 << pos) diff --git a/internal/geektime/geektime.go b/internal/geektime/geektime.go index 4f2e55f..066f716 100644 --- a/internal/geektime/geektime.go +++ b/internal/geektime/geektime.go @@ -24,6 +24,7 @@ const ( UserAgent = "User-Agent" // GeekBangUniversityBaseURL ... GeekBangUniversityBaseURL = "https://u.geekbang.org" + GeekBangEnterpriseBaseURL = "https://b.geekbang.org" // GeekBangAccountBaseURL ... GeekBangAccountBaseURL = "https://account.geekbang.org" // LoginPath ... @@ -49,6 +50,11 @@ const ( // UniversityV1MyClassInfoPath get university class info and all articles info in it UniversityV1MyClassInfoPath = "/serv/v1/myclass/info" + V1EnterpriseProductInfoPath = "/app/v1/course/info" + V1EnterpriseArticlesInfoPath = "/app/v1/course/articles" + V1EnterpriseArticleDetailInfoPath = "/app/v1/article/detail" + V1EnterpriseVideoPlayAuthPath = "/app/v1/source_auth/video_play_auth" + // GeekBangCookieDomain ... GeekBangCookieDomain = ".geekbang.org" @@ -141,6 +147,19 @@ func NewUniversityClient(cs []*http.Cookie) *Client { return c } +// NewEnterpriseClient +func NewEnterpriseClient(cs []*http.Cookie) *Client { + httpClient := resty.New(). + SetCookies(cs). + SetRetryCount(1). + SetTimeout(10*time.Second). + SetHeader("User-Agent", DefaultUserAgent). + SetLogger(logger.DiscardLogger{}) + + c := &Client{HTTPClient: httpClient, BaseURL: GeekBangEnterpriseBaseURL, Cookies: cs} + return c +} + // Login call geektime login api and return auth cookies func (c *Client) Login(phone, password string) ([]*http.Cookie, error) { var res struct { @@ -328,6 +347,108 @@ func (c *Client) ProductInfo(productID int) (response.V3ProductInfoResponse, err return res, nil } +func (c *Client) enterpriseProductInfo(productID int) (response.V1EnterpriseProductInfoResponse, error) { + var res response.V1EnterpriseProductInfoResponse + r := c.newRequest(resty.MethodPost, + V1EnterpriseProductInfoPath, + nil, + map[string]interface{}{ + "id": productID, + }, + &res, + ) + if _, err := do(r); err != nil { + return response.V1EnterpriseProductInfoResponse{}, err + } + return res, nil +} + +func (c *Client) ArticlesInfo(id int) (Product, error) { + var p Product + productInfo, err := c.enterpriseProductInfo(id) + if err != nil { + return p, err + } + + var res response.V1EnterpriseArticlesResponse + r := c.newRequest(resty.MethodPost, + V1EnterpriseArticlesInfoPath, + nil, + map[string]interface{}{ + "id": id, + }, + &res, + ) + + resp, err := do(r) + if err != nil { + return p, err + } + + if res.Code != 0 { + if res.Data.IsShow == false && productInfo.Data.Extra.IsMyCourse == false { + p.Access = false + return p, nil + } + return p, ErrGeekTimeAPIBadCode{V1EnterpriseArticlesInfoPath, resp.String()} + } + + p = Product{ + Access: true, + ID: id, + Title: productInfo.Data.Title, + Type: "", + IsVideo: true, + } + var articles []Article + + for _, sections := range res.Data.List { + for _, a := range sections.ArticleList { + articleId, _ := strconv.Atoi(a.Article.ID) + articles = append(articles, Article{ + AID: articleId, + Title: a.Article.Title, + }) + } + } + p.Articles = articles + + return p, nil +} + +func (c *Client) V1EnterpriseArticleDetailInfo(articleID string) (response.V1EnterpriseArticlesDetailResponse, error) { + var res response.V1EnterpriseArticlesDetailResponse + r := c.newRequest(resty.MethodPost, + V1EnterpriseArticleDetailInfoPath, + nil, + map[string]interface{}{ + "article_id": articleID, + }, + &res, + ) + if _, err := do(r); err != nil { + return response.V1EnterpriseArticlesDetailResponse{}, err + } + return res, nil +} + +func (c *Client) EnterPriseVideoPlayAuth(articleID, videoID string) (string, error) { + var res response.V3VideoPlayAuthResponse + r := c.newRequest(resty.MethodPost, + V1EnterpriseVideoPlayAuthPath, + nil, + map[string]interface{}{ + "aid": articleID, + "video_id": videoID, + }, + &res, + ) + if _, err := do(r); err != nil { + return "", err + } + return res.Data.PlayAuth, nil +} + // V3ArticleInfo used to get daily lesson or qconplus article info func (c *Client) V3ArticleInfo(articleID int) (response.V3ArticleInfoResponse, error) { var res response.V3ArticleInfoResponse diff --git a/internal/geektime/response/struct_v1_enterprise_article_info.go b/internal/geektime/response/struct_v1_enterprise_article_info.go new file mode 100644 index 0000000..b78b3a5 --- /dev/null +++ b/internal/geektime/response/struct_v1_enterprise_article_info.go @@ -0,0 +1,195 @@ +package response + +type V1EnterpriseArticlesDetailResponse struct { + Code int `json:"code"` + Data struct { + ID string `json:"id"` + Time string `json:"time"` + Type string `json:"type"` + FavoriteID int `json:"favorite_id"` + DiscussionNumber int `json:"discussion_number"` + ColumnTitle string `json:"column_title"` + Rights bool `json:"rights"` + Show bool `json:"show"` + RichType int `json:"rich_type"` + PID int `json:"pid"` + SKU int `json:"sku"` + Action string `json:"action"` + Score int `json:"score"` + IsRequired bool `json:"is_required"` + URI string `json:"uri"` + ColumnType int `json:"column_type"` + EnterpriseID string `json:"enterprise_id"` + NodeType int `json:"node_type"` + Published int `json:"published"` + ArtStatus int `json:"art_status"` + SKUStatus int `json:"sku_status"` + IsSell int `json:"is_sell"` + Name string `json:"name"` + ProductType string `json:"product_type"` + ArticleSource int `json:"article_source"` + ArticleVendorID int `json:"article_vendor_id"` + Author struct { + Name string `json:"name"` + Avatar string `json:"avatar"` + Info string `json:"info"` + Intro string `json:"intro"` + } `json:"author"` + Article struct { + ID string `json:"id"` + Title string `json:"title"` + Content string `json:"content"` + ContentMD string `json:"content_md"` + CTime int `json:"ctime"` + PosterWxlite string `json:"poster_wxlite"` + CoverHidden int `json:"cover_hidden"` + Subtitle string `json:"subtitle"` + Summary string `json:"summary"` + CouldPreview bool `json:"could_preview"` + BCouldPreview bool `json:"b_could_preview"` + ContentJSON string `json:"content_json"` + ContentJSONShort string `json:"content_json_short"` + InlineVideo struct { + Rights []interface{} `json:"rights"` + Preview []interface{} `json:"preview"` + } `json:"inline_video"` + Cover struct { + ColumnCover string `json:"column_cover"` + Default string `json:"default"` + CoverID int `json:"cover_id"` + CoverStatus int `json:"cover_status"` + SKUCover struct { + Ratio16 string `json:"ratio_16"` + Ratio16URL string `json:"ratio_16_url"` + Ratio4 string `json:"ratio_4"` + Ratio4URL string `json:"ratio_4_url"` + Ratio1 string `json:"ratio_1"` + Ratio1URL string `json:"ratio_1_url"` + ShowCover int `json:"show_cover"` + } `json:"sku_cover"` + } `json:"cover"` + Share struct { + Title string `json:"title"` + Content string `json:"content"` + Cover string `json:"cover"` + Poster string `json:"poster"` + } `json:"share"` + Relation struct { + PrevID string `json:"prev_id"` + PrevChapterTitle string `json:"prev_chapter_title"` + PrevArticleTitle string `json:"prev_article_title"` + NextID string `json:"next_id"` + NextChapterTitle string `json:"next_chapter_title"` + NextArticleTitle string `json:"next_article_title"` + } `json:"relation"` + } `json:"article"` + Chapter struct { + SourceID int `json:"source_id"` + Title string `json:"title"` + SKU string `json:"sku"` + Score string `json:"score"` + PChapterSourceID string `json:"pchapter_source_id"` + PChapterTitle string `json:"p_chapter_title"` + ChapterStatus int `json:"chapter_status"` + } `json:"chapter"` + Audio struct { + URL string `json:"url"` + DownloadURL string `json:"download_url"` + Size int `json:"size"` + Title string `json:"title"` + Time string `json:"time"` + MD5 string `json:"md5"` + Dubber string `json:"dubber"` + ID string `json:"id"` + Status int `json:"status"` + } `json:"audio"` + Video struct { + ID string `json:"id"` + MD5 string `json:"md5"` + URL string `json:"url"` + Cover struct { + Type int `json:"type"` + ID int `json:"id"` + URL string `json:"url"` + } `json:"cover"` + Width int `json:"width"` + Height int `json:"height"` + Size int `json:"size"` + Time string `json:"time"` + HLSMedias []struct { + Quality string `json:"quality"` + Size int `json:"size"` + URL string `json:"url"` + } `json:"hls_medias"` + HLSVid string `json:"hls_vid"` + Version int `json:"version"` + Medias interface{} `json:"medias"` + MediaOpen string `json:"media_open"` + CouldPreview int `json:"could_preview"` + Preview struct { + Duration int `json:"duration"` + Medias []struct { + Quality string `json:"quality"` + Size int `json:"size"` + URL string `json:"url"` + } `json:"medias"` + } `json:"preview"` + Subtitles struct { + Rights interface{} `json:"rights"` + Preview interface{} `json:"preview"` + } `json:"subtitles"` + Status int `json:"status"` + } `json:"video"` + Files []interface{} `json:"files"` + Extra struct { + Process struct { + ArticleID string `json:"article_id"` + LearnPercent int `json:"learn_percent"` + ArticleOffset struct { + CurOffset int `json:"cur_offset"` + MaxOffset int `json:"max_offset"` + Length int `json:"length"` + Version int `json:"version"` + Process int `json:"process"` + LearnTime int `json:"learn_time"` + LearnStatus int `json:"learn_status"` + } `json:"article_offset"` + AudioOffset struct { + CurOffset int `json:"cur_offset"` + MaxOffset int `json:"max_offset"` + Length int `json:"length"` + Version int `json:"version"` + Process int `json:"process"` + LearnTime int `json:"learn_time"` + LearnStatus int `json:"learn_status"` + } `json:"audio_offset"` + VideoOffset struct { + CurOffset int `json:"cur_offset"` + MaxOffset int `json:"max_offset"` + Length int `json:"length"` + Version int `json:"version"` + Process int `json:"process"` + LearnTime int `json:"learn_time"` + LearnStatus int `json:"learn_status"` + } `json:"video_offset"` + } `json:"process"` + IsLast bool `json:"is_last"` + Fav struct { + HasDone bool `json:"has_done"` + TotalCount int `json:"total_count"` + FavID int `json:"fav_id"` + FavType int `json:"fav_type"` + } `json:"fav"` + IsShow bool `json:"IsShow"` + Attachments []interface{} `json:"attachments"` + } `json:"extra"` + AnyreadTotal int `json:"anyread_total"` + AnyreadUsed int `json:"anyread_used"` + AnyreadHit bool `json:"anyread_hit"` + } `json:"data"` + Error interface{} `json:"error"` + Extra struct { + Cost float64 `json:"cost"` + RequestID string `json:"request-id"` + } `json:"extra"` +} diff --git a/internal/geektime/response/struct_v1_enterprise_articles.go b/internal/geektime/response/struct_v1_enterprise_articles.go new file mode 100644 index 0000000..3386f32 --- /dev/null +++ b/internal/geektime/response/struct_v1_enterprise_articles.go @@ -0,0 +1,208 @@ +package response + +type V1EnterpriseArticlesResponse struct { + Code int `json:"code"` + Data struct { + List []struct { + ID int `json:"id"` + Title string `json:"title"` + Count int `json:"count"` + Score int `json:"score"` + IsLast bool `json:"is_last"` + ArticleList []struct { + ID string `json:"id"` + Time string `json:"time"` + Type string `json:"type"` + FavoriteID int `json:"favorite_id"` + DiscussionNumber int `json:"discussion_number"` + ColumnTitle string `json:"column_title"` + Rights bool `json:"rights"` + Show bool `json:"show"` + RichType int `json:"rich_type"` + PID int `json:"pid"` + SKU int `json:"sku"` + Action string `json:"action"` + Score int `json:"score"` + IsRequired bool `json:"is_required"` + URI string `json:"uri"` + ColumnType int `json:"column_type"` + EnterpriseID string `json:"enterprise_id"` + NodeType int `json:"node_type"` + Published int `json:"published"` + ArtStatus int `json:"art_status"` + SKUStatus int `json:"sku_status"` + IsSell int `json:"is_sell"` + Name string `json:"name"` + ProductType string `json:"product_type"` + ArticleSource int `json:"article_source"` + ArticleVendorID int `json:"article_vendor_id"` + Author struct { + Name string `json:"name"` + Avatar string `json:"avatar"` + Info string `json:"info"` + Intro string `json:"intro"` + } `json:"author"` + Article struct { + ID string `json:"id"` + Title string `json:"title"` + Content string `json:"content"` + ContentMD string `json:"content_md"` + CTime int `json:"ctime"` + PosterWxlite string `json:"poster_wxlite"` + CoverHidden int `json:"cover_hidden"` + Subtitle string `json:"subtitle"` + Summary string `json:"summary"` + CouldPreview bool `json:"could_preview"` + BCouldPreview bool `json:"b_could_preview"` + ContentJSON string `json:"content_json"` + ContentJSONShort string `json:"content_json_short"` + InlineVideo struct { + Rights []interface{} `json:"rights"` + Preview []interface{} `json:"preview"` + } `json:"inline_video"` + Cover struct { + ColumnCover string `json:"column_cover"` + Default string `json:"default"` + CoverID int `json:"cover_id"` + CoverStatus int `json:"cover_status"` + SKUCover struct { + Ratio16 string `json:"ratio_16"` + Ratio16URL string `json:"ratio_16_url"` + Ratio4 string `json:"ratio_4"` + Ratio4URL string `json:"ratio_4_url"` + Ratio1 string `json:"ratio_1"` + Ratio1URL string `json:"ratio_1_url"` + ShowCover int `json:"show_cover"` + } `json:"sku_cover"` + } `json:"cover"` + Share struct { + Title string `json:"title"` + Content string `json:"content"` + Cover string `json:"cover"` + Poster string `json:"poster"` + } `json:"share"` + Relation struct { + PrevID string `json:"prev_id"` + PrevChapterTitle string `json:"prev_chapter_title"` + PrevArticleTitle string `json:"prev_article_title"` + NextID string `json:"next_id"` + NextChapterTitle string `json:"next_chapter_title"` + NextArticleTitle string `json:"next_article_title"` + } `json:"relation"` + } `json:"article"` + Chapter struct { + SourceID int `json:"source_id"` + Title string `json:"title"` + SKU string `json:"sku"` + Score string `json:"score"` + PChapterSourceID string `json:"pchapter_source_id"` + PChapterTitle string `json:"p_chapter_title"` + ChapterStatus int `json:"chapter_status"` + } `json:"chapter"` + Audio struct { + URL string `json:"url"` + DownloadURL string `json:"download_url"` + Size int `json:"size"` + Title string `json:"title"` + Time string `json:"time"` + MD5 string `json:"md5"` + Dubber string `json:"dubber"` + ID string `json:"id"` + Status int `json:"status"` + } `json:"audio"` + Video struct { + ID string `json:"id"` + MD5 string `json:"md5"` + URL string `json:"url"` + Cover struct { + Type int `json:"type"` + ID int `json:"id"` + URL string `json:"url"` + } `json:"cover"` + Width int `json:"width"` + Height int `json:"height"` + Size int `json:"size"` + Time string `json:"time"` + HlsMedias []struct { + Quality string `json:"quality"` + Size int `json:"size"` + URL string `json:"url"` + } `json:"hls_medias"` + HlsVid string `json:"hls_vid"` + Version int `json:"version"` + Medias interface{} `json:"medias"` + MediaOpen string `json:"media_open"` + CouldPreview int `json:"could_preview"` + Preview struct { + Duration int `json:"duration"` + Medias []struct { + Quality string `json:"quality"` + Size int `json:"size"` + URL string `json:"url"` + } `json:"medias"` + } `json:"preview"` + Subtitles struct { + Rights interface{} `json:"rights"` + Preview []interface{} `json:"preview"` + } `json:"subtitles"` + Status int `json:"status"` + } `json:"video"` + Files []interface{} `json:"files"` + Extra struct { + Process struct { + ArticleID string `json:"article_id"` + LearnPercent int `json:"learn_percent"` + ArticleOffset struct { + CurOffset int `json:"cur_offset"` + MaxOffset int `json:"max_offset"` + Length int `json:"length"` + Version int `json:"version"` + Process int `json:"process"` + LearnTime int `json:"learn_time"` + LearnStatus int `json:"learn_status"` + } `json:"article_offset"` + AudioOffset struct { + CurOffset int `json:"cur_offset"` + MaxOffset int `json:"max_offset"` + Length int `json:"length"` + Version int `json:"version"` + Process int `json:"process"` + LearnTime int `json:"learn_time"` + LearnStatus int `json:"learn_status"` + } `json:"audio_offset"` + VideoOffset struct { + CurOffset int `json:"cur_offset"` + MaxOffset int `json:"max_offset"` + Length int `json:"length"` + Version int `json:"version"` + Process int `json:"process"` + LearnTime int `json:"learn_time"` + LearnStatus int `json:"learn_status"` + } `json:"video_offset"` + } `json:"process"` + IsLast bool `json:"is_last"` + Fav struct { + HasDone bool `json:"has_done"` + TotalCount int `json:"total_count"` + FavID int `json:"fav_id"` + FavType int `json:"fav_type"` + } `json:"fav"` + IsShow bool `json:"IsShow"` + Attachments []interface{} `json:"attachments"` + } `json:"extra"` + AnyreadTotal int `json:"anyread_total"` + AnyreadUsed int `json:"anyread_used"` + AnyreadHit bool `json:"anyread_hit"` + } `json:"article_list"` + } `json:"list"` + HasChapter bool `json:"has_chapter"` + IsShow bool `json:"is_show"` + AnyreadTotal int `json:"anyread_total"` + AnyreadUsed int `json:"anyread_used"` + + Extra struct { + Cost float64 `json:"cost"` + RequestID string `json:"request-id"` + } `json:"extra"` + } `json:"data"` +} diff --git a/internal/geektime/response/struct_v1_enterprise_product_info.go b/internal/geektime/response/struct_v1_enterprise_product_info.go new file mode 100644 index 0000000..cc0a44f --- /dev/null +++ b/internal/geektime/response/struct_v1_enterprise_product_info.go @@ -0,0 +1,166 @@ +package response + +type V1EnterpriseProductInfoResponse struct { + Code int `json:"code"` + Data struct { + ID int `json:"id"` + SKU int `json:"sku"` + Title string `json:"title"` + SubTitle string `json:"sub_title"` + ProductType string `json:"product_type"` + ColumnType int `json:"column_type"` + CourseType int `json:"course_type"` + UpdateFreq string `json:"update_frequency"` + Author struct { + Name string `json:"name"` + Intro string `json:"intro"` + Info string `json:"info"` + Avatar string `json:"avatar"` + BriefHTML string `json:"brief_html"` + Brief string `json:"brief"` + } `json:"author"` + Cover struct { + Square string `json:"square"` + Rectangle string `json:"rectangle"` + Horizontal string `json:"horizontal"` + LectureHorizontal string `json:"lecture_horizontal"` + LearnHorizontal string `json:"learn_horizontal"` + Transparent string `json:"transparent"` + Color string `json:"color"` + Cover string `json:"cover"` + RectCover string `json:"rect_cover"` + Ratio1 string `json:"ratio_1"` + Ratio4 string `json:"ratio_4"` + Ratio16 string `json:"ratio_16"` + CoverID int `json:"cover_id"` + CoverStatus int `json:"cover_status"` + } `json:"cover"` + TeachTypeList []string `json:"teach_type_list"` + TeachTypeNameList []string `json:"teach_type_name_list"` + Article struct { + Count int `json:"count"` + CountReq int `json:"count_req"` + CountPub int `json:"count_pub"` + FirstArticleID string `json:"first_article_id"` + TotalLength int `json:"total_length"` + TotalTimeStr string `json:"total_time_str"` + TotalTimeHourStr string `json:"total_time_hour_str"` + } `json:"article"` + SEO struct { + Keywords []string `json:"keywords"` + } `json:"seo"` + Category struct { + CategoryID int `json:"category_id"` + Name string `json:"name"` + PID int `json:"pid"` + } `json:"category"` + Path struct { + Desc string `json:"desc"` + DescHTML string `json:"desc_html"` + } `json:"path"` + DL struct { + Article struct { + ArticleID string `json:"article_id"` + Duration string `json:"duration"` + Hot int `json:"hot"` + CouldPreview bool `json:"could_preview"` + DurationSeconds int `json:"duration_seconds"` + } `json:"article"` + CollectionIDs interface{} `json:"collection_ids"` + } `json:"dl"` + Share struct { + PicURL string `json:"pic_url"` + Title string `json:"title"` + PicName string `json:"pic_name"` + Content string `json:"content"` + } `json:"share"` + IsFinish bool `json:"is_finish"` + Unit string `json:"unit"` + BannerCover string `json:"banner_cover"` + CatalogPicURL string `json:"catalog_pic_url"` + Extra struct { + Fav struct { + HasDone bool `json:"has_done"` + TotalCount int `json:"total_count"` + FavID int `json:"fav_id"` + FavType int `json:"fav_type"` + } `json:"fav"` + IsSVIP bool `json:"is_svip"` + IsMyCourse bool `json:"is_my_course"` + Rate struct { + ArticleCount int `json:"article_count"` + ArticleCountReq int `json:"article_count_req"` + IsFinished bool `json:"is_finished"` + RatePercent int `json:"rate_percent"` + VideoSeconds int `json:"video_seconds"` + LastArticleID string `json:"last_article_id"` + LastChapterID int `json:"last_chapter_id"` + HasLearn bool `json:"has_learn"` + } `json:"rate"` + StudyCount int `json:"study_count"` + Modules []struct { + Name string `json:"name"` + IsTop bool `json:"is_top"` + Title string `json:"title"` + Type string `json:"type"` + Content string `json:"content"` + } `json:"modules"` + TplType int `json:"tpl_type"` + CollectionType int `json:"collection_type"` + WithVideo bool `json:"with_video"` + PIDs []interface{} `json:"pids"` + Labels []interface{} `json:"labels"` + CategoryIDs []interface{} `json:"category_ids"` + Group struct { + Title string `json:"title"` + Description string `json:"description"` + StartTime int `json:"start_time"` + EndTime int `json:"end_time"` + QRCodeShow bool `json:"qrcode_show"` + QRCodeURL string `json:"qrcode_url"` + } `json:"group"` + VIP struct { + Show bool `json:"show"` + EndTime int `json:"end_time"` + } `json:"vip"` + CourseStatus int `json:"course_status"` + CID int `json:"cid"` + RelatedVIPSkus []struct { + ColumnTitle string `json:"column_title"` + DisplayType int `json:"display_type"` + EsPrice int `json:"es_price"` + EsSaleMaxLimit int `json:"es_sale_max_limit"` + EsSaleMinLimit int `json:"es_sale_min_limit"` + SKU int `json:"sku"` + Status int `json:"status"` + VIPDays int `json:"vip_days"` + VIPTitle string `json:"vip_title"` + } `json:"related_vip_skus"` + } `json:"extra"` + Intro string `json:"intro"` + IntroHTML string `json:"intro_html"` + BgColor string `json:"bgcolor"` + IsIncludePreview bool `json:"is_include_preview"` + ShowChapter bool `json:"show_chapter"` + DisplayType int `json:"display_type"` + IntroBGStyle int `json:"intro_bg_style"` + Sort int `json:"sort"` + CTime int `json:"ctime"` + SalePrice int `json:"sale_price"` + SaleLimit int `json:"sale_limit"` + Status int `json:"status"` + IsJoinSVIP int `json:"is_join_svip"` + IsJoinColumnVIP int `json:"is_join_column_vip"` + IsJoinCVIP int `json:"is_join_cvip"` + NeedGraduate int `json:"need_graduate"` + AuthorSignatureURL string `json:"author_signature_url"` + IsFreebie int `json:"is_freebie"` + IsDtai int `json:"is_dtai"` + } `json:"data"` + Error struct { + } `json:"error"` + Extra struct { + Cost float64 `json:"cost"` + RequestID string `json:"request-id"` + } `json:"extra"` +} diff --git a/internal/video/video.go b/internal/video/video.go index 45f3b19..c46dda8 100644 --- a/internal/video/video.go +++ b/internal/video/video.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "strconv" "strings" "time" @@ -76,6 +77,36 @@ func DownloadArticleVideo(ctx context.Context, concurrency) } +func DownloadEnterpriseArticleVideo(ctx context.Context, + client *geektime.Client, + articleID int, + sourceType int, + projectDir string, + quality string, + concurrency int, +) error { + + articleInfo, err := client.V1EnterpriseArticleDetailInfo(strconv.Itoa(articleID)) + if err != nil { + return err + } + if articleInfo.Data.Video.ID == "" { + return nil + } + playAuth, err := client.EnterPriseVideoPlayAuth(strconv.Itoa(articleID), articleInfo.Data.Video.ID) + if err != nil { + return err + } + return downloadAliyunVodEncryptVideo(ctx, + client, + playAuth, + articleInfo.Data.Article.Title, + projectDir, + quality, + articleInfo.Data.Video.ID, + concurrency) +} + // DownloadUniversityVideo ... func DownloadUniversityVideo(ctx context.Context, client *geektime.Client, @@ -185,11 +216,11 @@ func download(ctx context.Context, for _, tsFileName := range tsFileNames { u := tsURLPrefix + tsFileName dst := filepath.Join(tempVideoDir, tsFileName) - + headers := make(map[string]string, 2) headers[geektime.Origin] = geektime.DefaultBaseURL headers[geektime.UserAgent] = geektime.DefaultUserAgent - + fileSize, err := downloader.DownloadFileConcurrently(ctx, dst, u, headers, 5) if err != nil { return err From d3ce6bbb93d7c92c0a4d59dd79109889a2f0d15d Mon Sep 17 00:00:00 2001 From: yinyicao Date: Thu, 28 Mar 2024 15:03:55 +0800 Subject: [PATCH 2/5] fix(Enterprise training camps): fix no sections folder --- cmd/root.go | 51 ++++++++++++++++++++++++----------- internal/geektime/geektime.go | 10 ++++--- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 92d6e99..573ca39 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "github.com/nicoxiang/geektime-downloader/internal/geektime/response" - "io/ioutil" "math" "math/rand" "net/http" @@ -426,18 +425,24 @@ func handleDownloadAll(ctx context.Context) { } } else { for _, a := range selectedProduct.Articles { + sectionDir := projectDir fileName := filenamify.Filenamify(a.Title) + video.TSExtension if _, ok := downloaded[fileName]; ok { continue } + // add sub dir + if a.SectionTitle != "" { + sectionDir, err = mkDownloadProjectSectionDir(projectDir, a.SectionTitle) + checkError(err) + } if isUniversity() { - err := video.DownloadUniversityVideo(ctx, universityClient, a.AID, selectedProduct, projectDir, quality, concurrency) + err := video.DownloadUniversityVideo(ctx, universityClient, a.AID, selectedProduct, sectionDir, quality, concurrency) checkError(err) } else if isEnterpriseUniversity() { - err := video.DownloadEnterpriseArticleVideo(ctx, geekEnterpriseClient, a.AID, selectedProductType.SourceType, projectDir, quality, concurrency) + err := video.DownloadEnterpriseArticleVideo(ctx, geekEnterpriseClient, a.AID, selectedProductType.SourceType, sectionDir, quality, concurrency) checkError(err) } else { - err := video.DownloadArticleVideo(ctx, geektimeClient, a.AID, selectedProductType.SourceType, projectDir, quality, concurrency) + err := video.DownloadArticleVideo(ctx, geektimeClient, a.AID, selectedProductType.SourceType, sectionDir, quality, concurrency) checkError(err) } } @@ -572,17 +577,24 @@ func readCookiesFromInput() []*http.Cookie { } func findDownloadedArticleFileNames(projectDir string) (map[string]struct{}, error) { - files, err := ioutil.ReadDir(projectDir) - res := make(map[string]struct{}, len(files)) - if err != nil { - return res, err - } - if len(files) == 0 { - return res, nil - } - for _, f := range files { - res[f.Name()] = struct{}{} - } + res := make(map[string]struct{}) + limit := 2 + err := filepath.Walk(projectDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + fmt.Printf("访问路径时出错：%v\n", err) + return err + } + // 计算当前路径的深度 + depth := len(filepath.SplitList(path)) - len(filepath.SplitList(projectDir)) + if depth >= limit { + return filepath.SkipDir // 如果达到限制深度，则跳过该文件夹及其子文件夹 + } + if !info.IsDir() { + res[info.Name()] = struct{}{} + } + return nil + }) + checkError(err) return res, nil } @@ -599,6 +611,15 @@ func mkDownloadProjectDir(downloadFolder, phone, gcid, projectName string) (stri return path, nil } +func mkDownloadProjectSectionDir(downloadFolder, sectionName string) (string, error) { + path := filepath.Join(downloadFolder, filenamify.Filenamify(sectionName)) + err := os.MkdirAll(path, os.ModePerm) + if err != nil { + return "", err + } + return path, nil +} + func checkProductType(productType string) bool { for _, pt := range selectedProductType.AcceptProductTypes { if pt == productType { diff --git a/internal/geektime/geektime.go b/internal/geektime/geektime.go index 066f716..77e983d 100644 --- a/internal/geektime/geektime.go +++ b/internal/geektime/geektime.go @@ -87,8 +87,9 @@ type Product struct { // Article ... type Article struct { - AID int - Title string + AID int + SectionTitle string + Title string } // ErrGeekTimeAPIBadCode ... @@ -406,8 +407,9 @@ func (c *Client) ArticlesInfo(id int) (Product, error) { for _, a := range sections.ArticleList { articleId, _ := strconv.Atoi(a.Article.ID) articles = append(articles, Article{ - AID: articleId, - Title: a.Article.Title, + AID: articleId, + SectionTitle: sections.Title, + Title: a.Article.Title, }) } } From 29a91e398715be4bd8ed0cecc0340ab7df5e84bb Mon Sep 17 00:00:00 2001 From: yinyicao Date: Thu, 28 Mar 2024 15:05:02 +0800 Subject: [PATCH 3/5] fix(Enterprise): fix error 'cannot unmarshal number into Go struct field .data.teach_type_list of type string' --- internal/geektime/response/struct_v1_enterprise_product_info.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/geektime/response/struct_v1_enterprise_product_info.go b/internal/geektime/response/struct_v1_enterprise_product_info.go index cc0a44f..3096a24 100644 --- a/internal/geektime/response/struct_v1_enterprise_product_info.go +++ b/internal/geektime/response/struct_v1_enterprise_product_info.go @@ -35,7 +35,7 @@ type V1EnterpriseProductInfoResponse struct { CoverID int `json:"cover_id"` CoverStatus int `json:"cover_status"` } `json:"cover"` - TeachTypeList []string `json:"teach_type_list"` + TeachTypeList []int `json:"teach_type_list"` TeachTypeNameList []string `json:"teach_type_name_list"` Article struct { Count int `json:"count"` From 38145484cdd15259c5c09a7eb7e7047fe99bdf5a Mon Sep 17 00:00:00 2001 From: yinyicao Date: Tue, 2 Apr 2024 13:55:27 +0800 Subject: [PATCH 4/5] style(Enterprise): modify code style --- cmd/root.go | 2 +- internal/geektime/geektime.go | 14 +++++++++----- internal/video/video.go | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 573ca39..37d254b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -237,7 +237,7 @@ func loadProduct(ctx context.Context, productID int) { // university don't need check product type // if input invalid id, access mark is 0 } else if isEnterpriseUniversity() { - p, err = geekEnterpriseClient.ArticlesInfo(productID) + p, err = geekEnterpriseClient.EnterpriseArticlesInfo(productID) } else { p, err = geektimeClient.ColumnInfo(productID) if err == nil { diff --git a/internal/geektime/geektime.go b/internal/geektime/geektime.go index 77e983d..aeb5146 100644 --- a/internal/geektime/geektime.go +++ b/internal/geektime/geektime.go @@ -50,10 +50,14 @@ const ( // UniversityV1MyClassInfoPath get university class info and all articles info in it UniversityV1MyClassInfoPath = "/serv/v1/myclass/info" - V1EnterpriseProductInfoPath = "/app/v1/course/info" - V1EnterpriseArticlesInfoPath = "/app/v1/course/articles" + // V1EnterpriseProductInfoPath used in enterprise course product info + V1EnterpriseProductInfoPath = "/app/v1/course/info" + // V1EnterpriseArticlesInfoPath used in enterprise course articles info + V1EnterpriseArticlesInfoPath = "/app/v1/course/articles" + // V1EnterpriseArticleDetailInfoPath used in enterprise course article detail info V1EnterpriseArticleDetailInfoPath = "/app/v1/article/detail" - V1EnterpriseVideoPlayAuthPath = "/app/v1/source_auth/video_play_auth" + // V1EnterpriseVideoPlayAuthPath used in enterprise course video play auth + V1EnterpriseVideoPlayAuthPath = "/app/v1/source_auth/video_play_auth" // GeekBangCookieDomain ... GeekBangCookieDomain = ".geekbang.org" @@ -364,7 +368,7 @@ func (c *Client) enterpriseProductInfo(productID int) (response.V1EnterpriseProd return res, nil } -func (c *Client) ArticlesInfo(id int) (Product, error) { +func (c *Client) EnterpriseArticlesInfo(id int) (Product, error) { var p Product productInfo, err := c.enterpriseProductInfo(id) if err != nil { @@ -434,7 +438,7 @@ func (c *Client) V1EnterpriseArticleDetailInfo(articleID string) (response.V1Ent return res, nil } -func (c *Client) EnterPriseVideoPlayAuth(articleID, videoID string) (string, error) { +func (c *Client) EnterpriseVideoPlayAuth(articleID, videoID string) (string, error) { var res response.V3VideoPlayAuthResponse r := c.newRequest(resty.MethodPost, V1EnterpriseVideoPlayAuthPath, diff --git a/internal/video/video.go b/internal/video/video.go index c46dda8..408ae02 100644 --- a/internal/video/video.go +++ b/internal/video/video.go @@ -93,7 +93,7 @@ func DownloadEnterpriseArticleVideo(ctx context.Context, if articleInfo.Data.Video.ID == "" { return nil } - playAuth, err := client.EnterPriseVideoPlayAuth(strconv.Itoa(articleID), articleInfo.Data.Video.ID) + playAuth, err := client.EnterpriseVideoPlayAuth(strconv.Itoa(articleID), articleInfo.Data.Video.ID) if err != nil { return err } From 7062a7cc9f12dbeb2ffb5d231495fd7ad99aec72 Mon Sep 17 00:00:00 2001 From: yinyicao Date: Sun, 7 Apr 2024 10:43:03 +0800 Subject: [PATCH 5/5] style(Enterprise): modify code style --- cmd/root.go | 2 -- internal/geektime/geektime.go | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 37d254b..01d54f0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/nicoxiang/geektime-downloader/internal/geektime/response" "math" "math/rand" "net/http" @@ -37,7 +36,6 @@ var ( downloadFolder string sp *spinner.Spinner selectedProduct geektime.Product - selectedEnterProduct response.V1EnterpriseArticlesResponse quality string downloadComments bool selectedProductType productTypeSelectOption diff --git a/internal/geektime/geektime.go b/internal/geektime/geektime.go index aeb5146..24f72e2 100644 --- a/internal/geektime/geektime.go +++ b/internal/geektime/geektime.go @@ -391,7 +391,7 @@ func (c *Client) EnterpriseArticlesInfo(id int) (Product, error) { } if res.Code != 0 { - if res.Data.IsShow == false && productInfo.Data.Extra.IsMyCourse == false { + if !res.Data.IsShow && !productInfo.Data.Extra.IsMyCourse { p.Access = false return p, nil } @@ -409,9 +409,9 @@ func (c *Client) EnterpriseArticlesInfo(id int) (Product, error) { for _, sections := range res.Data.List { for _, a := range sections.ArticleList { - articleId, _ := strconv.Atoi(a.Article.ID) + articleID, _ := strconv.Atoi(a.Article.ID) articles = append(articles, Article{ - AID: articleId, + AID: articleID, SectionTitle: sections.Title, Title: a.Article.Title, })