From 3e9ed2c026de8225268f0ef970968683ef60fd07 Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Sat, 23 Nov 2024 18:18:10 +1000 Subject: [PATCH 1/8] readme update --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b6e05ea4..f10769f7 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,12 @@ suggested a feature, helped to reproduce, or spent time chatting with me on the Telegram or Slack to help to understand the issue and tested the proposed solution. -Also, I'd like to thank all those who made a donation to support the project: +Also, I'd like to thank current sponsors: + +- @malsatin + +And everyone who made a donation to support the project in the past and keep +supporting the project: - Vivek R. - Fabian I. From d9b0cbabd1813275a0c2a1f497d69539f04155a4 Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Sat, 23 Nov 2024 18:20:20 +1000 Subject: [PATCH 2/8] html is hard --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f10769f7..3892423a 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,7 @@ solution. Also, I'd like to thank current sponsors: -- @malsatin +- [@malsatin](https://github.com/malsatin) @malsatin And everyone who made a donation to support the project in the past and keep supporting the project: From d15d0ef0126aa82704c02b7e6e0fdc786441be1d Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Sat, 23 Nov 2024 18:07:31 +1000 Subject: [PATCH 3/8] emoji experiment --- clienter_mock_test.go | 16 ++++ cmd/slackdump/internal/emoji/emojidl/emoji.go | 12 +-- .../internal/emoji/emojidl/emoji_mock_test.go | 15 ++-- .../internal/emoji/emojidl/emoji_test.go | 27 +++--- .../workspaceui/{new.go => workspaceui.go} | 4 +- emoji.go | 22 +++++ go.mod | 2 +- go.sum | 2 + internal/edge/edge.go | 7 ++ internal/edge/emoji.go | 88 +++++++++++++++++++ internal/edge/files.go | 2 +- internal/edge/limits.go | 2 +- internal/edge/wrapper.go | 5 ++ slackdump.go | 1 + 14 files changed, 175 insertions(+), 30 deletions(-) rename cmd/slackdump/internal/workspace/workspaceui/{new.go => workspaceui.go} (97%) create mode 100644 internal/edge/emoji.go diff --git a/clienter_mock_test.go b/clienter_mock_test.go index fa0a8871..f9398554 100644 --- a/clienter_mock_test.go +++ b/clienter_mock_test.go @@ -239,6 +239,22 @@ func (m *mockClienter) EXPECT() *mockClienterMockRecorder { return m.recorder } +// AdminEmojiList mocks base method. +func (m *mockClienter) AdminEmojiList(ctx context.Context, params slack.AdminEmojiListParameters) (map[string]slack.Emoji, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdminEmojiList", ctx, params) + ret0, _ := ret[0].(map[string]slack.Emoji) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// AdminEmojiList indicates an expected call of AdminEmojiList. +func (mr *mockClienterMockRecorder) AdminEmojiList(ctx, params any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdminEmojiList", reflect.TypeOf((*mockClienter)(nil).AdminEmojiList), ctx, params) +} + // AuthTestContext mocks base method. func (m *mockClienter) AuthTestContext(arg0 context.Context) (*slack.AuthTestResponse, error) { m.ctrl.T.Helper() diff --git a/cmd/slackdump/internal/emoji/emojidl/emoji.go b/cmd/slackdump/internal/emoji/emojidl/emoji.go index 7b6bd55b..8dd22db6 100644 --- a/cmd/slackdump/internal/emoji/emojidl/emoji.go +++ b/cmd/slackdump/internal/emoji/emojidl/emoji.go @@ -27,6 +27,7 @@ import ( "sync" "github.com/rusq/fsadapter" + "github.com/rusq/slack" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" ) @@ -39,12 +40,13 @@ var fetchFn = fetchEmoji //go:generate mockgen -source emoji.go -destination emoji_mock_test.go -package emojidl type emojidumper interface { - DumpEmojis(ctx context.Context) (map[string]string, error) + // DumpEmojis(ctx context.Context) (map[string]string, error) + DumpEmojisAdmin(ctx context.Context) (map[string]slack.Emoji, error) } // DlFS downloads all emojis from the workspace and saves them to the fsa. func DlFS(ctx context.Context, sess emojidumper, fsa fsadapter.FS, failFast bool) error { - emojis, err := sess.DumpEmojis(ctx) + emojis, err := sess.DumpEmojisAdmin(ctx) if err != nil { return fmt.Errorf("error during emoji dump: %w", err) } @@ -62,7 +64,7 @@ func DlFS(ctx context.Context, sess emojidumper, fsa fsadapter.FS, failFast bool // fetch downloads the emojis and saves them to the fsa. It spawns numWorker // goroutines for getting the files. It will call fetchFn for each emoji. -func fetch(ctx context.Context, fsa fsadapter.FS, emojis map[string]string, failFast bool) error { +func fetch(ctx context.Context, fsa fsadapter.FS, emojis map[string]slack.Emoji, failFast bool) error { lg := cfg.Log.With("in", "fetch", "dir", emojiDir, "numWorkers", numWorkers, "failFast", failFast) var ( @@ -75,11 +77,11 @@ func fetch(ctx context.Context, fsa fsadapter.FS, emojis map[string]string, fail // 1. generator, send emojis into the emojiC channel. go func() { defer close(emojiC) - for name, uri := range emojis { + for name, em := range emojis { select { case <-ctx.Done(): return - case emojiC <- emoji{name, uri}: + case emojiC <- emoji{name, em.URL}: } } }() diff --git a/cmd/slackdump/internal/emoji/emojidl/emoji_mock_test.go b/cmd/slackdump/internal/emoji/emojidl/emoji_mock_test.go index 17f964d9..762aec9d 100644 --- a/cmd/slackdump/internal/emoji/emojidl/emoji_mock_test.go +++ b/cmd/slackdump/internal/emoji/emojidl/emoji_mock_test.go @@ -13,6 +13,7 @@ import ( context "context" reflect "reflect" + slack "github.com/rusq/slack" gomock "go.uber.org/mock/gomock" ) @@ -40,17 +41,17 @@ func (m *Mockemojidumper) EXPECT() *MockemojidumperMockRecorder { return m.recorder } -// DumpEmojis mocks base method. -func (m *Mockemojidumper) DumpEmojis(ctx context.Context) (map[string]string, error) { +// DumpEmojisAdmin mocks base method. +func (m *Mockemojidumper) DumpEmojisAdmin(ctx context.Context) (map[string]slack.Emoji, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DumpEmojis", ctx) - ret0, _ := ret[0].(map[string]string) + ret := m.ctrl.Call(m, "DumpEmojisAdmin", ctx) + ret0, _ := ret[0].(map[string]slack.Emoji) ret1, _ := ret[1].(error) return ret0, ret1 } -// DumpEmojis indicates an expected call of DumpEmojis. -func (mr *MockemojidumperMockRecorder) DumpEmojis(ctx any) *gomock.Call { +// DumpEmojisAdmin indicates an expected call of DumpEmojisAdmin. +func (mr *MockemojidumperMockRecorder) DumpEmojisAdmin(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DumpEmojis", reflect.TypeOf((*Mockemojidumper)(nil).DumpEmojis), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DumpEmojisAdmin", reflect.TypeOf((*Mockemojidumper)(nil).DumpEmojisAdmin), ctx) } diff --git a/cmd/slackdump/internal/emoji/emojidl/emoji_test.go b/cmd/slackdump/internal/emoji/emojidl/emoji_test.go index 94a11e66..6e3d4110 100644 --- a/cmd/slackdump/internal/emoji/emojidl/emoji_test.go +++ b/cmd/slackdump/internal/emoji/emojidl/emoji_test.go @@ -17,6 +17,7 @@ import ( "go.uber.org/mock/gomock" "github.com/rusq/fsadapter" + "github.com/rusq/slack" ) type fetchFunc func(ctx context.Context, fsa fsadapter.FS, dir string, name string, uri string) error @@ -225,10 +226,10 @@ func Test_fetch(t *testing.T) { } } -func generateEmojis(n int) (ret map[string]string) { - ret = make(map[string]string, n) +func generateEmojis(n int) (ret map[string]slack.Emoji) { + ret = make(map[string]slack.Emoji, n) for i := 0; i < n; i++ { - ret[randString(10)] = "https://emoji.slack.com/" + randString(20) + ret[randString(10)] = slack.Emoji{URL: "https://emoji.slack.com/" + randString(20)} } return } @@ -268,9 +269,9 @@ func Test_download(t *testing.T) { emptyFetchFn, func(m *Mockemojidumper) { m.EXPECT(). - DumpEmojis(gomock.Any()). - Return(map[string]string{ - "test": "https://blahblah.png", + DumpEmojisAdmin(gomock.Any()). + Return(map[string]slack.Emoji{ + "test": {URL: "https://blahblah.png"}, }, nil) }, false, @@ -285,9 +286,9 @@ func Test_download(t *testing.T) { emptyFetchFn, func(m *Mockemojidumper) { m.EXPECT(). - DumpEmojis(gomock.Any()). - Return(map[string]string{ - "test": "https://blahblah.png", + DumpEmojisAdmin(gomock.Any()). + Return(map[string]slack.Emoji{ + "test": {URL: "https://blahblah.png"}, }, nil) }, false, @@ -302,9 +303,9 @@ func Test_download(t *testing.T) { errorFetchFn, func(m *Mockemojidumper) { m.EXPECT(). - DumpEmojis(gomock.Any()). - Return(map[string]string{ - "test": "https://blahblah.png", + DumpEmojisAdmin(gomock.Any()). + Return(map[string]slack.Emoji{ + "test": {URL: "https://blahblah.png"}, }, nil) }, true, @@ -319,7 +320,7 @@ func Test_download(t *testing.T) { errorFetchFn, func(m *Mockemojidumper) { m.EXPECT(). - DumpEmojis(gomock.Any()). + DumpEmojisAdmin(gomock.Any()). Return(nil, errors.New("no emojis for you, it's 1991.")) }, true, diff --git a/cmd/slackdump/internal/workspace/workspaceui/new.go b/cmd/slackdump/internal/workspace/workspaceui/workspaceui.go similarity index 97% rename from cmd/slackdump/internal/workspace/workspaceui/new.go rename to cmd/slackdump/internal/workspace/workspaceui/workspaceui.go index a6dfeaa4..fa7c36b8 100644 --- a/cmd/slackdump/internal/workspace/workspaceui/new.go +++ b/cmd/slackdump/internal/workspace/workspaceui/workspaceui.go @@ -76,7 +76,7 @@ func ShowUI(ctx context.Context, opts ...UIOption) error { }, { ID: actBrowserOpts, - Name: "Browser Options", + Name: "Browser Options...", Help: "Show browser options", Preview: true, Model: cfgui.NewConfigUI(cfgui.DefaultStyle(), configuration(&brwsOpts)), @@ -123,7 +123,7 @@ func ShowUI(ctx context.Context, opts ...UIOption) error { var lastID string = actLogin LOOP: for { - m := menu.New(uiOpts.title, items, uiOpts.quicklogin) + m := menu.New(uiOpts.title, items, true) m.Select(lastID) if _, err := tea.NewProgram(&wizModel{m: m}, tea.WithContext(ctx)).Run(); err != nil { return err diff --git a/emoji.go b/emoji.go index 35e2c4c6..a582d4b8 100644 --- a/emoji.go +++ b/emoji.go @@ -2,6 +2,8 @@ package slackdump import ( "context" + + "github.com/rusq/slack" ) func (s *Session) DumpEmojis(ctx context.Context) (map[string]string, error) { @@ -11,3 +13,23 @@ func (s *Session) DumpEmojis(ctx context.Context) (map[string]string, error) { } return emoji, nil } + +func (s *Session) DumpEmojisAdmin(ctx context.Context) (map[string]slack.Emoji, error) { + var ret = make(map[string]slack.Emoji, 100) + + p := slack.AdminEmojiListParameters{Cursor: "", Limit: 100} + for { + emoji, next, err := s.client.AdminEmojiList(ctx, p) + if err != nil { + return nil, err + } + for k, v := range emoji { + ret[k] = v + } + if next == "" { + break + } + p.Cursor = next + } + return ret, nil +} diff --git a/go.mod b/go.mod index 93866f11..2f60cf66 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/rusq/fsadapter v1.0.2 github.com/rusq/osenv/v2 v2.0.1 github.com/rusq/rbubbles v0.0.2 - github.com/rusq/slack v0.9.6-0.20241104074952-d9b6e02955fa + github.com/rusq/slack v0.9.6-0.20241122224849-576a79dc22f1 github.com/rusq/slackauth v0.5.1 github.com/rusq/tagops v0.0.2 github.com/rusq/tracer v1.0.1 diff --git a/go.sum b/go.sum index 0d1d15df..c55accf1 100644 --- a/go.sum +++ b/go.sum @@ -137,6 +137,8 @@ github.com/rusq/secure v0.0.4 h1:svpiZHfHnx89eEDCCFI9OXG1Y8hL9kUWUG6fJbrWUOI= github.com/rusq/secure v0.0.4/go.mod h1:F1QilMKreuFRjov0UY7DZSIXn77/8RqMVGu2zV0RtqY= github.com/rusq/slack v0.9.6-0.20241104074952-d9b6e02955fa h1:meNaDH2eLwjAqvOxMlgb5+gaLz3Kufm9rVFkALhsCRs= github.com/rusq/slack v0.9.6-0.20241104074952-d9b6e02955fa/go.mod h1:9O0zQAFN6W47z4KpTQbe6vOHOzBO76vMg1+gthPwaTI= +github.com/rusq/slack v0.9.6-0.20241122224849-576a79dc22f1 h1:70BrReHUHQ/ERHqGxUgXJrlXKE5jA++bzo+F9Q2b9Pw= +github.com/rusq/slack v0.9.6-0.20241122224849-576a79dc22f1/go.mod h1:9O0zQAFN6W47z4KpTQbe6vOHOzBO76vMg1+gthPwaTI= github.com/rusq/slackauth v0.5.1 h1:l+Gj96kYzHmljMYglRv76kgzuOJr/QbXDDA8JHyN71Q= github.com/rusq/slackauth v0.5.1/go.mod h1:wAtNCbeKH0pnaZnqJjG5RKY3e5BF9F2L/YTzhOjBIb0= github.com/rusq/tagops v0.0.2 h1:LkWsmpYSH5Q5IX3pv0Qm5PEKOtfjKqrwbJ3c19C1pvM= diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 7e8c351d..3e439c16 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -205,6 +205,13 @@ func (cl *Client) callEdgeAPI(ctx context.Context, v any, endpoint string, req P return cl.ParseResponse(v, r) } +// PostForm sends a POST request to a webclient API, it marshals the form +// values to url.Values, omitting empty fields, and sends the request. +func (cl *Client) Post(ctx context.Context, path string, a any) (*http.Response, error) { + return cl.PostFormRaw(ctx, cl.webclientAPI+path, values(a, true)) +} + +// PostForm sends a POST request to a webclient API with form values. func (cl *Client) PostForm(ctx context.Context, path string, form url.Values) (*http.Response, error) { return cl.PostFormRaw(ctx, cl.webclientAPI+path, form) } diff --git a/internal/edge/emoji.go b/internal/edge/emoji.go new file mode 100644 index 00000000..a1939342 --- /dev/null +++ b/internal/edge/emoji.go @@ -0,0 +1,88 @@ +package edge + +import "context" + +type EmojiResponse struct { + BaseResponse + EmojiResult + CustomEmojiTotalCount int64 `json:"custom_emoji_total_count"` + Paging Paging `json:"paging"` +} + +type EmojiResult struct { + Emoji []Emoji `json:"emoji"` + DisabledEmoji []Emoji `json:"disabled_emoji"` +} + +type Emoji struct { + Name string `json:"name"` + IsAlias int64 `json:"is_alias"` + AliasFor string `json:"alias_for"` + URL string `json:"url"` + TeamID string `json:"team_id"` + UserID string `json:"user_id"` + Created int64 `json:"created"` + IsBad bool `json:"is_bad"` + UserDisplayName string `json:"user_display_name"` + AvatarHash string `json:"avatar_hash"` + CanDelete bool `json:"can_delete"` + Synonyms []string `json:"synonyms"` +} + +type Paging struct { + Count int64 `json:"count,omitempty"` + Total int64 `json:"total,omitempty"` + Page int64 `json:"page,omitempty"` + Pages int64 `json:"pages,omitempty"` +} + +func (p *Paging) isLastPage() bool { + return p.Page >= p.Pages || p.Pages == 0 +} + +func (p *Paging) nextPage() int64 { + old := p.Page + p.Page++ + return old +} + +type adminEmojiListRequest struct { + BaseRequest + WebClientFields + Paging +} + +func (cl *Client) AdminEmojiList(ctx context.Context) (EmojiResult, error) { + var res EmojiResult + req := adminEmojiListRequest{ + BaseRequest: BaseRequest{ + Token: cl.token, + }, + Paging: Paging{ + Page: 1, + Count: 100, + }, + WebClientFields: webclientReason("customize-emoji-new-query"), + } + l := tier3.limiter() + for { + resp, err := cl.Post(ctx, "emoji.adminList", req) + if err != nil { + return res, err + } + var r EmojiResponse + if err := cl.ParseResponse(&r, resp); err != nil { + return res, err + } + res.Emoji = append(res.Emoji, r.Emoji...) + res.DisabledEmoji = append(res.DisabledEmoji, r.DisabledEmoji...) + if r.Paging.isLastPage() { + break + } + r.Paging.nextPage() + if err := l.Wait(ctx); err != nil { + return res, err + } + } + return res, nil +} diff --git a/internal/edge/files.go b/internal/edge/files.go index 2432258d..b6c9c85f 100644 --- a/internal/edge/files.go +++ b/internal/edge/files.go @@ -32,7 +32,7 @@ func (cl *Client) FilesList(ctx context.Context, channel string, count int) ([]s lim := tier3.limiter() var ff []slack.File for { - resp, err := cl.PostForm(ctx, "files.list", values(form, true)) + resp, err := cl.Post(ctx, "files.list", form) if err != nil { return nil, err } diff --git a/internal/edge/limits.go b/internal/edge/limits.go index c5c8c27c..7c672dae 100644 --- a/internal/edge/limits.go +++ b/internal/edge/limits.go @@ -7,7 +7,7 @@ import ( ) type tier struct { - // once eveyr + // once every t time.Duration // burst b int diff --git a/internal/edge/wrapper.go b/internal/edge/wrapper.go index c4b05b5a..86306267 100644 --- a/internal/edge/wrapper.go +++ b/internal/edge/wrapper.go @@ -79,3 +79,8 @@ func (w *Wrapper) SearchMessagesContext(ctx context.Context, query string, param func (w *Wrapper) SearchFilesContext(ctx context.Context, query string, params slack.SearchParameters) (*slack.SearchFiles, error) { return w.cl.SearchFilesContext(ctx, query, params) } + +func (w *Wrapper) AdminEmojiList(ctx context.Context, params slack.AdminEmojiListParameters) (map[string]slack.Emoji, string, error) { + // TODO: switch to edge client + return w.cl.AdminEmojiList(ctx, params) +} diff --git a/slackdump.go b/slackdump.go index f9197196..26d848da 100644 --- a/slackdump.go +++ b/slackdump.go @@ -64,6 +64,7 @@ type clienter interface { GetFile(downloadURL string, writer io.Writer) error GetUsersContext(ctx context.Context, options ...slack.GetUsersOption) ([]slack.User, error) GetEmojiContext(ctx context.Context) (map[string]string, error) + AdminEmojiList(ctx context.Context, params slack.AdminEmojiListParameters) (map[string]slack.Emoji, string, error) } // ErrNoUserCache is returned when the user cache is not initialised. From 5c574060adf890a3a2e8a4e02e7cc06f88543b45 Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Sat, 23 Nov 2024 18:40:33 +1000 Subject: [PATCH 4/8] rollback emoji test --- clienter_mock_test.go | 16 ----------- cmd/slackdump/internal/emoji/emojidl/emoji.go | 12 ++++----- .../internal/emoji/emojidl/emoji_mock_test.go | 15 +++++------ .../internal/emoji/emojidl/emoji_test.go | 27 +++++++++---------- emoji.go | 22 --------------- go.mod | 2 +- go.sum | 2 ++ internal/edge/emoji.go | 3 ++- internal/edge/wrapper.go | 5 ---- slackdump.go | 1 - 10 files changed, 30 insertions(+), 75 deletions(-) diff --git a/clienter_mock_test.go b/clienter_mock_test.go index f9398554..fa0a8871 100644 --- a/clienter_mock_test.go +++ b/clienter_mock_test.go @@ -239,22 +239,6 @@ func (m *mockClienter) EXPECT() *mockClienterMockRecorder { return m.recorder } -// AdminEmojiList mocks base method. -func (m *mockClienter) AdminEmojiList(ctx context.Context, params slack.AdminEmojiListParameters) (map[string]slack.Emoji, string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AdminEmojiList", ctx, params) - ret0, _ := ret[0].(map[string]slack.Emoji) - ret1, _ := ret[1].(string) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// AdminEmojiList indicates an expected call of AdminEmojiList. -func (mr *mockClienterMockRecorder) AdminEmojiList(ctx, params any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdminEmojiList", reflect.TypeOf((*mockClienter)(nil).AdminEmojiList), ctx, params) -} - // AuthTestContext mocks base method. func (m *mockClienter) AuthTestContext(arg0 context.Context) (*slack.AuthTestResponse, error) { m.ctrl.T.Helper() diff --git a/cmd/slackdump/internal/emoji/emojidl/emoji.go b/cmd/slackdump/internal/emoji/emojidl/emoji.go index 8dd22db6..7b6bd55b 100644 --- a/cmd/slackdump/internal/emoji/emojidl/emoji.go +++ b/cmd/slackdump/internal/emoji/emojidl/emoji.go @@ -27,7 +27,6 @@ import ( "sync" "github.com/rusq/fsadapter" - "github.com/rusq/slack" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" ) @@ -40,13 +39,12 @@ var fetchFn = fetchEmoji //go:generate mockgen -source emoji.go -destination emoji_mock_test.go -package emojidl type emojidumper interface { - // DumpEmojis(ctx context.Context) (map[string]string, error) - DumpEmojisAdmin(ctx context.Context) (map[string]slack.Emoji, error) + DumpEmojis(ctx context.Context) (map[string]string, error) } // DlFS downloads all emojis from the workspace and saves them to the fsa. func DlFS(ctx context.Context, sess emojidumper, fsa fsadapter.FS, failFast bool) error { - emojis, err := sess.DumpEmojisAdmin(ctx) + emojis, err := sess.DumpEmojis(ctx) if err != nil { return fmt.Errorf("error during emoji dump: %w", err) } @@ -64,7 +62,7 @@ func DlFS(ctx context.Context, sess emojidumper, fsa fsadapter.FS, failFast bool // fetch downloads the emojis and saves them to the fsa. It spawns numWorker // goroutines for getting the files. It will call fetchFn for each emoji. -func fetch(ctx context.Context, fsa fsadapter.FS, emojis map[string]slack.Emoji, failFast bool) error { +func fetch(ctx context.Context, fsa fsadapter.FS, emojis map[string]string, failFast bool) error { lg := cfg.Log.With("in", "fetch", "dir", emojiDir, "numWorkers", numWorkers, "failFast", failFast) var ( @@ -77,11 +75,11 @@ func fetch(ctx context.Context, fsa fsadapter.FS, emojis map[string]slack.Emoji, // 1. generator, send emojis into the emojiC channel. go func() { defer close(emojiC) - for name, em := range emojis { + for name, uri := range emojis { select { case <-ctx.Done(): return - case emojiC <- emoji{name, em.URL}: + case emojiC <- emoji{name, uri}: } } }() diff --git a/cmd/slackdump/internal/emoji/emojidl/emoji_mock_test.go b/cmd/slackdump/internal/emoji/emojidl/emoji_mock_test.go index 762aec9d..17f964d9 100644 --- a/cmd/slackdump/internal/emoji/emojidl/emoji_mock_test.go +++ b/cmd/slackdump/internal/emoji/emojidl/emoji_mock_test.go @@ -13,7 +13,6 @@ import ( context "context" reflect "reflect" - slack "github.com/rusq/slack" gomock "go.uber.org/mock/gomock" ) @@ -41,17 +40,17 @@ func (m *Mockemojidumper) EXPECT() *MockemojidumperMockRecorder { return m.recorder } -// DumpEmojisAdmin mocks base method. -func (m *Mockemojidumper) DumpEmojisAdmin(ctx context.Context) (map[string]slack.Emoji, error) { +// DumpEmojis mocks base method. +func (m *Mockemojidumper) DumpEmojis(ctx context.Context) (map[string]string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DumpEmojisAdmin", ctx) - ret0, _ := ret[0].(map[string]slack.Emoji) + ret := m.ctrl.Call(m, "DumpEmojis", ctx) + ret0, _ := ret[0].(map[string]string) ret1, _ := ret[1].(error) return ret0, ret1 } -// DumpEmojisAdmin indicates an expected call of DumpEmojisAdmin. -func (mr *MockemojidumperMockRecorder) DumpEmojisAdmin(ctx any) *gomock.Call { +// DumpEmojis indicates an expected call of DumpEmojis. +func (mr *MockemojidumperMockRecorder) DumpEmojis(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DumpEmojisAdmin", reflect.TypeOf((*Mockemojidumper)(nil).DumpEmojisAdmin), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DumpEmojis", reflect.TypeOf((*Mockemojidumper)(nil).DumpEmojis), ctx) } diff --git a/cmd/slackdump/internal/emoji/emojidl/emoji_test.go b/cmd/slackdump/internal/emoji/emojidl/emoji_test.go index 6e3d4110..94a11e66 100644 --- a/cmd/slackdump/internal/emoji/emojidl/emoji_test.go +++ b/cmd/slackdump/internal/emoji/emojidl/emoji_test.go @@ -17,7 +17,6 @@ import ( "go.uber.org/mock/gomock" "github.com/rusq/fsadapter" - "github.com/rusq/slack" ) type fetchFunc func(ctx context.Context, fsa fsadapter.FS, dir string, name string, uri string) error @@ -226,10 +225,10 @@ func Test_fetch(t *testing.T) { } } -func generateEmojis(n int) (ret map[string]slack.Emoji) { - ret = make(map[string]slack.Emoji, n) +func generateEmojis(n int) (ret map[string]string) { + ret = make(map[string]string, n) for i := 0; i < n; i++ { - ret[randString(10)] = slack.Emoji{URL: "https://emoji.slack.com/" + randString(20)} + ret[randString(10)] = "https://emoji.slack.com/" + randString(20) } return } @@ -269,9 +268,9 @@ func Test_download(t *testing.T) { emptyFetchFn, func(m *Mockemojidumper) { m.EXPECT(). - DumpEmojisAdmin(gomock.Any()). - Return(map[string]slack.Emoji{ - "test": {URL: "https://blahblah.png"}, + DumpEmojis(gomock.Any()). + Return(map[string]string{ + "test": "https://blahblah.png", }, nil) }, false, @@ -286,9 +285,9 @@ func Test_download(t *testing.T) { emptyFetchFn, func(m *Mockemojidumper) { m.EXPECT(). - DumpEmojisAdmin(gomock.Any()). - Return(map[string]slack.Emoji{ - "test": {URL: "https://blahblah.png"}, + DumpEmojis(gomock.Any()). + Return(map[string]string{ + "test": "https://blahblah.png", }, nil) }, false, @@ -303,9 +302,9 @@ func Test_download(t *testing.T) { errorFetchFn, func(m *Mockemojidumper) { m.EXPECT(). - DumpEmojisAdmin(gomock.Any()). - Return(map[string]slack.Emoji{ - "test": {URL: "https://blahblah.png"}, + DumpEmojis(gomock.Any()). + Return(map[string]string{ + "test": "https://blahblah.png", }, nil) }, true, @@ -320,7 +319,7 @@ func Test_download(t *testing.T) { errorFetchFn, func(m *Mockemojidumper) { m.EXPECT(). - DumpEmojisAdmin(gomock.Any()). + DumpEmojis(gomock.Any()). Return(nil, errors.New("no emojis for you, it's 1991.")) }, true, diff --git a/emoji.go b/emoji.go index a582d4b8..35e2c4c6 100644 --- a/emoji.go +++ b/emoji.go @@ -2,8 +2,6 @@ package slackdump import ( "context" - - "github.com/rusq/slack" ) func (s *Session) DumpEmojis(ctx context.Context) (map[string]string, error) { @@ -13,23 +11,3 @@ func (s *Session) DumpEmojis(ctx context.Context) (map[string]string, error) { } return emoji, nil } - -func (s *Session) DumpEmojisAdmin(ctx context.Context) (map[string]slack.Emoji, error) { - var ret = make(map[string]slack.Emoji, 100) - - p := slack.AdminEmojiListParameters{Cursor: "", Limit: 100} - for { - emoji, next, err := s.client.AdminEmojiList(ctx, p) - if err != nil { - return nil, err - } - for k, v := range emoji { - ret[k] = v - } - if next == "" { - break - } - p.Cursor = next - } - return ret, nil -} diff --git a/go.mod b/go.mod index 2f60cf66..2d121e54 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/rusq/fsadapter v1.0.2 github.com/rusq/osenv/v2 v2.0.1 github.com/rusq/rbubbles v0.0.2 - github.com/rusq/slack v0.9.6-0.20241122224849-576a79dc22f1 + github.com/rusq/slack v0.9.6-0.20241117083852-278084780c45 github.com/rusq/slackauth v0.5.1 github.com/rusq/tagops v0.0.2 github.com/rusq/tracer v1.0.1 diff --git a/go.sum b/go.sum index c55accf1..a4248588 100644 --- a/go.sum +++ b/go.sum @@ -137,6 +137,8 @@ github.com/rusq/secure v0.0.4 h1:svpiZHfHnx89eEDCCFI9OXG1Y8hL9kUWUG6fJbrWUOI= github.com/rusq/secure v0.0.4/go.mod h1:F1QilMKreuFRjov0UY7DZSIXn77/8RqMVGu2zV0RtqY= github.com/rusq/slack v0.9.6-0.20241104074952-d9b6e02955fa h1:meNaDH2eLwjAqvOxMlgb5+gaLz3Kufm9rVFkALhsCRs= github.com/rusq/slack v0.9.6-0.20241104074952-d9b6e02955fa/go.mod h1:9O0zQAFN6W47z4KpTQbe6vOHOzBO76vMg1+gthPwaTI= +github.com/rusq/slack v0.9.6-0.20241117083852-278084780c45 h1:tsZKbEaziqVowGaQ7zRsrxpy9JDk6CCkihR5PrMk48s= +github.com/rusq/slack v0.9.6-0.20241117083852-278084780c45/go.mod h1:9O0zQAFN6W47z4KpTQbe6vOHOzBO76vMg1+gthPwaTI= github.com/rusq/slack v0.9.6-0.20241122224849-576a79dc22f1 h1:70BrReHUHQ/ERHqGxUgXJrlXKE5jA++bzo+F9Q2b9Pw= github.com/rusq/slack v0.9.6-0.20241122224849-576a79dc22f1/go.mod h1:9O0zQAFN6W47z4KpTQbe6vOHOzBO76vMg1+gthPwaTI= github.com/rusq/slackauth v0.5.1 h1:l+Gj96kYzHmljMYglRv76kgzuOJr/QbXDDA8JHyN71Q= diff --git a/internal/edge/emoji.go b/internal/edge/emoji.go index a1939342..a5fe91e2 100644 --- a/internal/edge/emoji.go +++ b/internal/edge/emoji.go @@ -52,6 +52,7 @@ type adminEmojiListRequest struct { Paging } +// AdminEmojiList returns a list of custom emoji for the workspace. func (cl *Client) AdminEmojiList(ctx context.Context) (EmojiResult, error) { var res EmojiResult req := adminEmojiListRequest{ @@ -64,7 +65,7 @@ func (cl *Client) AdminEmojiList(ctx context.Context) (EmojiResult, error) { }, WebClientFields: webclientReason("customize-emoji-new-query"), } - l := tier3.limiter() + l := tier2boost.limiter() for { resp, err := cl.Post(ctx, "emoji.adminList", req) if err != nil { diff --git a/internal/edge/wrapper.go b/internal/edge/wrapper.go index 86306267..c4b05b5a 100644 --- a/internal/edge/wrapper.go +++ b/internal/edge/wrapper.go @@ -79,8 +79,3 @@ func (w *Wrapper) SearchMessagesContext(ctx context.Context, query string, param func (w *Wrapper) SearchFilesContext(ctx context.Context, query string, params slack.SearchParameters) (*slack.SearchFiles, error) { return w.cl.SearchFilesContext(ctx, query, params) } - -func (w *Wrapper) AdminEmojiList(ctx context.Context, params slack.AdminEmojiListParameters) (map[string]slack.Emoji, string, error) { - // TODO: switch to edge client - return w.cl.AdminEmojiList(ctx, params) -} diff --git a/slackdump.go b/slackdump.go index 26d848da..f9197196 100644 --- a/slackdump.go +++ b/slackdump.go @@ -64,7 +64,6 @@ type clienter interface { GetFile(downloadURL string, writer io.Writer) error GetUsersContext(ctx context.Context, options ...slack.GetUsersOption) ([]slack.User, error) GetEmojiContext(ctx context.Context) (map[string]string, error) - AdminEmojiList(ctx context.Context, params slack.AdminEmojiListParameters) (map[string]slack.Emoji, string, error) } // ErrNoUserCache is returned when the user cache is not initialised. From 1cac68244b2de59ce7d00455c39bb7b5484f4505 Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:57:30 +1000 Subject: [PATCH 5/8] emoji on edge --- cmd/slackdump/internal/diag/edge.go | 15 ++++++++++++--- cmd/slackdump/internal/diag/tools.go | 2 +- cmd/slackdump/internal/emoji/emoji.go | 14 ++++++++++---- cmd/slackdump/internal/emoji/emojidl/emoji.go | 13 +++++++------ internal/edge/emoji.go | 4 ++-- 5 files changed, 32 insertions(+), 16 deletions(-) diff --git a/cmd/slackdump/internal/diag/edge.go b/cmd/slackdump/internal/diag/edge.go index 6bef250c..c15e6959 100644 --- a/cmd/slackdump/internal/diag/edge.go +++ b/cmd/slackdump/internal/diag/edge.go @@ -49,12 +49,21 @@ func runEdge(ctx context.Context, cmd *base.Command, args []string) error { defer cl.Close() lg.Info("connected") - lg.Info("*** Search for Channels test ***") - channels, err := cl.SearchChannels(ctx, "") + // lg.Info("*** Search for Channels test ***") + // channels, err := cl.SearchChannels(ctx, "") + // if err != nil { + // return err + // } + // if err := save("channels.json", channels); err != nil { + // return err + // } + + lg.Info("*** AdminEmojiList test ***") + emojis, err := cl.AdminEmojiList(ctx) if err != nil { return err } - if err := save("channels.json", channels); err != nil { + if err := save("emoji.json", emojis); err != nil { return err } diff --git a/cmd/slackdump/internal/diag/tools.go b/cmd/slackdump/internal/diag/tools.go index 98e49e42..5608b25c 100644 --- a/cmd/slackdump/internal/diag/tools.go +++ b/cmd/slackdump/internal/diag/tools.go @@ -20,7 +20,7 @@ Tools command contains different tools, running which may be requested if you op PrintFlags: false, RequireAuth: false, Commands: []*base.Command{ - // cmdEdge, + cmdEdge, dmdEncrypt, dmdEzTest, dmdInfo, diff --git a/cmd/slackdump/internal/emoji/emoji.go b/cmd/slackdump/internal/emoji/emoji.go index e5905c0b..4412d27f 100644 --- a/cmd/slackdump/internal/emoji/emoji.go +++ b/cmd/slackdump/internal/emoji/emoji.go @@ -5,11 +5,11 @@ import ( "fmt" "github.com/rusq/fsadapter" - "github.com/rusq/slackdump/v3" - "github.com/rusq/slackdump/v3/cmd/slackdump/internal/bootstrap" + "github.com/rusq/slackdump/v3/auth" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/emoji/emojidl" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/golang/base" + "github.com/rusq/slackdump/v3/internal/edge" ) var CmdEmoji = &base.Command{ @@ -43,11 +43,17 @@ func run(ctx context.Context, cmd *base.Command, args []string) error { } defer fsa.Close() - sess, err := bootstrap.SlackdumpSession(ctx, slackdump.WithFilesystem(fsa)) + prov, err := auth.FromContext(ctx) if err != nil { base.SetExitStatus(base.SApplicationError) - return fmt.Errorf("application error: %s", err) + return err + } + sess, err := edge.New(ctx, prov) + if err != nil { + base.SetExitStatus(base.SApplicationError) + return err } + defer sess.Close() if err := emojidl.DlFS(ctx, sess, fsa, cmdFlags.ignoreErrors); err != nil { base.SetExitStatus(base.SApplicationError) diff --git a/cmd/slackdump/internal/emoji/emojidl/emoji.go b/cmd/slackdump/internal/emoji/emojidl/emoji.go index 7b6bd55b..b4278c5b 100644 --- a/cmd/slackdump/internal/emoji/emojidl/emoji.go +++ b/cmd/slackdump/internal/emoji/emojidl/emoji.go @@ -28,6 +28,7 @@ import ( "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" + "github.com/rusq/slackdump/v3/internal/edge" ) const ( @@ -39,12 +40,12 @@ var fetchFn = fetchEmoji //go:generate mockgen -source emoji.go -destination emoji_mock_test.go -package emojidl type emojidumper interface { - DumpEmojis(ctx context.Context) (map[string]string, error) + AdminEmojiList(ctx context.Context) (edge.EmojiResult, error) } // DlFS downloads all emojis from the workspace and saves them to the fsa. func DlFS(ctx context.Context, sess emojidumper, fsa fsadapter.FS, failFast bool) error { - emojis, err := sess.DumpEmojis(ctx) + emojis, err := sess.AdminEmojiList(ctx) if err != nil { return fmt.Errorf("error during emoji dump: %w", err) } @@ -62,7 +63,7 @@ func DlFS(ctx context.Context, sess emojidumper, fsa fsadapter.FS, failFast bool // fetch downloads the emojis and saves them to the fsa. It spawns numWorker // goroutines for getting the files. It will call fetchFn for each emoji. -func fetch(ctx context.Context, fsa fsadapter.FS, emojis map[string]string, failFast bool) error { +func fetch(ctx context.Context, fsa fsadapter.FS, emojis edge.EmojiResult, failFast bool) error { lg := cfg.Log.With("in", "fetch", "dir", emojiDir, "numWorkers", numWorkers, "failFast", failFast) var ( @@ -75,11 +76,11 @@ func fetch(ctx context.Context, fsa fsadapter.FS, emojis map[string]string, fail // 1. generator, send emojis into the emojiC channel. go func() { defer close(emojiC) - for name, uri := range emojis { + for _, e := range emojis.Emoji { select { case <-ctx.Done(): return - case emojiC <- emoji{name, uri}: + case emojiC <- emoji{e.Name, e.URL}: } } }() @@ -102,7 +103,7 @@ func fetch(ctx context.Context, fsa fsadapter.FS, emojis map[string]string, fail // 4. Result processor, receives download results and logs any errors that // may have occurred. var ( - total = len(emojis) + total = len(emojis.Emoji) count = 0 ) lg = lg.With("total", total) diff --git a/internal/edge/emoji.go b/internal/edge/emoji.go index a5fe91e2..b9f19654 100644 --- a/internal/edge/emoji.go +++ b/internal/edge/emoji.go @@ -11,7 +11,7 @@ type EmojiResponse struct { type EmojiResult struct { Emoji []Emoji `json:"emoji"` - DisabledEmoji []Emoji `json:"disabled_emoji"` + DisabledEmoji []Emoji `json:"disabled_emoji,omitempty"` } type Emoji struct { @@ -80,7 +80,7 @@ func (cl *Client) AdminEmojiList(ctx context.Context) (EmojiResult, error) { if r.Paging.isLastPage() { break } - r.Paging.nextPage() + req.Paging.nextPage() if err := l.Wait(ctx); err != nil { return res, err } From 80d4729b5607856c62ee454f6074be474aeee30b Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Sun, 24 Nov 2024 09:17:06 +1000 Subject: [PATCH 6/8] add legacy emoji dump back, as it is faster --- cmd/slackdump/internal/diag/edge.go | 18 +- cmd/slackdump/internal/emoji/emoji.go | 24 ++- .../internal/emoji/emojidl/emoedge.go | 161 ++++++++++++++++++ cmd/slackdump/internal/emoji/emojidl/emoji.go | 17 +- internal/edge/edge.go | 8 +- internal/edge/emoji.go | 76 +++++++-- internal/edge/limits.go | 2 +- 7 files changed, 276 insertions(+), 30 deletions(-) create mode 100644 cmd/slackdump/internal/emoji/emojidl/emoedge.go diff --git a/cmd/slackdump/internal/diag/edge.go b/cmd/slackdump/internal/diag/edge.go index c15e6959..94568b84 100644 --- a/cmd/slackdump/internal/diag/edge.go +++ b/cmd/slackdump/internal/diag/edge.go @@ -3,6 +3,7 @@ package diag import ( "context" "encoding/json" + "log/slog" "os" "github.com/rusq/slackdump/v3/auth" @@ -59,11 +60,20 @@ func runEdge(ctx context.Context, cmd *base.Command, args []string) error { // } lg.Info("*** AdminEmojiList test ***") - emojis, err := cl.AdminEmojiList(ctx) - if err != nil { - return err + var allEmoji edge.EmojiResult + + var iter = 0 + for res, err := range cl.AdminEmojiList(ctx) { + if err != nil { + return err + } + slog.Info("got emojis", "count", len(res.Emoji), "disabled", len(res.DisabledEmoji), "iter", iter) + iter++ + allEmoji.Emoji = append(allEmoji.Emoji, res.Emoji...) + allEmoji.DisabledEmoji = append(allEmoji.DisabledEmoji, res.DisabledEmoji...) } - if err := save("emoji.json", emojis); err != nil { + + if err := save("emoji.json", allEmoji); err != nil { return err } diff --git a/cmd/slackdump/internal/emoji/emoji.go b/cmd/slackdump/internal/emoji/emoji.go index 4412d27f..ad413cc6 100644 --- a/cmd/slackdump/internal/emoji/emoji.go +++ b/cmd/slackdump/internal/emoji/emoji.go @@ -5,7 +5,9 @@ import ( "fmt" "github.com/rusq/fsadapter" + "github.com/rusq/slackdump/v3" "github.com/rusq/slackdump/v3/auth" + "github.com/rusq/slackdump/v3/cmd/slackdump/internal/bootstrap" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/emoji/emojidl" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/golang/base" @@ -24,6 +26,7 @@ var CmdEmoji = &base.Command{ type options struct { ignoreErrors bool + fullInfo bool } // emoji specific flags @@ -34,6 +37,7 @@ var cmdFlags = options{ func init() { CmdEmoji.Wizard = wizard CmdEmoji.Flag.BoolVar(&cmdFlags.ignoreErrors, "ignore-errors", true, "ignore download errors (skip failed emojis)") + CmdEmoji.Flag.BoolVar(&cmdFlags.fullInfo, "full-info", true, "fetch emojis using Edge API to get full emoji information, including usernames") } func run(ctx context.Context, cmd *base.Command, args []string) error { @@ -48,6 +52,24 @@ func run(ctx context.Context, cmd *base.Command, args []string) error { base.SetExitStatus(base.SApplicationError) return err } + if cmdFlags.fullInfo { + return runEdge(ctx, fsa, prov) + } else { + return runLegacy(ctx, fsa) + } +} + +func runLegacy(ctx context.Context, fsa fsadapter.FS) error { + sess, err := bootstrap.SlackdumpSession(ctx, slackdump.WithFilesystem(fsa)) + if err != nil { + base.SetExitStatus(base.SApplicationError) + return err + } + + return emojidl.DlFS(ctx, sess, fsa, cmdFlags.ignoreErrors) +} + +func runEdge(ctx context.Context, fsa fsadapter.FS, prov auth.Provider) error { sess, err := edge.New(ctx, prov) if err != nil { base.SetExitStatus(base.SApplicationError) @@ -55,7 +77,7 @@ func run(ctx context.Context, cmd *base.Command, args []string) error { } defer sess.Close() - if err := emojidl.DlFS(ctx, sess, fsa, cmdFlags.ignoreErrors); err != nil { + if err := emojidl.DlEdgeFS(ctx, sess, fsa, cmdFlags.ignoreErrors); err != nil { base.SetExitStatus(base.SApplicationError) return fmt.Errorf("application error: %s", err) } diff --git a/cmd/slackdump/internal/emoji/emojidl/emoedge.go b/cmd/slackdump/internal/emoji/emojidl/emoedge.go new file mode 100644 index 00000000..f31cd335 --- /dev/null +++ b/cmd/slackdump/internal/emoji/emojidl/emoedge.go @@ -0,0 +1,161 @@ +// Package emojidl provides functions to dump the all slack emojis for a workspace. +// It skips the "alias" emojis, so only original an emoji with an original name +// is present. If you need to find the alias - lookup the index.json. The +// directory structure is the following: +// +// . +// +- emojis +// | +- foo.png +// | +- bar.png +// : : +// | +- baz.png +// +- index.json +// +// Where index.json contains the emoji index, and *.png files under emojis +// directory are individual emojis. +package emojidl + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "iter" + "sync" + + "github.com/rusq/fsadapter" + "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" + "github.com/rusq/slackdump/v3/internal/edge" +) + +//go:generate mockgen -source emoji.go -destination emoji_mock_test.go -package emojidl +type EdgeEmojiLister interface { + AdminEmojiList(ctx context.Context) iter.Seq2[edge.EmojiResult, error] +} + +// DlEdgeFS downloads the emojis and saves them to the fsa. It spawns numWorker +// goroutines for getting the files. It will call fetchFn for each emoji. +func DlEdgeFS(ctx context.Context, sess EdgeEmojiLister, fsa fsadapter.FS, failFast bool) error { + lg := cfg.Log.With("in", "fetch", "dir", emojiDir, "numWorkers", numWorkers, "failFast", failFast) + + var ( + emojiC = make(chan edge.Emoji) + totalC = make(chan int) + genErrC = make(chan error) + resultC = make(chan edgeResult) + ) + + // Async download pipeline. + + // 1. generator, send emojis into the emojiC channel. + go func() { + var once sync.Once + defer close(totalC) + + defer close(emojiC) + + for chunk, err := range sess.AdminEmojiList(ctx) { + if err != nil { + genErrC <- err + return + } + lg.DebugContext(ctx, "got emojis", "count", len(chunk.Emoji), "disabled", len(chunk.DisabledEmoji), "total", chunk.Total) + once.Do(func() { totalC <- chunk.Total }) // send total count once. + for _, emoji := range chunk.Emoji { + select { + case <-ctx.Done(): + return + case emojiC <- emoji: + } + } + } + }() + + // 2. Download workers, download the emojis. + var wg sync.WaitGroup + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func() { + worker2(ctx, fsa, emojiC, resultC) + wg.Done() + }() + } + // 3. Sentinel, closes the result channel once all workers are finished. + go func() { + wg.Wait() + close(resultC) + }() + + // 4. Result processor, receives download results and logs any errors that + // may have occurred. + var ( + count = 0 + total = <-totalC + ) + var emojis = make(map[string]edge.Emoji, total) +LOOP: + for { + select { + + case genErr := <-genErrC: + if genErr != nil { + return fmt.Errorf("failed to get emoji list: %w", genErr) + } + case res, more := <-resultC: + if !more { + break LOOP + } + if res.err != nil { + if errors.Is(res.err, context.Canceled) { + return res.err + } + if failFast { + return fmt.Errorf("failed: %q: %w", res.emoji.Name, res.err) + } + lg.WarnContext(ctx, "failed", "name", res.emoji.Name, "error", res.err) + } + emojis[res.emoji.Name] = res.emoji // to resemble the legacy code. + count++ + lg.InfoContext(ctx, "downloaded", "count", count, "total", total, "name", res.emoji.Name) + } + } + out, err := fsa.Create("index.json") + if err != nil { + return err + } + defer out.Close() + if err := json.NewEncoder(out).Encode(emojis); err != nil { + return err + } + + return nil +} + +type edgeResult struct { + emoji edge.Emoji + skipped bool + err error +} + +// worker is the function that runs in a separate goroutine and downloads emoji +// received from emojiC. The result of the operation is sent to resultC channel. +// fn is called for each received emoji. +func worker2(ctx context.Context, fsa fsadapter.FS, emojiC <-chan edge.Emoji, resultC chan<- edgeResult) { + for { + select { + case <-ctx.Done(): + resultC <- edgeResult{err: ctx.Err()} + return + case em, more := <-emojiC: + if !more { + return + } + if em.IsAlias != 0 { + resultC <- edgeResult{emoji: em, skipped: true} + break + } + err := fetchFn(ctx, fsa, emojiDir, em.Name, em.URL) + resultC <- edgeResult{emoji: em, err: err} + } + } +} diff --git a/cmd/slackdump/internal/emoji/emojidl/emoji.go b/cmd/slackdump/internal/emoji/emojidl/emoji.go index b4278c5b..0c085634 100644 --- a/cmd/slackdump/internal/emoji/emojidl/emoji.go +++ b/cmd/slackdump/internal/emoji/emojidl/emoji.go @@ -28,7 +28,6 @@ import ( "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" - "github.com/rusq/slackdump/v3/internal/edge" ) const ( @@ -39,13 +38,13 @@ const ( var fetchFn = fetchEmoji //go:generate mockgen -source emoji.go -destination emoji_mock_test.go -package emojidl -type emojidumper interface { - AdminEmojiList(ctx context.Context) (edge.EmojiResult, error) +type EmojiDumper interface { + DumpEmojis(ctx context.Context) (map[string]string, error) } // DlFS downloads all emojis from the workspace and saves them to the fsa. -func DlFS(ctx context.Context, sess emojidumper, fsa fsadapter.FS, failFast bool) error { - emojis, err := sess.AdminEmojiList(ctx) +func DlFS(ctx context.Context, sess EmojiDumper, fsa fsadapter.FS, failFast bool) error { + emojis, err := sess.DumpEmojis(ctx) if err != nil { return fmt.Errorf("error during emoji dump: %w", err) } @@ -63,7 +62,7 @@ func DlFS(ctx context.Context, sess emojidumper, fsa fsadapter.FS, failFast bool // fetch downloads the emojis and saves them to the fsa. It spawns numWorker // goroutines for getting the files. It will call fetchFn for each emoji. -func fetch(ctx context.Context, fsa fsadapter.FS, emojis edge.EmojiResult, failFast bool) error { +func fetch(ctx context.Context, fsa fsadapter.FS, emojis map[string]string, failFast bool) error { lg := cfg.Log.With("in", "fetch", "dir", emojiDir, "numWorkers", numWorkers, "failFast", failFast) var ( @@ -76,11 +75,11 @@ func fetch(ctx context.Context, fsa fsadapter.FS, emojis edge.EmojiResult, failF // 1. generator, send emojis into the emojiC channel. go func() { defer close(emojiC) - for _, e := range emojis.Emoji { + for name, uri := range emojis { select { case <-ctx.Done(): return - case emojiC <- emoji{e.Name, e.URL}: + case emojiC <- emoji{name, uri}: } } }() @@ -103,7 +102,7 @@ func fetch(ctx context.Context, fsa fsadapter.FS, emojis edge.EmojiResult, failF // 4. Result processor, receives download results and logs any errors that // may have occurred. var ( - total = len(emojis.Emoji) + total = len(emojis) count = 0 ) lg = lg.With("total", total) diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 3e439c16..74002a6d 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -15,6 +15,7 @@ import ( "net/http" "net/url" "os" + "runtime/trace" "strings" "time" @@ -255,11 +256,16 @@ func (cl *Client) ParseResponse(req any, r *http.Response) error { // if it receives another rate limit error, it returns slack.RateLimitedError // to let the caller handle it. func do(ctx context.Context, cl *http.Client, req *http.Request) (*http.Response, error) { + ctx, task := trace.NewTask(ctx, "edge.do") + defer task.End() + lg := slog.Default() req.Header.Set("Accept-Language", "en-NZ,en-AU;q=0.9,en;q=0.8") req.Header.Set("User-Agent", slackauth.DefaultUserAgent) + rgn := trace.StartRegion(ctx, "http.Do") resp, err := cl.Do(req) + rgn.End() if err != nil { return nil, err } @@ -268,7 +274,7 @@ func do(ctx context.Context, cl *http.Client, req *http.Request) (*http.Response if err != nil { return nil, err } - lg.Debug("got rate limited, waiting", "delay", wait) + lg.InfoContext(ctx, "got rate limited, waiting", "delay", wait) time.Sleep(wait) resp, err = cl.Do(req) diff --git a/internal/edge/emoji.go b/internal/edge/emoji.go index b9f19654..da4b92c5 100644 --- a/internal/edge/emoji.go +++ b/internal/edge/emoji.go @@ -1,8 +1,12 @@ package edge -import "context" +import ( + "context" + "iter" + "runtime/trace" +) -type EmojiResponse struct { +type emojiResponse struct { BaseResponse EmojiResult CustomEmojiTotalCount int64 `json:"custom_emoji_total_count"` @@ -12,21 +16,22 @@ type EmojiResponse struct { type EmojiResult struct { Emoji []Emoji `json:"emoji"` DisabledEmoji []Emoji `json:"disabled_emoji,omitempty"` + Total int } type Emoji struct { Name string `json:"name"` - IsAlias int64 `json:"is_alias"` - AliasFor string `json:"alias_for"` + IsAlias int64 `json:"is_alias,omitempty"` + AliasFor string `json:"alias_for,omitempty"` URL string `json:"url"` - TeamID string `json:"team_id"` - UserID string `json:"user_id"` - Created int64 `json:"created"` - IsBad bool `json:"is_bad"` - UserDisplayName string `json:"user_display_name"` - AvatarHash string `json:"avatar_hash"` - CanDelete bool `json:"can_delete"` - Synonyms []string `json:"synonyms"` + TeamID string `json:"team_id,omitempty"` + UserID string `json:"user_id,omitempty"` + Created int64 `json:"created,omitempty"` + IsBad bool `json:"is_bad,omitempty"` + UserDisplayName string `json:"user_display_name,omitempty"` + AvatarHash string `json:"avatar_hash,omitempty"` + CanDelete bool `json:"can_delete,omitempty"` + Synonyms []string `json:"synonyms,omitempty"` } type Paging struct { @@ -52,8 +57,51 @@ type adminEmojiListRequest struct { Paging } +func (cl *Client) AdminEmojiList(ctx context.Context) iter.Seq2[EmojiResult, error] { + return func(yield func(EmojiResult, error) bool) { + ctx, task := trace.NewTask(ctx, "edge.AdminEmojiList") + defer task.End() + + var res EmojiResult + req := adminEmojiListRequest{ + BaseRequest: BaseRequest{ + Token: cl.token, + }, + Paging: Paging{ + Page: 1, + Count: 100, + }, + WebClientFields: webclientReason("customize-emoji-new-query"), + } + for { + resp, err := cl.Post(ctx, "emoji.adminList", req) + if err != nil { + yield(res, err) + return + } + var r = emojiResponse{ + EmojiResult: EmojiResult{ + Emoji: make([]Emoji, 0, 100), + }, + } + if err := cl.ParseResponse(&r, resp); err != nil { + yield(res, err) + return + } + r.Total = int(r.Paging.Total) + if !yield(r.EmojiResult, nil) { + return + } + if r.Paging.isLastPage() { + return + } + req.Paging.nextPage() + } + } +} + // AdminEmojiList returns a list of custom emoji for the workspace. -func (cl *Client) AdminEmojiList(ctx context.Context) (EmojiResult, error) { +func (cl *Client) AdminEmojiListFull(ctx context.Context) (EmojiResult, error) { var res EmojiResult req := adminEmojiListRequest{ BaseRequest: BaseRequest{ @@ -71,7 +119,7 @@ func (cl *Client) AdminEmojiList(ctx context.Context) (EmojiResult, error) { if err != nil { return res, err } - var r EmojiResponse + var r emojiResponse if err := cl.ParseResponse(&r, resp); err != nil { return res, err } diff --git a/internal/edge/limits.go b/internal/edge/limits.go index 7c672dae..c5132981 100644 --- a/internal/edge/limits.go +++ b/internal/edge/limits.go @@ -22,5 +22,5 @@ var ( // tier2 = tier{t: 3 * time.Second, b: 3} tier2boost = tier{t: 300 * time.Millisecond, b: 5} tier3 = tier{t: 1200 * time.Millisecond, b: 4} - // tier4 = tier{t: 60 * time.Millisecond, b: 5} + // tier4 = tier{t: 60 * time.Millisecond, b: 5} ) From 82605aee1c0303f6fe0e5ec4ea4ea515648394da Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Sun, 24 Nov 2024 11:29:48 +1000 Subject: [PATCH 7/8] switch renderer to glamour, add emoji long help --- cmd/slackdump/internal/emoji/assets/emoji.md | 61 ++++++++++++++++++++ cmd/slackdump/internal/emoji/emoji.go | 12 ++-- cmd/slackdump/internal/emoji/wizard.go | 11 ++++ cmd/slackdump/internal/golang/base/base.go | 31 +++++----- go.mod | 8 ++- go.sum | 28 +++++++-- 6 files changed, 126 insertions(+), 25 deletions(-) create mode 100644 cmd/slackdump/internal/emoji/assets/emoji.md diff --git a/cmd/slackdump/internal/emoji/assets/emoji.md b/cmd/slackdump/internal/emoji/assets/emoji.md new file mode 100644 index 00000000..a8b801f3 --- /dev/null +++ b/cmd/slackdump/internal/emoji/assets/emoji.md @@ -0,0 +1,61 @@ +# Emoji Command +This command allows you to download all the custom emojis from the Slack +workspace. + +There are two modes of operation: +- **Standard**: download only the names and URLs of the custom emojis; +- **Full**: Download all the custom emojis from the workspace. + +In both modes: +- aliases are skipped, as they just point to the main emoji; +- emoji files and saves in the "emojis" directory within the archive directory + or ZIP file. + + +## Standard Mode +In this mode, the command uses the standard Slack API that returns a mapping +of the custom emoji names to their URLs, including the standard Slack emojis. + +The output is a JSON file with the following structure: +```json +{ + "emoji_name": "emoji_url", + // ... +} +``` + +## Full Mode +In this mode, the command uses Slack Client API to download all information +about the custom emojis. This includes: +- the emoji name; +- the URL of the emoji image; +- the user display name of the user who created the emoji and their ID; +- the date when the emoji was created; +- it's aliases; +- team ID; +- user's avatar hash. + +NOTE: This API endpoint is not documented by Slack, and it's not guaranteed to +be stable. The command uses the undocumented API endpoint to download the +information about the custom emojis. + +It is slower than the standard mode, but slackdump does it's best to do things +in parallel to speed up the process. + +The output is a JSON file with the following structure: + +```json +{ + "emoji_name": { + "name": "emoji_name", + "url": "emoji_url", + "team": "team_id", + "user_id": "user_id", + "created": 1670466722, + "user_display_name": "user_name", + "aliases": ["alias1", "alias2"], + "avatar": "avatar_hash" + }, + // ... +} +``` diff --git a/cmd/slackdump/internal/emoji/emoji.go b/cmd/slackdump/internal/emoji/emoji.go index ad413cc6..9f7d602e 100644 --- a/cmd/slackdump/internal/emoji/emoji.go +++ b/cmd/slackdump/internal/emoji/emoji.go @@ -2,6 +2,7 @@ package emoji import ( "context" + _ "embed" "fmt" "github.com/rusq/fsadapter" @@ -14,12 +15,15 @@ import ( "github.com/rusq/slackdump/v3/internal/edge" ) +//go:embed assets/emoji.md +var emojiMD string + var CmdEmoji = &base.Command{ Run: run, UsageLine: "slackdump emoji [flags]", - Short: "download workspace emojis", - Long: "", // TODO: add long description - FlagMask: cfg.OmitDownloadFlag | cfg.OmitConfigFlag, + Short: "download custom workspace emojis", + Long: emojiMD, // TODO: add long description + FlagMask: cfg.OmitDownloadFlag | cfg.OmitConfigFlag | cfg.OmitChunkCacheFlag | cfg.OmitUserCacheFlag, RequireAuth: true, PrintFlags: true, } @@ -37,7 +41,7 @@ var cmdFlags = options{ func init() { CmdEmoji.Wizard = wizard CmdEmoji.Flag.BoolVar(&cmdFlags.ignoreErrors, "ignore-errors", true, "ignore download errors (skip failed emojis)") - CmdEmoji.Flag.BoolVar(&cmdFlags.fullInfo, "full-info", true, "fetch emojis using Edge API to get full emoji information, including usernames") + CmdEmoji.Flag.BoolVar(&cmdFlags.fullInfo, "full-info", false, "fetch emojis using Edge API to get full emoji information, including usernames") } func run(ctx context.Context, cmd *base.Command, args []string) error { diff --git a/cmd/slackdump/internal/emoji/wizard.go b/cmd/slackdump/internal/emoji/wizard.go index 28c02924..8400cf1f 100644 --- a/cmd/slackdump/internal/emoji/wizard.go +++ b/cmd/slackdump/internal/emoji/wizard.go @@ -21,6 +21,17 @@ func wizard(ctx context.Context, cmd *base.Command, args []string) error { func (o *options) configuration() cfgui.Configuration { return cfgui.Configuration{ + cfgui.ParamGroup{ + Name: "API Options", + Params: []cfgui.Parameter{ + { + Name: "Full Emoji Information", + Value: cfgui.Checkbox(o.fullInfo), + Description: "Uses edge API to fetch full emoji information, including usernames", + Updater: updaters.NewBool(&o.fullInfo), + }, + }, + }, cfgui.ParamGroup{ Name: "Download Options", Params: []cfgui.Parameter{ diff --git a/cmd/slackdump/internal/golang/base/base.go b/cmd/slackdump/internal/golang/base/base.go index 8c56ac9c..ef9089ee 100644 --- a/cmd/slackdump/internal/golang/base/base.go +++ b/cmd/slackdump/internal/golang/base/base.go @@ -17,9 +17,10 @@ import ( "strings" "sync" - "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" + "github.com/charmbracelet/glamour" "golang.org/x/term" - "src.elv.sh/pkg/md" + + "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" ) var CmdName string @@ -170,22 +171,24 @@ func Executable() string { // escape sequences for the terminal output. The width of output is calculated // based on the terminal width. func Render(s string) string { - const ( - defWidth = 80 - ) - width, _, err := term.GetSize(int(os.Stdout.Fd())) + _, _, err := term.GetSize(int(os.Stdout.Fd())) if err != nil { // we're not running in the terminal, output the markdown source. return s } - if width < 40 { - width = defWidth - } + return renderGlam(s) +} - return md.RenderString(s, &md.TTYCodec{Width: width - 2}) +func renderGlam(s string) string { + r, err := glamour.NewTermRenderer(glamour.WithAutoStyle()) + if err != nil { + return s + } + defer r.Close() - // heavy-weight alternative: - // leftIndent := int(float64(width) * 0.075) - // rightIndent := int(float64(width) * 0.02) - // return string(markdown.Render(s, width-rightIndent, leftIndent)) + out, err := r.Render(s) + if err != nil { + return s + } + return out } diff --git a/go.mod b/go.mod index 2d121e54..cc38dc94 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/ProtonMail/go-crypto v1.1.2 github.com/charmbracelet/bubbles v0.20.0 github.com/charmbracelet/bubbletea v1.1.2 + github.com/charmbracelet/glamour v0.8.0 github.com/charmbracelet/huh v0.6.0 github.com/charmbracelet/huh/spinner v0.0.0-20241028115900-20a4d21717a8 github.com/charmbracelet/lipgloss v1.0.0 @@ -41,12 +42,13 @@ require ( golang.org/x/term v0.25.0 golang.org/x/text v0.19.0 golang.org/x/time v0.7.0 - src.elv.sh v0.21.0 ) require ( + github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/catppuccin/go v0.2.0 // indirect github.com/charmbracelet/x/ansi v0.4.2 // indirect github.com/charmbracelet/x/exp/strings v0.0.0-20241101155414-3df16cb7eefd // indirect @@ -54,11 +56,13 @@ require ( github.com/cloudflare/circl v1.3.7 // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/denisbrodbeck/machineid v1.0.1 // indirect + github.com/dlclark/regexp2 v1.11.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/gabriel-vasile/mimetype v1.4.6 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-stack/stack v1.8.1 // indirect + github.com/gorilla/css v1.0.1 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -67,10 +71,12 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect diff --git a/go.sum b/go.sum index a4248588..9e108981 100644 --- a/go.sum +++ b/go.sum @@ -6,18 +6,28 @@ github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 h github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403/go.mod h1:mM6WvakkX2m+NgMiPCfFFjwfH4KzENC07zeGEqq9U7s= github.com/ProtonMail/go-crypto v1.1.2 h1:A7JbD57ThNqh7XjmHE+PXpQ3Dqt3BrSAC0AL0Go3KS0= github.com/ProtonMail/go-crypto v1.1.2/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= +github.com/alecthomas/assert/v2 v2.7.0 h1:QtqSACNS3tF7oasA8CU6A6sXZSBDqnm7RfpLl9bZqbE= +github.com/alecthomas/assert/v2 v2.7.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46aU4V9E= +github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/catppuccin/go v0.2.0 h1:ktBeIrIP42b/8FGiScP9sgrWOss3lw0Z5SktRoithGA= github.com/catppuccin/go v0.2.0/go.mod h1:8IHJuMGaUUjQM82qBrGNBv7LFq6JI3NnQCF6MOlZjpc= github.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQWD9LIutE= github.com/charmbracelet/bubbles v0.20.0/go.mod h1:39slydyswPy+uVOHZ5x/GjwVAFkCsV8IIVy+4MhzwwU= github.com/charmbracelet/bubbletea v1.1.2 h1:naQXF2laRxyLyil/i7fxdpiz1/k06IKquhm4vBfHsIc= github.com/charmbracelet/bubbletea v1.1.2/go.mod h1:9HIU/hBV24qKjlehyj8z1r/tR9TYTQEag+cWZnuXo8E= +github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs= +github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw= github.com/charmbracelet/huh v0.6.0 h1:mZM8VvZGuE0hoDXq6XLxRtgfWyTI3b2jZNKh0xWmax8= github.com/charmbracelet/huh v0.6.0/go.mod h1:GGNKeWCeNzKpEOh/OJD8WBwTQjV3prFAtQPpLv+AVwU= github.com/charmbracelet/huh/spinner v0.0.0-20241028115900-20a4d21717a8 h1:g+Bz64hsMLTf3lAgUqI6Rj1YEAlm/HN39IuhyneCokc= @@ -44,6 +54,8 @@ github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80N github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog= @@ -78,9 +90,13 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -98,8 +114,11 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= @@ -110,6 +129,8 @@ github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54x+RKAA6JiFFHlHDY1UbkxqppT7wYOg= github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -118,6 +139,7 @@ github.com/playwright-community/playwright-go v0.4702.0 h1:3CwNpk4RoA42tyhmlgPDM github.com/playwright-community/playwright-go v0.4702.0/go.mod h1:bpArn5TqNzmP0jroCgw4poSOG9gSeQg490iLqWAaa7w= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -135,12 +157,8 @@ github.com/rusq/rbubbles v0.0.2 h1:U+rkywxtmBw0fdXABTCyND2YUZW9xydsxE12Co0tsFA= github.com/rusq/rbubbles v0.0.2/go.mod h1:wOrwl1AiCCmaL9fLnjKDajOP4IglSC84fH7a74VsnLk= github.com/rusq/secure v0.0.4 h1:svpiZHfHnx89eEDCCFI9OXG1Y8hL9kUWUG6fJbrWUOI= github.com/rusq/secure v0.0.4/go.mod h1:F1QilMKreuFRjov0UY7DZSIXn77/8RqMVGu2zV0RtqY= -github.com/rusq/slack v0.9.6-0.20241104074952-d9b6e02955fa h1:meNaDH2eLwjAqvOxMlgb5+gaLz3Kufm9rVFkALhsCRs= -github.com/rusq/slack v0.9.6-0.20241104074952-d9b6e02955fa/go.mod h1:9O0zQAFN6W47z4KpTQbe6vOHOzBO76vMg1+gthPwaTI= github.com/rusq/slack v0.9.6-0.20241117083852-278084780c45 h1:tsZKbEaziqVowGaQ7zRsrxpy9JDk6CCkihR5PrMk48s= github.com/rusq/slack v0.9.6-0.20241117083852-278084780c45/go.mod h1:9O0zQAFN6W47z4KpTQbe6vOHOzBO76vMg1+gthPwaTI= -github.com/rusq/slack v0.9.6-0.20241122224849-576a79dc22f1 h1:70BrReHUHQ/ERHqGxUgXJrlXKE5jA++bzo+F9Q2b9Pw= -github.com/rusq/slack v0.9.6-0.20241122224849-576a79dc22f1/go.mod h1:9O0zQAFN6W47z4KpTQbe6vOHOzBO76vMg1+gthPwaTI= github.com/rusq/slackauth v0.5.1 h1:l+Gj96kYzHmljMYglRv76kgzuOJr/QbXDDA8JHyN71Q= github.com/rusq/slackauth v0.5.1/go.mod h1:wAtNCbeKH0pnaZnqJjG5RKY3e5BF9F2L/YTzhOjBIb0= github.com/rusq/tagops v0.0.2 h1:LkWsmpYSH5Q5IX3pv0Qm5PEKOtfjKqrwbJ3c19C1pvM= @@ -242,5 +260,3 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -src.elv.sh v0.21.0 h1:DXtdzaaGoc+VctRnDmeS8Xv1bknbRWTRMDZf2DI3sGI= -src.elv.sh v0.21.0/go.mod h1:SCiBbiD5+gVCBPfY17ixCBrce+7jAMFHRz2eh90aCig= From d93226a7c9a0c09468da808460588031fe2aa187 Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Sun, 24 Nov 2024 13:06:31 +1000 Subject: [PATCH 8/8] emojis, progress etc. --- cmd/slackdump/internal/emoji/assets/emoji.md | 3 + cmd/slackdump/internal/emoji/emoji.go | 52 ++++++++++--- .../internal/emoji/emojidl/emoedge.go | 34 +++++---- cmd/slackdump/internal/emoji/emojidl/emoji.go | 76 +++++++++---------- .../internal/emoji/emojidl/emoji_test.go | 30 ++++---- cmd/slackdump/internal/emoji/wizard.go | 4 +- internal/edge/bookmarks.go | 2 +- internal/edge/client.go | 4 +- internal/edge/client_boot.go | 2 +- internal/edge/conversations.go | 4 +- internal/edge/dms.go | 2 +- internal/edge/edge.go | 4 +- internal/edge/emoji.go | 13 +++- internal/edge/files.go | 2 +- internal/edge/pins.go | 2 +- internal/edge/search.go | 2 +- internal/edge/userlist.go | 6 +- 17 files changed, 143 insertions(+), 99 deletions(-) diff --git a/cmd/slackdump/internal/emoji/assets/emoji.md b/cmd/slackdump/internal/emoji/assets/emoji.md index a8b801f3..94625023 100644 --- a/cmd/slackdump/internal/emoji/assets/emoji.md +++ b/cmd/slackdump/internal/emoji/assets/emoji.md @@ -6,6 +6,9 @@ There are two modes of operation: - **Standard**: download only the names and URLs of the custom emojis; - **Full**: Download all the custom emojis from the workspace. +Full mode is approx. 2.3 times slower than the standard mode, but it provides +more information about the custom emojis. + In both modes: - aliases are skipped, as they just point to the main emoji; - emoji files and saves in the "emojis" directory within the archive directory diff --git a/cmd/slackdump/internal/emoji/emoji.go b/cmd/slackdump/internal/emoji/emoji.go index 9f7d602e..80686adb 100644 --- a/cmd/slackdump/internal/emoji/emoji.go +++ b/cmd/slackdump/internal/emoji/emoji.go @@ -4,8 +4,14 @@ import ( "context" _ "embed" "fmt" + "io" + "log/slog" + "sync" + "time" "github.com/rusq/fsadapter" + "github.com/schollz/progressbar/v3" + "github.com/rusq/slackdump/v3" "github.com/rusq/slackdump/v3/auth" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/bootstrap" @@ -30,7 +36,7 @@ var CmdEmoji = &base.Command{ type options struct { ignoreErrors bool - fullInfo bool + full bool } // emoji specific flags @@ -41,7 +47,7 @@ var cmdFlags = options{ func init() { CmdEmoji.Wizard = wizard CmdEmoji.Flag.BoolVar(&cmdFlags.ignoreErrors, "ignore-errors", true, "ignore download errors (skip failed emojis)") - CmdEmoji.Flag.BoolVar(&cmdFlags.fullInfo, "full-info", false, "fetch emojis using Edge API to get full emoji information, including usernames") + CmdEmoji.Flag.BoolVar(&cmdFlags.full, "full", false, "fetch emojis using Edge API to get full emoji information, including usernames") } func run(ctx context.Context, cmd *base.Command, args []string) error { @@ -56,24 +62,52 @@ func run(ctx context.Context, cmd *base.Command, args []string) error { base.SetExitStatus(base.SApplicationError) return err } - if cmdFlags.fullInfo { - return runEdge(ctx, fsa, prov) + + start := time.Now() + r, cl := statusReporter() + defer cl.Close() + if cmdFlags.full { + err = runEdge(ctx, fsa, prov, r) } else { - return runLegacy(ctx, fsa) + err = runLegacy(ctx, fsa, r) + } + cl.Close() + if err != nil { + base.SetExitStatus(base.SApplicationError) + return err } + + slog.InfoContext(ctx, "Emojis downloaded", "dir", cfg.Output, "took", time.Since(start).String()) + return nil +} + +func statusReporter() (emojidl.StatusFunc, io.Closer) { + pb := progressbar.NewOptions(0, + progressbar.OptionSetDescription("Downloading emojis"), + progressbar.OptionClearOnFinish(), + progressbar.OptionShowCount(), + ) + var once sync.Once + return func(name string, total, count int) { + once.Do(func() { + pb.ChangeMax(total) + }) + pb.Add(1) + }, pb + } -func runLegacy(ctx context.Context, fsa fsadapter.FS) error { +func runLegacy(ctx context.Context, fsa fsadapter.FS, cb emojidl.StatusFunc) error { sess, err := bootstrap.SlackdumpSession(ctx, slackdump.WithFilesystem(fsa)) if err != nil { base.SetExitStatus(base.SApplicationError) return err } - return emojidl.DlFS(ctx, sess, fsa, cmdFlags.ignoreErrors) + return emojidl.DlFS(ctx, sess, fsa, cmdFlags.ignoreErrors, cb) } -func runEdge(ctx context.Context, fsa fsadapter.FS, prov auth.Provider) error { +func runEdge(ctx context.Context, fsa fsadapter.FS, prov auth.Provider, cb emojidl.StatusFunc) error { sess, err := edge.New(ctx, prov) if err != nil { base.SetExitStatus(base.SApplicationError) @@ -81,7 +115,7 @@ func runEdge(ctx context.Context, fsa fsadapter.FS, prov auth.Provider) error { } defer sess.Close() - if err := emojidl.DlEdgeFS(ctx, sess, fsa, cmdFlags.ignoreErrors); err != nil { + if err := emojidl.DlEdgeFS(ctx, sess, fsa, cmdFlags.ignoreErrors, cb); err != nil { base.SetExitStatus(base.SApplicationError) return fmt.Errorf("application error: %s", err) } diff --git a/cmd/slackdump/internal/emoji/emojidl/emoedge.go b/cmd/slackdump/internal/emoji/emojidl/emoedge.go index f31cd335..d07fd46d 100644 --- a/cmd/slackdump/internal/emoji/emojidl/emoedge.go +++ b/cmd/slackdump/internal/emoji/emojidl/emoedge.go @@ -33,16 +33,22 @@ type EdgeEmojiLister interface { AdminEmojiList(ctx context.Context) iter.Seq2[edge.EmojiResult, error] } +type StatusFunc func(name string, total, count int) + // DlEdgeFS downloads the emojis and saves them to the fsa. It spawns numWorker // goroutines for getting the files. It will call fetchFn for each emoji. -func DlEdgeFS(ctx context.Context, sess EdgeEmojiLister, fsa fsadapter.FS, failFast bool) error { - lg := cfg.Log.With("in", "fetch", "dir", emojiDir, "numWorkers", numWorkers, "failFast", failFast) +func DlEdgeFS(ctx context.Context, sess EdgeEmojiLister, fsa fsadapter.FS, failFast bool, cb StatusFunc) error { + lg := cfg.Log + lg.DebugContext(ctx, "startup params", "dir", emojiDir, "numWorkers", numWorkers, "failFast", failFast) + if cb == nil { + cb = func(name string, total, count int) {} + } var ( emojiC = make(chan edge.Emoji) totalC = make(chan int) genErrC = make(chan error) - resultC = make(chan edgeResult) + resultC = make(chan result) ) // Async download pipeline. @@ -51,7 +57,6 @@ func DlEdgeFS(ctx context.Context, sess EdgeEmojiLister, fsa fsadapter.FS, failF go func() { var once sync.Once defer close(totalC) - defer close(emojiC) for chunk, err := range sess.AdminEmojiList(ctx) { @@ -76,7 +81,7 @@ func DlEdgeFS(ctx context.Context, sess EdgeEmojiLister, fsa fsadapter.FS, failF for i := 0; i < numWorkers; i++ { wg.Add(1) go func() { - worker2(ctx, fsa, emojiC, resultC) + worker(ctx, fsa, emojiC, resultC) wg.Done() }() } @@ -90,14 +95,14 @@ func DlEdgeFS(ctx context.Context, sess EdgeEmojiLister, fsa fsadapter.FS, failF // may have occurred. var ( count = 0 - total = <-totalC + total = <-totalC // if there's a generator error, this will receive 0. ) var emojis = make(map[string]edge.Emoji, total) LOOP: for { select { - case genErr := <-genErrC: + // generator error. if genErr != nil { return fmt.Errorf("failed to get emoji list: %w", genErr) } @@ -105,6 +110,7 @@ LOOP: if !more { break LOOP } + lg := lg.With("name", res.emoji.Name) if res.err != nil { if errors.Is(res.err, context.Canceled) { return res.err @@ -112,11 +118,11 @@ LOOP: if failFast { return fmt.Errorf("failed: %q: %w", res.emoji.Name, res.err) } - lg.WarnContext(ctx, "failed", "name", res.emoji.Name, "error", res.err) + lg.WarnContext(ctx, "failed", "error", res.err) } emojis[res.emoji.Name] = res.emoji // to resemble the legacy code. count++ - lg.InfoContext(ctx, "downloaded", "count", count, "total", total, "name", res.emoji.Name) + cb(res.emoji.Name, total, count) } } out, err := fsa.Create("index.json") @@ -131,7 +137,7 @@ LOOP: return nil } -type edgeResult struct { +type result struct { emoji edge.Emoji skipped bool err error @@ -140,22 +146,22 @@ type edgeResult struct { // worker is the function that runs in a separate goroutine and downloads emoji // received from emojiC. The result of the operation is sent to resultC channel. // fn is called for each received emoji. -func worker2(ctx context.Context, fsa fsadapter.FS, emojiC <-chan edge.Emoji, resultC chan<- edgeResult) { +func worker(ctx context.Context, fsa fsadapter.FS, emojiC <-chan edge.Emoji, resultC chan<- result) { for { select { case <-ctx.Done(): - resultC <- edgeResult{err: ctx.Err()} + resultC <- result{err: ctx.Err()} return case em, more := <-emojiC: if !more { return } if em.IsAlias != 0 { - resultC <- edgeResult{emoji: em, skipped: true} + resultC <- result{emoji: em, skipped: true} break } err := fetchFn(ctx, fsa, emojiDir, em.Name, em.URL) - resultC <- edgeResult{emoji: em, err: err} + resultC <- result{emoji: em, err: err} } } } diff --git a/cmd/slackdump/internal/emoji/emojidl/emoji.go b/cmd/slackdump/internal/emoji/emojidl/emoji.go index 0c085634..40a09add 100644 --- a/cmd/slackdump/internal/emoji/emojidl/emoji.go +++ b/cmd/slackdump/internal/emoji/emojidl/emoji.go @@ -28,6 +28,7 @@ import ( "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" + "github.com/rusq/slackdump/v3/internal/edge" ) const ( @@ -43,7 +44,7 @@ type EmojiDumper interface { } // DlFS downloads all emojis from the workspace and saves them to the fsa. -func DlFS(ctx context.Context, sess EmojiDumper, fsa fsadapter.FS, failFast bool) error { +func DlFS(ctx context.Context, sess EmojiDumper, fsa fsadapter.FS, failFast bool, cb StatusFunc) error { emojis, err := sess.DumpEmojis(ctx) if err != nil { return fmt.Errorf("error during emoji dump: %w", err) @@ -57,29 +58,54 @@ func DlFS(ctx context.Context, sess EmojiDumper, fsa fsadapter.FS, failFast bool return fmt.Errorf("failed writing emoji index: %w", err) } - return fetch(ctx, fsa, emojis, failFast) + return fetch(ctx, fsa, emojis, failFast, cb) +} + +func ift[T any](cond bool, t, f T) T { + if cond { + return t + } + return f } // fetch downloads the emojis and saves them to the fsa. It spawns numWorker // goroutines for getting the files. It will call fetchFn for each emoji. -func fetch(ctx context.Context, fsa fsadapter.FS, emojis map[string]string, failFast bool) error { - lg := cfg.Log.With("in", "fetch", "dir", emojiDir, "numWorkers", numWorkers, "failFast", failFast) +func fetch(ctx context.Context, fsa fsadapter.FS, emojis map[string]string, failFast bool, cb StatusFunc) error { + lg := cfg.Log + lg.DebugContext(ctx, "startup params", "dir", emojiDir, "numWorkers", numWorkers, "failFast", failFast) + + if cb == nil { + cb = func(name string, total, count int) {} + } var ( - emojiC = make(chan emoji) + emojiC = make(chan edge.Emoji) resultC = make(chan result) ) + const ( + aliasPrefix = "alias:" + aliasLen = len(aliasPrefix) + ) + // Async download pipeline. // 1. generator, send emojis into the emojiC channel. go func() { defer close(emojiC) + for name, uri := range emojis { + isAlias := strings.HasPrefix(uri, aliasPrefix) + emoji := edge.Emoji{ + Name: name, + URL: uri, + IsAlias: ift(isAlias, 1, 0), + AliasFor: ift(isAlias, uri[aliasLen:], ""), + } select { case <-ctx.Done(): return - case emojiC <- emoji{name, uri}: + case emojiC <- emoji: } } }() @@ -107,53 +133,23 @@ func fetch(ctx context.Context, fsa fsadapter.FS, emojis map[string]string, fail ) lg = lg.With("total", total) for res := range resultC { + lg := lg.With("name", res.emoji.Name) if res.err != nil { if errors.Is(res.err, context.Canceled) { return res.err } if failFast { - return fmt.Errorf("failed: %q: %w", res.name, res.err) + return fmt.Errorf("failed: %q: %w", res.emoji.Name, res.err) } - lg.WarnContext(ctx, "failed", "name", res.name, "error", res.err) + lg.WarnContext(ctx, "failed", "error", res.err) } count++ - lg.InfoContext(ctx, "downloaded", "count", count, "name", res.name) + cb(res.emoji.Name, total, count) } return nil } -// emoji is an array containing name and url of the emoji. -type emoji [2]string - -type result struct { - name string - err error -} - -// worker is the function that runs in a separate goroutine and downloads emoji -// received from emojiC. The result of the operation is sent to resultC channel. -// fn is called for each received emoji. -func worker(ctx context.Context, fsa fsadapter.FS, emojiC <-chan emoji, resultC chan<- result) { - for { - select { - case <-ctx.Done(): - resultC <- result{err: ctx.Err()} - return - case emoji, more := <-emojiC: - if !more { - return - } - if strings.HasPrefix(emoji[1], "alias:") { - resultC <- result{name: emoji[0] + "(alias, skipped)"} - break - } - err := fetchFn(ctx, fsa, emojiDir, emoji[0], emoji[1]) - resultC <- result{name: emoji[0], err: err} - } - } -} - // fetchEmoji downloads one emoji file from uri into the filename dir/name.png // within the filesystem adapter fsa. func fetchEmoji(ctx context.Context, fsa fsadapter.FS, dir string, name, uri string) error { diff --git a/cmd/slackdump/internal/emoji/emojidl/emoji_test.go b/cmd/slackdump/internal/emoji/emojidl/emoji_test.go index 94a11e66..7468a34b 100644 --- a/cmd/slackdump/internal/emoji/emojidl/emoji_test.go +++ b/cmd/slackdump/internal/emoji/emojidl/emoji_test.go @@ -14,9 +14,11 @@ import ( "sync" "testing" + "github.com/rusq/fsadapter" + "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" - "github.com/rusq/fsadapter" + "github.com/rusq/slackdump/v3/internal/edge" ) type fetchFunc func(ctx context.Context, fsa fsadapter.FS, dir string, name string, uri string) error @@ -110,8 +112,8 @@ func Test_fetchEmoji(t *testing.T) { } } -func testEmojiC(emojis []emoji, wantClosed bool) <-chan emoji { - ch := make(chan emoji) +func testEmojiC(emojis []edge.Emoji, wantClosed bool) <-chan edge.Emoji { + ch := make(chan edge.Emoji) go func() { for _, e := range emojis { ch <- e @@ -126,7 +128,7 @@ func testEmojiC(emojis []emoji, wantClosed bool) <-chan emoji { func Test_worker(t *testing.T) { type args struct { ctx context.Context - emojiC <-chan emoji + emojiC <-chan edge.Emoji } tests := []struct { name string @@ -138,39 +140,39 @@ func Test_worker(t *testing.T) { "all ok", args{ ctx: context.Background(), - emojiC: testEmojiC([]emoji{{"test", "passed"}}, true), + emojiC: testEmojiC([]edge.Emoji{{Name: "test", URL: "passed"}}, true), }, func(ctx context.Context, fsa fsadapter.FS, dir string, name string, uri string) error { return nil }, []result{ - {name: "test"}, + {emoji: edge.Emoji{Name: "test", URL: "passed"}}, }, }, { "cancelled context", args{ ctx: func() context.Context { ctx, cancel := context.WithCancel(context.Background()); cancel(); return ctx }(), - emojiC: testEmojiC([]emoji{}, false), + emojiC: testEmojiC([]edge.Emoji{}, false), }, func(ctx context.Context, fsa fsadapter.FS, dir string, name string, uri string) error { return nil }, []result{ - {name: "", err: context.Canceled}, + {emoji: edge.Emoji{Name: ""}, err: context.Canceled}, }, }, { "fetch error", args{ ctx: context.Background(), - emojiC: testEmojiC([]emoji{{"test", "passed"}}, true), + emojiC: testEmojiC([]edge.Emoji{{Name: "test", URL: "passed"}}, true), }, func(ctx context.Context, fsa fsadapter.FS, dir string, name string, uri string) error { return io.EOF }, []result{ - {name: "test", err: io.EOF}, + {emoji: edge.Emoji{Name: "test", URL: "passed"}, err: io.EOF}, }, }, } @@ -195,9 +197,7 @@ func Test_worker(t *testing.T) { for r := range resultC { results = append(results, r) } - if !reflect.DeepEqual(results, tt.wantResult) { - t.Errorf("results mismatch:\n\twant=%v\n\tgot =%v", tt.wantResult, results) - } + assert.Equal(t, tt.wantResult, results) }) } } @@ -216,7 +216,7 @@ func Test_fetch(t *testing.T) { return nil }) - err := fetch(context.Background(), fsa, emojis, true) + err := fetch(context.Background(), fsa, emojis, true, nil) if err != nil { t.Errorf("unexpected error: %s", err) } @@ -334,7 +334,7 @@ func Test_download(t *testing.T) { if err != nil { t.Fatal(err) } - if err := DlFS(tt.args.ctx, sess, fs, tt.args.failFast); (err != nil) != tt.wantErr { + if err := DlFS(tt.args.ctx, sess, fs, tt.args.failFast, nil); (err != nil) != tt.wantErr { t.Errorf("download() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/cmd/slackdump/internal/emoji/wizard.go b/cmd/slackdump/internal/emoji/wizard.go index 8400cf1f..523deb48 100644 --- a/cmd/slackdump/internal/emoji/wizard.go +++ b/cmd/slackdump/internal/emoji/wizard.go @@ -26,9 +26,9 @@ func (o *options) configuration() cfgui.Configuration { Params: []cfgui.Parameter{ { Name: "Full Emoji Information", - Value: cfgui.Checkbox(o.fullInfo), + Value: cfgui.Checkbox(o.full), Description: "Uses edge API to fetch full emoji information, including usernames", - Updater: updaters.NewBool(&o.fullInfo), + Updater: updaters.NewBool(&o.full), }, }, }, diff --git a/internal/edge/bookmarks.go b/internal/edge/bookmarks.go index 7df9af31..232d69e5 100644 --- a/internal/edge/bookmarks.go +++ b/internal/edge/bookmarks.go @@ -16,7 +16,7 @@ type bookmarksListForm struct { } type bookmarksListResponse struct { - BaseResponse + baseResponse Bookmarks []Bookmark `json:"bookmarks"` } diff --git a/internal/edge/client.go b/internal/edge/client.go index 4e37118b..db84ea83 100644 --- a/internal/edge/client.go +++ b/internal/edge/client.go @@ -19,7 +19,7 @@ type clientCountsForm struct { } type ClientCountsResponse struct { - BaseResponse + baseResponse Channels []ChannelSnapshot `json:"channels,omitempty"` MPIMs []ChannelSnapshot `json:"mpims,omitempty"` IMs []ChannelSnapshot `json:"ims,omitempty"` @@ -68,7 +68,7 @@ type clientDMsForm struct { } type clientDMsResponse struct { - BaseResponse + baseResponse IMs []ClientDM `json:"ims,omitempty"` MPIMs []ClientDM `json:"mpims,omitempty"` //TODO } diff --git a/internal/edge/client_boot.go b/internal/edge/client_boot.go index 4e885e92..399a1c94 100644 --- a/internal/edge/client_boot.go +++ b/internal/edge/client_boot.go @@ -57,7 +57,7 @@ func (r *ClientUserBootResponse) Marshal() ([]byte, error) { // "client.userBoot" type ClientUserBootResponse struct { - BaseResponse + baseResponse Self Self `json:"self"` Team Team `json:"team"` IMs []IM `json:"ims"` diff --git a/internal/edge/conversations.go b/internal/edge/conversations.go index e82734fb..7bdcc306 100644 --- a/internal/edge/conversations.go +++ b/internal/edge/conversations.go @@ -18,7 +18,7 @@ type conversationsGenericInfoForm struct { } type conversationsGenericInfoResponse struct { - BaseResponse + baseResponse Channels []slack.Channel `json:"channels"` UnchangedChannelIDs []string `json:"unchanged_channel_ids"` } @@ -105,7 +105,7 @@ func (cl *Client) ConversationsView(ctx context.Context, channelID string) (Conv return ConversationsViewResponse{}, err } var r = struct { - BaseResponse + baseResponse ConversationsViewResponse }{} if err := cl.ParseResponse(&r, resp); err != nil { diff --git a/internal/edge/dms.go b/internal/edge/dms.go index 784dec06..8b29c2e4 100644 --- a/internal/edge/dms.go +++ b/internal/edge/dms.go @@ -16,7 +16,7 @@ type imListForm struct { } type imListResponse struct { - BaseResponse + baseResponse IMs []IM `json:"ims,omitempty"` } diff --git a/internal/edge/edge.go b/internal/edge/edge.go index 74002a6d..2ef86667 100644 --- a/internal/edge/edge.go +++ b/internal/edge/edge.go @@ -140,13 +140,13 @@ type BaseRequest struct { Token string `json:"token"` } -type BaseResponse struct { +type baseResponse struct { Ok bool `json:"ok"` Error string `json:"error,omitempty"` ResponseMetadata ResponseMetadata `json:"response_metadata,omitempty"` } -func (r BaseResponse) validate(ep string) error { +func (r baseResponse) validate(ep string) error { if !r.Ok { return &APIError{Err: r.Error, Metadata: r.ResponseMetadata, Endpoint: ep} } diff --git a/internal/edge/emoji.go b/internal/edge/emoji.go index da4b92c5..38582e96 100644 --- a/internal/edge/emoji.go +++ b/internal/edge/emoji.go @@ -7,21 +7,26 @@ import ( ) type emojiResponse struct { - BaseResponse + baseResponse EmojiResult CustomEmojiTotalCount int64 `json:"custom_emoji_total_count"` Paging Paging `json:"paging"` } +// EmojiResult is a subset of the response from the emoji.adminList API. type EmojiResult struct { - Emoji []Emoji `json:"emoji"` + // Emoji is the list of custom emoji. + Emoji []Emoji `json:"emoji"` + // DisabledEmoji is the list of disabled custom emoji (supposedly). DisabledEmoji []Emoji `json:"disabled_emoji,omitempty"` - Total int + // Total is the total number of custom emoji. + Total int } +// Emoji represents a custom emoji as read by the Client API. type Emoji struct { Name string `json:"name"` - IsAlias int64 `json:"is_alias,omitempty"` + IsAlias int `json:"is_alias,omitempty"` AliasFor string `json:"alias_for,omitempty"` URL string `json:"url"` TeamID string `json:"team_id,omitempty"` diff --git a/internal/edge/files.go b/internal/edge/files.go index b6c9c85f..b45fc95c 100644 --- a/internal/edge/files.go +++ b/internal/edge/files.go @@ -17,7 +17,7 @@ type filesListForm struct { } type filesListResponse struct { - BaseResponse + baseResponse Files []slack.File `json:"files"` Pagination } diff --git a/internal/edge/pins.go b/internal/edge/pins.go index d6e2f19e..f6df153f 100644 --- a/internal/edge/pins.go +++ b/internal/edge/pins.go @@ -14,7 +14,7 @@ type pinsListRequest struct { } type pinsListResponse struct { - BaseResponse + baseResponse Items []PinnedItem `json:"items"` } diff --git a/internal/edge/search.go b/internal/edge/search.go index 4d944f74..04010a16 100644 --- a/internal/edge/search.go +++ b/internal/edge/search.go @@ -20,7 +20,7 @@ var ( ) type SearchResponse[T any] struct { - BaseResponse + baseResponse Module string `json:"module"` Query string `json:"query"` Filters json.RawMessage `json:"filters"` diff --git a/internal/edge/userlist.go b/internal/edge/userlist.go index ce3a0ccd..07bf8291 100644 --- a/internal/edge/userlist.go +++ b/internal/edge/userlist.go @@ -23,7 +23,7 @@ type UsersListRequest struct { type UsersListResponse struct { Results []User `json:"results"` NextMarker string `json:"next_marker"` // pagination, marker value which must be used in the next request, if not empty. - BaseResponse + baseResponse } type User struct { @@ -86,7 +86,7 @@ type UserInfoResponse struct { FailedIDS []string `json:"failed_ids"` PendingIDS []string `json:"pending_ids"` CanInteract map[string]bool `json:"can_interact"` - BaseResponse + baseResponse } type UserInfo struct { @@ -170,7 +170,7 @@ type ChannelsMembershipRequest struct { type ChannelsMembershipResponse struct { Channel string `json:"channel"` NonMembers []string `json:"non_members"` - BaseResponse + baseResponse } // ChannelsMembership calls channels/membership endpoint.