Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support use server time for file creation #359

Merged
merged 5 commits into from
Jan 30, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ jobs:

go build -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -buildmode=c-shared -o ui/flutter/windows/libgopeed.dll github.com/GopeedLab/gopeed/bind/desktop
cd ui/flutter
$TAG = "v$env:VERSION"
flutter build windows
$mingw = "C:\Program Files\Git\mingw64\bin"
$system = "C:\Windows\System32"
Expand All @@ -65,7 +66,7 @@ jobs:
cp $system\vcruntime140_1.dll $release

New-Item -Path build\windows\Output -ItemType Directory
Compress-Archive -Path "$release*" -DestinationPath "build\windows\Output\Gopeed-$env:TAG-windows-amd64-portable.zip"
Compress-Archive -Path "$release*" -DestinationPath "build\windows\Output\Gopeed-$TAG-windows-amd64-portable.zip"

cd build/windows
echo @"
Expand Down Expand Up @@ -124,7 +125,6 @@ jobs:
Filename: "{app}\{#MyAppExeName}"; Description: "{cm:LaunchProgram,{#StringChange(MyAppName, '&', '&&')}}"; Flags: nowait postinstall skipifsilent
"@ > setup.iss
iscc.exe setup.iss
$TAG = "v$env:VERSION"
mv "Output\gopeed.exe" "Output\Gopeed-$TAG-windows-amd64.exe"
Compress-Archive -Path "Output\Gopeed-$TAG-windows-amd64.exe" -DestinationPath "Output\Gopeed-$TAG-windows-amd64.zip"
Remove-Item -Path "Output\Gopeed-$TAG-windows-amd64.exe"
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ require (
github.com/stretchr/testify v1.8.4 // indirect
github.com/tidwall/btree v1.7.0 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad // indirect
go.opentelemetry.io/otel v1.22.0 // indirect
go.opentelemetry.io/otel/metric v1.22.0 // indirect
go.opentelemetry.io/otel/trace v1.22.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,8 @@ github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPyS
github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad h1:QtuWk6dsrNXc/ugwCBEN0jG52q/4tXRzMvZm4fH6T9g=
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad/go.mod h1:Jj8p9bgKGTPQ+M8CdUMS9p7Mmdoxa3OAcAjJQBu0CcI=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
Expand Down
5 changes: 3 additions & 2 deletions internal/protocol/http/config.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package http

type config struct {
UserAgent string `json:"userAgent"`
Connections int `json:"connections"`
UserAgent string `json:"userAgent"`
Connections int `json:"connections"`
UseServerCtime bool `json:"useServerCtime"`
}
18 changes: 16 additions & 2 deletions internal/protocol/http/fetcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
"github.com/xiaoqidun/setft"
"golang.org/x/sync/errgroup"
"io"
"mime"
Expand Down Expand Up @@ -131,8 +132,17 @@ func (f *Fetcher) Resolve(req *base.Request) error {
} else {
return NewRequestError(httpResp.StatusCode, httpResp.Status)
}
// Parse last modified time
var lastModifiedTime *time.Time
lastModified := httpResp.Header.Get(base.HttpHeaderLastModified)
if lastModified != "" {
// ignore parse error
t, _ := time.Parse(time.RFC1123, lastModified)
lastModifiedTime = &t
}
file := &base.FileInfo{
Size: res.Size,
Size: res.Size,
Ctime: lastModifiedTime,
}
contentDisposition := httpResp.Header.Get(base.HttpHeaderContentDisposition)
if contentDisposition != "" {
Expand Down Expand Up @@ -258,6 +268,10 @@ func (f *Fetcher) fetch() {
return
}
f.file.Close()
// Update file last modified time
if f.config.UseServerCtime && f.meta.Res.Files[0].Ctime != nil {
setft.SetFileTime(f.file.Name(), time.Now(), *f.meta.Res.Files[0].Ctime, *f.meta.Res.Files[0].Ctime)
}
f.doneCh <- err
}()
}
Expand All @@ -276,7 +290,7 @@ func (f *Fetcher) fetchChunk(index int, ctx context.Context) (err error) {
maxRetries = 3
)
// retry until all remain chunks failed
for true {
for {
// if chunk is completed, return
if f.meta.Res.Range && chunk.Downloaded >= chunk.End-chunk.Begin+1 {
return
Expand Down
38 changes: 29 additions & 9 deletions internal/protocol/http/fetcher_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"github.com/GopeedLab/gopeed/pkg/protocol/http"
"github.com/GopeedLab/gopeed/pkg/util"
"net"
"reflect"
"testing"
"time"
)
Expand Down Expand Up @@ -69,7 +68,7 @@ func testResolve(startTestServer func() net.Listener, path string, want *base.Re
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(want, fetcher.meta.Res) {
if !test.AssertResourceEqual(want, fetcher.meta.Res) {
t.Errorf("Resolve() got = %v, want %v", fetcher.meta.Res, want)
}
}
Expand Down Expand Up @@ -151,10 +150,34 @@ func TestFetcher_DownloadWithProxy(t *testing.T) {
downloadWithProxy(httpListener, proxyListener, t)
}

func TestFetcher_Config(t *testing.T) {
func TestFetcher_ConfigConnections(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
fetcher := doDownloadReady(buildConfigFetcher(), listener, 0, t)
fetcher := doDownloadReady(buildConfigFetcher(config{
Connections: 16,
}), listener, 0, t)
err := fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}

func TestFetcher_ConfigUseServerCtime(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
fetcher := doDownloadReady(buildConfigFetcher(config{
Connections: 16,
UseServerCtime: true,
}), listener, 0, t)
err := fetcher.Start()
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -338,14 +361,11 @@ func buildFetcher() *Fetcher {
return fetcher.(*Fetcher)
}

func buildConfigFetcher() fetcher.Fetcher {
func buildConfigFetcher(cfg config) fetcher.Fetcher {
fetcher := new(FetcherBuilder).Build()
newController := controller.NewController()
mockCfg := config{
Connections: 16,
}
newController.GetConfig = func(v any) bool {
if err := json.Unmarshal([]byte(test.ToJson(mockCfg)), v); err != nil {
if err := json.Unmarshal([]byte(test.ToJson(cfg)), v); err != nil {
return false
}
return true
Expand Down
10 changes: 10 additions & 0 deletions internal/test/httptest.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/armon/go-socks5"
"io"
"math/rand"
"net"
"net/http"
"os"
"reflect"
"strconv"
"strings"
"sync/atomic"
Expand Down Expand Up @@ -337,3 +339,11 @@ func StartSocks5Server(usr, pwd string) net.Listener {
go server.Serve(listener)
return listener
}

func AssertResourceEqual(want, got *base.Resource) bool {
// Ignore ctime
if got != nil && len(got.Files) > 0 {
got.Files[0].Ctime = nil
}
return reflect.DeepEqual(want, got)
}
3 changes: 2 additions & 1 deletion pkg/base/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ const (
HttpHeaderContentLength = "Content-Length"
HttpHeaderContentRange = "Content-Range"
HttpHeaderContentDisposition = "Content-Disposition"
HttpHeaderUserAgent = "Usr-Agent"
HttpHeaderUserAgent = "User-Agent"
HttpHeaderLastModified = "Last-Modified"

HttpHeaderBytes = "bytes"
HttpHeaderRangeFormat = "bytes=%d-%d"
Expand Down
8 changes: 5 additions & 3 deletions pkg/base/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"github.com/GopeedLab/gopeed/pkg/util"
"golang.org/x/exp/slices"
"time"
)

// Request download request
Expand Down Expand Up @@ -59,9 +60,10 @@ func (r *Resource) CalcSize(selectFiles []int) {
}

type FileInfo struct {
Name string `json:"name"`
Path string `json:"path"`
Size int64 `json:"size"`
Name string `json:"name"`
Path string `json:"path"`
Size int64 `json:"size"`
Ctime *time.Time `json:"ctime"`

Req *Request `json:"req"`
}
Expand Down
5 changes: 5 additions & 0 deletions pkg/download/downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,11 @@ func (d *Downloader) Setup() error {
}
// init default config
d.cfg.DownloaderStoreConfig.Init()
for _, fb := range d.fetcherBuilders {
f := fb.Build()
d.setupFetcher(f)

}
// load tasks from storage
var tasks []*Task
if err = d.storage.List(bucketTask, &tasks); err != nil {
Expand Down
5 changes: 2 additions & 3 deletions pkg/download/downloader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"github.com/GopeedLab/gopeed/internal/test"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/protocol/http"
"reflect"
"sync"
"testing"
"time"
Expand Down Expand Up @@ -37,7 +36,7 @@ func TestDownloader_Resolve(t *testing.T) {
},
},
}
if !reflect.DeepEqual(want, rr.Res) {
if !test.AssertResourceEqual(want, rr.Res) {
t.Errorf("Resolve() got = %v, want %v", rr.Res, want)
}
}
Expand Down Expand Up @@ -155,7 +154,7 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig f
},
},
}
if !reflect.DeepEqual(want, rr.Res) {
if !test.AssertResourceEqual(want, rr.Res) {
t.Errorf("Resolve() got = %v, want %v", rr.Res, want)
}
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/rest/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ var (
func TestResolve(t *testing.T) {
doTest(func() {
resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", taskReq)
if !test.JsonEqual(taskRes, resp.Res) {
if !test.AssertResourceEqual(taskRes, resp.Res) {
t.Errorf("Resolve() got = %v, want %v", test.ToJson(resp.Res), test.ToJson(taskRes))
}
})
Expand Down
57 changes: 39 additions & 18 deletions ui/flutter/lib/api/model/downloader_config.dart
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,16 @@ part 'downloader_config.g.dart';

@JsonSerializable(explicitToJson: true)
class DownloaderConfig {
String downloadDir = '';
int maxRunning = 0;
String downloadDir;
int maxRunning;
ProtocolConfig protocolConfig = ProtocolConfig();
ExtraConfig extra = ExtraConfig();
ProxyConfig proxy = ProxyConfig();

DownloaderConfig();
DownloaderConfig({
this.downloadDir = '',
this.maxRunning = 0,
});

factory DownloaderConfig.fromJson(Map<String, dynamic> json) =>
_$DownloaderConfigFromJson(json);
Expand All @@ -31,11 +34,16 @@ class ProtocolConfig {

@JsonSerializable()
class HttpConfig {
String userAgent =
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36';
int connections = 0;
String userAgent;
int connections;
bool useServerCtime;

HttpConfig();
HttpConfig({
this.userAgent =
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
this.connections = 16,
this.useServerCtime = false,
});

factory HttpConfig.fromJson(Map<String, dynamic> json) =>
_$HttpConfigFromJson(json);
Expand Down Expand Up @@ -69,13 +77,19 @@ class ExtraConfig {

@JsonSerializable()
class ProxyConfig {
bool enable = false;
String scheme = '';
String host = '';
String usr = '';
String pwd = '';

ProxyConfig();
bool enable;
String scheme;
String host;
String usr;
String pwd;

ProxyConfig({
this.enable = false,
this.scheme = '',
this.host = '',
this.usr = '',
this.pwd = '',
});

factory ProxyConfig.fromJson(Map<String, dynamic> json) =>
_$ProxyConfigFromJson(json);
Expand All @@ -85,13 +99,20 @@ class ProxyConfig {

@JsonSerializable()
class ExtraConfigBt {
List<String> trackerSubscribeUrls = [];
List<String> subscribeTrackers = [];
List<String> trackerSubscribeUrls;
List<String> subscribeTrackers;
bool autoUpdateTrackers;
DateTime? lastTrackerUpdateTime;

List<String> customTrackers = [];
List<String> customTrackers;

ExtraConfigBt();
ExtraConfigBt({
this.trackerSubscribeUrls = const [],
this.subscribeTrackers = const [],
this.autoUpdateTrackers = true,
this.lastTrackerUpdateTime,
this.customTrackers = const [],
});

factory ExtraConfigBt.fromJson(Map<String, dynamic> json) =>
_$ExtraConfigBtFromJson(json);
Expand Down
Loading
Loading