Skip to content

Commit

Permalink
drive: add support of SHA-1 and SHA-256 checksum
Browse files Browse the repository at this point in the history
  • Loading branch information
@rinsuki @ncw
rinsuki authored and ncw committed Sep 24, 2023
1 parent 9e80d48 commit 8fd66da
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 56 deletions.
26 changes: 19 additions & 7 deletions backend/drive/drive.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ const (
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
minChunkSize = fs.SizeSuffix(googleapi.MinUploadChunkSize)
defaultChunkSize = 8 * fs.Mebi
partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
partialFields = "id,name,size,md5Checksum,sha1Checksum,sha256Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
listRGrouping = 50 // number of IDs to search at once when using ListR
listRInputBuffer = 1000 // size of input buffer when using ListR
defaultXDGIcon = "text-html"
Expand Down Expand Up @@ -323,13 +323,13 @@ rather than shortcuts themselves when doing server side copies.`,
}, {
Name: "skip_checksum_gphotos",
Default: false,
Help: `Skip MD5 checksum on Google photos and videos only.
Help: `Skip checksums on Google photos and videos only.
Use this if you get checksum errors when transferring Google photos or
videos.
Setting this flag will cause Google photos and videos to return a
blank MD5 checksum.
blank checksums.
Google photos are identified by being in the "photos" space.
Expand Down Expand Up @@ -751,6 +751,8 @@ type Object struct {
baseObject
url string // Download URL of this object
md5sum string // md5sum of the object
sha1sum string // sha1sum of the object
sha256sum string // sha256sum of the object
v2Download bool // generate v2 download link ondemand
}

Expand Down Expand Up @@ -1414,6 +1416,8 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
for _, space := range info.Spaces {
if space == "photos" {
info.Md5Checksum = ""
info.Sha1Checksum = ""
info.Sha256Checksum = ""
break
}
}
Expand All @@ -1422,6 +1426,8 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
baseObject: f.newBaseObject(remote, info),
url: fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
md5sum: strings.ToLower(info.Md5Checksum),
sha1sum: strings.ToLower(info.Sha1Checksum),
sha256sum: strings.ToLower(info.Sha256Checksum),
v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
}
if info.ResourceKey != "" {
Expand Down Expand Up @@ -3007,7 +3013,7 @@ func (f *Fs) DirCacheFlush() {

// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
return hash.Set(hash.MD5)
return hash.NewHashSet(hash.MD5, hash.SHA1, hash.SHA256)
}

func (f *Fs) changeChunkSize(chunkSizeString string) (err error) {
Expand Down Expand Up @@ -3568,10 +3574,16 @@ func (o *baseObject) Remote() string {

// Hash returns the Md5sum of an object returning a lowercase hex string
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
if t != hash.MD5 {
return "", hash.ErrUnsupported
if t == hash.MD5 {
return o.md5sum, nil
}
if t == hash.SHA1 {
return o.sha1sum, nil
}
if t == hash.SHA256 {
return o.sha256sum, nil
}
return o.md5sum, nil
return "", hash.ErrUnsupported
}
func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) {
if t != hash.MD5 {
Expand Down
8 changes: 6 additions & 2 deletions docs/content/drive.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -772,13 +772,13 @@ Properties:

#### --drive-skip-checksum-gphotos

Skip MD5 checksum on Google photos and videos only.
Skip checksums on Google photos and videos only.

Use this if you get checksum errors when transferring Google photos or
videos.

Setting this flag will cause Google photos and videos to return a
blank MD5 checksum.
blank checksums.

Google photos are identified by being in the "photos" space.

Expand Down Expand Up @@ -1526,6 +1526,10 @@ Waiting a moderate period of time between attempts (estimated to be
approximately 1 hour) and/or not using --fast-list both seem to be
effective in preventing the problem.

### Hashes

We need to say that all files have MD5 hashes, but a small fraction of files uploaded may not have SHA1 or SHA256 hashes especially if they were uploaded before 2018.

## Making your own client_id

When you use rclone with Google drive in its default configuration you
Expand Down
94 changes: 47 additions & 47 deletions docs/content/overview.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,53 +14,53 @@ show through.

Here is an overview of the major features of each cloud storage system.

| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | Metadata |
| ---------------------------- |:----------------:|:-------:|:----------------:|:---------------:|:---------:|:--------:|
| 1Fichier | Whirlpool | - | No | Yes | R | - |
| Akamai Netstorage | MD5, SHA256 | R/W | No | No | R | - |
| Amazon Drive | MD5 | - | Yes | No | R | - |
| Amazon S3 (or S3 compatible) | MD5 | R/W | No | No | R/W | RWU |
| Backblaze B2 | SHA1 | R/W | No | No | R/W | - |
| Box | SHA1 | R/W | Yes | No | - | - |
| Citrix ShareFile | MD5 | R/W | Yes | No | - | - |
| Dropbox | DBHASH ¹ | R | Yes | No | - | - |
| Enterprise File Fabric | - | R/W | Yes | No | R/W | - |
| FTP | - | R/W ¹⁰ | No | No | - | - |
| Google Cloud Storage | MD5 | R/W | No | No | R/W | - |
| Google Drive | MD5 | R/W | No | Yes | R/W | - |
| Google Photos | - | - | No | Yes | R | - |
| HDFS | - | R/W | No | No | - | - |
| HiDrive | HiDrive ¹² | R/W | No | No | - | - |
| HTTP | - | R | No | No | R | - |
| Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | RWU |
| Jottacloud | MD5 | R/W | Yes | No | R | - |
| Koofr | MD5 | - | Yes | No | - | - |
| Mail.ru Cloud | Mailru ⁶ | R/W | Yes | No | - | - |
| Mega | - | - | No | Yes | - | - |
| Memory | MD5 | R/W | No | No | - | - |
| Microsoft Azure Blob Storage | MD5 | R/W | No | No | R/W | - |
| Microsoft OneDrive | QuickXorHash ⁵ | R/W | Yes | No | R | - |
| OpenDrive | MD5 | R/W | Yes | Partial ⁸ | - | - |
| OpenStack Swift | MD5 | R/W | No | No | R/W | - |
| Oracle Object Storage | MD5 | R/W | No | No | R/W | - |
| pCloud | MD5, SHA1 ⁷ | R | No | No | W | - |
| PikPak | MD5 | R | No | No | R | - |
| premiumize.me | - | - | Yes | No | R | - |
| put.io | CRC-32 | R/W | No | Yes | R | - |
| Proton Drive | SHA1 | R/W | No | No | R | - |
| QingStor | MD5 | - ⁹ | No | No | R/W | - |
| Quatrix by Maytech | - | R/W | No | No | - | - |
| Seafile | - | - | No | No | - | - |
| SFTP | MD5, SHA1 ² | R/W | Depends | No | - | - |
| Sia | - | - | No | No | - | - |
| SMB | - | - | Yes | No | - | - |
| SugarSync | - | - | No | No | - | - |
| Storj | - | R | No | No | - | - |
| Uptobox | - | - | No | Yes | - | - |
| WebDAV | MD5, SHA1 ³ | R ⁴ | Depends | No | - | - |
| Yandex Disk | MD5 | R/W | No | No | R | - |
| Zoho WorkDrive | - | - | No | No | - | - |
| The local filesystem | All | R/W | Depends | No | - | RWU |
| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | Metadata |
| ---------------------------- |:-----------------:|:-------:|:----------------:|:---------------:|:---------:|:--------:|
| 1Fichier | Whirlpool | - | No | Yes | R | - |
| Akamai Netstorage | MD5, SHA256 | R/W | No | No | R | - |
| Amazon Drive | MD5 | - | Yes | No | R | - |
| Amazon S3 (or S3 compatible) | MD5 | R/W | No | No | R/W | RWU |
| Backblaze B2 | SHA1 | R/W | No | No | R/W | - |
| Box | SHA1 | R/W | Yes | No | - | - |
| Citrix ShareFile | MD5 | R/W | Yes | No | - | - |
| Dropbox | DBHASH ¹ | R | Yes | No | - | - |
| Enterprise File Fabric | - | R/W | Yes | No | R/W | - |
| FTP | - | R/W ¹⁰ | No | No | - | - |
| Google Cloud Storage | MD5 | R/W | No | No | R/W | - |
| Google Drive | MD5, SHA1, SHA256 | R/W | No | Yes | R/W | - |
| Google Photos | - | - | No | Yes | R | - |
| HDFS | - | R/W | No | No | - | - |
| HiDrive | HiDrive ¹² | R/W | No | No | - | - |
| HTTP | - | R | No | No | R | - |
| Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | RWU |
| Jottacloud | MD5 | R/W | Yes | No | R | - |
| Koofr | MD5 | - | Yes | No | - | - |
| Mail.ru Cloud | Mailru ⁶ | R/W | Yes | No | - | - |
| Mega | - | - | No | Yes | - | - |
| Memory | MD5 | R/W | No | No | - | - |
| Microsoft Azure Blob Storage | MD5 | R/W | No | No | R/W | - |
| Microsoft OneDrive | QuickXorHash ⁵ | R/W | Yes | No | R | - |
| OpenDrive | MD5 | R/W | Yes | Partial ⁸ | - | - |
| OpenStack Swift | MD5 | R/W | No | No | R/W | - |
| Oracle Object Storage | MD5 | R/W | No | No | R/W | - |
| pCloud | MD5, SHA1 ⁷ | R | No | No | W | - |
| PikPak | MD5 | R | No | No | R | - |
| premiumize.me | - | - | Yes | No | R | - |
| put.io | CRC-32 | R/W | No | Yes | R | - |
| Proton Drive | SHA1 | R/W | No | No | R | - |
| QingStor | MD5 | - ⁹ | No | No | R/W | - |
| Quatrix by Maytech | - | R/W | No | No | - | - |
| Seafile | - | - | No | No | - | - |
| SFTP | MD5, SHA1 ² | R/W | Depends | No | - | - |
| Sia | - | - | No | No | - | - |
| SMB | - | - | Yes | No | - | - |
| SugarSync | - | - | No | No | - | - |
| Storj | - | R | No | No | - | - |
| Uptobox | - | - | No | Yes | - | - |
| WebDAV | MD5, SHA1 ³ | R ⁴ | Depends | No | - | - |
| Yandex Disk | MD5 | R/W | No | No | R | - |
| Zoho WorkDrive | - | - | No | No | - | - |
| The local filesystem | All | R/W | Depends | No | - | RWU |

### Notes

Expand Down

0 comments on commit 8fd66da

Please sign in to comment.