sftpd: add SSH_FXP_FSETSTAT support

This change will fix file editing from sshfs, we need this patch

pkg/sftp#373

for pkg/sftp to support this feature

common/common.go

@@ -37,6 +37,7 @@ const (
 	chownLogSender           = "Chown"
 	chmodLogSender           = "Chmod"
 	chtimesLogSender         = "Chtimes"
+	truncateLogSender        = "Truncate"
 	operationDownload        = "download"
 	operationUpload          = "upload"
 	operationDelete          = "delete"
@@ -52,6 +53,7 @@ const (
 	StatAttrUIDGID = 1
 	StatAttrPerms  = 2
 	StatAttrTimes  = 4
+	StatAttrSize   = 8
 )
 
 // Transfer types
@@ -85,6 +87,8 @@ var (
 	ErrQuotaExceeded        = errors.New("denying write due to space limit")
 	ErrSkipPermissionsCheck = errors.New("permission check skipped")
 	ErrConnectionDenied     = errors.New("You are not allowed to connect")
+	errNoTransfer           = errors.New("requested transfer not found")
+	errTransferMismatch     = errors.New("transfer mismatch")
 )
 
 var (
@@ -141,6 +145,7 @@ type ActiveTransfer interface {
 	GetVirtualPath() string
 	GetStartTime() time.Time
 	SignalClose()
+	Truncate(fsPath string, size int64) error
 }
 
 // ActiveConnection defines the interface for the current active connections
@@ -168,6 +173,7 @@ type StatAttributes struct {
 	UID   int
 	GID   int
 	Flags int
+	Size  int64
 }
 
 // ConnectionTransfer defines the trasfer details to expose

common/connection.go

@@ -172,6 +172,20 @@ func (c *BaseConnection) SignalTransfersAbort() error {
 	return nil
 }
 
+func (c *BaseConnection) truncateOpenHandle(fsPath string, size int64) error {
+	c.RLock()
+	defer c.RUnlock()
+
+	for _, t := range c.activeTransfers {
+		err := t.Truncate(fsPath, size)
+		if err != errTransferMismatch {
+			return err
+		}
+	}
+
+	return errNoTransfer
+}
+
 // ListDir reads the directory named by fsPath and returns a list of directory entries
 func (c *BaseConnection) ListDir(fsPath, virtualPath string) ([]os.FileInfo, error) {
 	if !c.User.HasPerm(dataprovider.PermListItems, virtualPath) {
@@ -200,7 +214,7 @@ func (c *BaseConnection) CreateDir(fsPath, virtualPath string) error {
 	}
 	vfs.SetPathPermissions(c.Fs, fsPath, c.User.GetUID(), c.User.GetGID())
 
-	logger.CommandLog(mkdirLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
+	logger.CommandLog(mkdirLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1)
 	return nil
 }
 
@@ -233,7 +247,7 @@ func (c *BaseConnection) RemoveFile(fsPath, virtualPath string, info os.FileInfo
 		}
 	}
 
-	logger.CommandLog(removeLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
+	logger.CommandLog(removeLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1)
 	if info.Mode()&os.ModeSymlink != os.ModeSymlink {
 		vfolder, err := c.User.GetVirtualFolderForPath(path.Dir(virtualPath))
 		if err == nil {
@@ -302,7 +316,7 @@ func (c *BaseConnection) RemoveDir(fsPath, virtualPath string) error {
 		return c.GetFsError(err)
 	}
 
-	logger.CommandLog(rmdirLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
+	logger.CommandLog(rmdirLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1)
 	return nil
 }
 
@@ -363,7 +377,7 @@ func (c *BaseConnection) Rename(fsSourcePath, fsTargetPath, virtualSourcePath, v
 		c.updateQuotaAfterRename(virtualSourcePath, virtualTargetPath, fsTargetPath, initialSize) //nolint:errcheck
 	}
 	logger.CommandLog(renameLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1,
-		"", "", "")
+		"", "", "", -1)
 	action := newActionNotification(&c.User, operationRename, fsSourcePath, fsTargetPath, "", c.protocol, 0, nil)
 	// the returned error is used in test cases only, we already log the error inside action.execute
 	go action.execute() //nolint:errcheck
@@ -400,21 +414,27 @@ func (c *BaseConnection) CreateSymlink(fsSourcePath, fsTargetPath, virtualSource
 		c.Log(logger.LevelWarn, "failed to create symlink %#v -> %#v: %+v", fsSourcePath, fsTargetPath, err)
 		return c.GetFsError(err)
 	}
-	logger.CommandLog(symlinkLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
+	logger.CommandLog(symlinkLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1)
 	return nil
 }
 
-// SetStat set StatAttributes for the specified fsPath
-func (c *BaseConnection) SetStat(fsPath, virtualPath string, attributes *StatAttributes) error {
-	if Config.SetstatMode == 1 {
-		return nil
-	}
+func (c *BaseConnection) getPathForSetStatPerms(fsPath, virtualPath string) string {
 	pathForPerms := virtualPath
 	if fi, err := c.Fs.Lstat(fsPath); err == nil {
 		if fi.IsDir() {
 			pathForPerms = path.Dir(virtualPath)
 		}
 	}
+	return pathForPerms
+}
+
+// SetStat set StatAttributes for the specified fsPath
+func (c *BaseConnection) SetStat(fsPath, virtualPath string, attributes *StatAttributes) error {
+	if Config.SetstatMode == 1 {
+		return nil
+	}
+	pathForPerms := c.getPathForSetStatPerms(fsPath, virtualPath)
+
 	if attributes.Flags&StatAttrPerms != 0 {
 		if !c.User.HasPerm(dataprovider.PermChmod, pathForPerms) {
 			return c.GetPermissionDeniedError()
@@ -424,7 +444,7 @@ func (c *BaseConnection) SetStat(fsPath, virtualPath string, attributes *StatAtt
 			return c.GetFsError(err)
 		}
 		logger.CommandLog(chmodLogSender, fsPath, "", c.User.Username, attributes.Mode.String(), c.ID, c.protocol,
-			-1, -1, "", "", "")
+			-1, -1, "", "", "", -1)
 	}
 
 	if attributes.Flags&StatAttrUIDGID != 0 {
@@ -437,12 +457,12 @@ func (c *BaseConnection) SetStat(fsPath, virtualPath string, attributes *StatAtt
 			return c.GetFsError(err)
 		}
 		logger.CommandLog(chownLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, attributes.UID, attributes.GID,
-			"", "", "")
+			"", "", "", -1)
 	}
 
 	if attributes.Flags&StatAttrTimes != 0 {
 		if !c.User.HasPerm(dataprovider.PermChtimes, pathForPerms) {
-			return sftp.ErrSSHFxPermissionDenied
+			return c.GetPermissionDeniedError()
 		}
 
 		if err := c.Fs.Chtimes(fsPath, attributes.Atime, attributes.Mtime); err != nil {
@@ -453,12 +473,37 @@ func (c *BaseConnection) SetStat(fsPath, virtualPath string, attributes *StatAtt
 		accessTimeString := attributes.Atime.Format(chtimesFormat)
 		modificationTimeString := attributes.Mtime.Format(chtimesFormat)
 		logger.CommandLog(chtimesLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1,
-			accessTimeString, modificationTimeString, "")
+			accessTimeString, modificationTimeString, "", -1)
+	}
+
+	if attributes.Flags&StatAttrSize != 0 {
+		if !c.User.HasPerm(dataprovider.PermOverwrite, pathForPerms) {
+			return c.GetPermissionDeniedError()
+		}
+
+		if err := c.truncateFile(fsPath, attributes.Size); err != nil {
+			c.Log(logger.LevelWarn, "failed to truncate path %#v, size: %v, err: %+v", fsPath, attributes.Size, err)
+			return c.GetFsError(err)
+		}
+		logger.CommandLog(truncateLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", attributes.Size)
 	}
 
 	return nil
 }
 
+func (c *BaseConnection) truncateFile(fsPath string, size int64) error {
+	// check first if we have an open transfer for the given path and try to truncate the file already opened
+	// if we found no transfer we truncate by path.
+	// pkg/sftp should expose an optional interface and call truncate directly on the opened handle ...
+	// If we try to truncate by path an already opened file we get an error on Windows
+	err := c.truncateOpenHandle(fsPath, size)
+	if err == errNoTransfer {
+		c.Log(logger.LevelDebug, "file path %#v not found in active transfers, execute trucate by path", fsPath)
+		err = c.Fs.Truncate(fsPath, size)
+	}
+	return err
+}
+
 func (c *BaseConnection) checkRecursiveRenameDirPermissions(sourcePath, targetPath string) error {
 	dstPerms := []string{
 		dataprovider.PermCreateDirs,

common/connection_test.go

@@ -529,6 +529,30 @@ func TestSetStat(t *testing.T) {
 		Flags: StatAttrTimes,
 	})
 	assert.Error(t, err)
+	// truncate
+	err = c.SetStat(filepath.Join(user.GetHomeDir(), "missing"), "/missing", &StatAttributes{
+		Size:  1,
+		Flags: StatAttrSize,
+	})
+	assert.Error(t, err)
+	err = c.SetStat(filepath.Join(dir3, "afile.txt"), "/dir3/afile.txt", &StatAttributes{
+		Size:  1,
+		Flags: StatAttrSize,
+	})
+	assert.Error(t, err)
+
+	filePath := filepath.Join(user.GetHomeDir(), "afile.txt")
+	err = ioutil.WriteFile(filePath, []byte("hello"), os.ModePerm)
+	assert.NoError(t, err)
+	err = c.SetStat(filePath, "/afile.txt", &StatAttributes{
+		Flags: StatAttrSize,
+		Size:  1,
+	})
+	assert.NoError(t, err)
+	fi, err := os.Stat(filePath)
+	if assert.NoError(t, err) {
+		assert.Equal(t, int64(1), fi.Size())
+	}
 
 	err = os.RemoveAll(user.GetHomeDir())
 	assert.NoError(t, err)

common/transfer.go

@@ -108,6 +108,22 @@ func (t *BaseTransfer) SetCancelFn(cancelFn func()) {
 	t.cancelFn = cancelFn
 }
 
+// Truncate changes the size of the opened file.
+// Supported for local fs only
+func (t *BaseTransfer) Truncate(fsPath string, size int64) error {
+	if fsPath == t.GetFsPath() {
+		if t.File != nil {
+			return t.File.Truncate(size)
+		}
+		if size == 0 {
+			// for cloud providers the file is always truncated to zero, we don't support append/resume for uploads
+			return nil
+		}
+		return ErrOpUnsupported
+	}
+	return errTransferMismatch
+}
+
 // TransferError is called if there is an unexpected error.
 // For example network or client issues
 func (t *BaseTransfer) TransferError(err error) {

common/transfer_test.go

@@ -81,6 +81,53 @@ func TestTransferThrottling(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestTruncate(t *testing.T) {
+	testFile := filepath.Join(os.TempDir(), "transfer_test_file")
+	fs := vfs.NewOsFs("123", os.TempDir(), nil)
+	u := dataprovider.User{
+		Username: "user",
+		HomeDir:  os.TempDir(),
+	}
+	u.Permissions = make(map[string][]string)
+	u.Permissions["/"] = []string{dataprovider.PermAny}
+	file, err := os.Create(testFile)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, "unable to open test file")
+	}
+	_, err = file.Write([]byte("hello"))
+	assert.NoError(t, err)
+	conn := NewBaseConnection(fs.ConnectionID(), ProtocolSFTP, u, fs)
+	transfer := NewBaseTransfer(file, conn, nil, testFile, "/transfer_test_file", TransferUpload, 0, 0, true)
+
+	err = conn.SetStat(testFile, "/transfer_test_file", &StatAttributes{
+		Size:  2,
+		Flags: StatAttrSize,
+	})
+	assert.NoError(t, err)
+	err = transfer.Close()
+	assert.NoError(t, err)
+	fi, err := os.Stat(testFile)
+	if assert.NoError(t, err) {
+		assert.Equal(t, int64(2), fi.Size())
+	}
+
+	transfer = NewBaseTransfer(nil, conn, nil, testFile, "", TransferUpload, 0, 0, true)
+	err = transfer.Truncate("mismatch", 0)
+	assert.EqualError(t, err, errTransferMismatch.Error())
+	err = transfer.Truncate(testFile, 0)
+	assert.NoError(t, err)
+	err = transfer.Truncate(testFile, 1)
+	assert.EqualError(t, err, ErrOpUnsupported.Error())
+
+	err = transfer.Close()
+	assert.NoError(t, err)
+
+	err = os.Remove(testFile)
+	assert.NoError(t, err)
+
+	assert.Len(t, conn.GetTransfers(), 0)
+}
+
 func TestTransferErrors(t *testing.T) {
 	isCancelled := false
 	cancelFn := func() {

docs/logs.md

@@ -20,7 +20,7 @@ The logs can be divided into the following categories:
   - `connection_id` string. Unique connection identifier
   - `protocol` string. `SFTP` or `SCP`
 - **"command logs"**, SFTP/SCP command logs:
-  - `sender` string. `Rename`, `Rmdir`, `Mkdir`, `Symlink`, `Remove`, `Chmod`, `Chown`, `Chtimes`, `SSHCommand`
+  - `sender` string. `Rename`, `Rmdir`, `Mkdir`, `Symlink`, `Remove`, `Chmod`, `Chown`, `Chtimes`, `Truncate`, `SSHCommand`
   - `level` string
   - `username`, string
   - `file_path` string
@@ -30,6 +30,7 @@ The logs can be divided into the following categories:
   - `gid` integer. Valid for sender `Chown` otherwise -1
   - `access_time` datetime as YYYY-MM-DDTHH:MM:SS. Valid for sender `Chtimes` otherwise empty
   - `modification_time` datetime as YYYY-MM-DDTHH:MM:SS. Valid for sender `Chtimes` otherwise empty
+  - `size` int64. Valid for sender `Truncate` otherwise -1
   - `ssh_command`, string. Valid for sender `SSHCommand` otherwise empty
   - `connection_id` string. Unique connection identifier
   - `protocol` string. `SFTP`, `SCP` or `SSH`

go.mod

@@ -3,10 +3,10 @@ module github.com/drakkan/sftpgo
 go 1.13
 
 require (
-	cloud.google.com/go v0.63.0 // indirect
+	cloud.google.com/go v0.64.0 // indirect
 	cloud.google.com/go/storage v1.10.0
 	github.com/alexedwards/argon2id v0.0.0-20200802152012-2464efd3196b
-	github.com/aws/aws-sdk-go v1.34.5
+	github.com/aws/aws-sdk-go v1.34.8
 	github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d
 	github.com/fclairamb/ftpserverlib v0.8.1-0.20200729230026-7f0ab9d81bb6
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
@@ -17,7 +17,7 @@ require (
 	github.com/grandcat/zeroconf v1.0.0
 	github.com/jlaffaye/ftp v0.0.0-20200720194710-13949d38913e
 	github.com/lib/pq v1.8.0
-	github.com/mattn/go-sqlite3 v1.14.0
+	github.com/mattn/go-sqlite3 v1.14.1
 	github.com/miekg/dns v1.1.31 // indirect
 	github.com/mitchellh/mapstructure v1.3.3 // indirect
 	github.com/nathanaelle/password/v2 v2.0.1
@@ -26,7 +26,7 @@ require (
 	github.com/pires/go-proxyproto v0.1.3
 	github.com/pkg/sftp v1.11.1-0.20200819110714-3ee8d0ba91c0
 	github.com/prometheus/client_golang v1.7.1
-	github.com/prometheus/common v0.12.0 // indirect
+	github.com/prometheus/common v0.13.0 // indirect
 	github.com/rs/cors v1.7.1-0.20200626170627-8b4a00bd362b
 	github.com/rs/xid v1.2.1
 	github.com/rs/zerolog v1.19.0
@@ -41,17 +41,17 @@ require (
 	go.etcd.io/bbolt v1.3.5
 	golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de
 	golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc
-	golang.org/x/sys v0.0.0-20200819091447-39769834ee22
-	golang.org/x/tools v0.0.0-20200814230902-9882f1d1823d // indirect
+	golang.org/x/sys v0.0.0-20200819171115-d785dc25833f
+	golang.org/x/tools v0.0.0-20200820010801-b793a1359eac // indirect
 	google.golang.org/api v0.30.0
-	google.golang.org/genproto v0.0.0-20200815001618-f69a88009b70 // indirect
-	gopkg.in/ini.v1 v1.58.0 // indirect
+	gopkg.in/ini.v1 v1.60.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
 replace (
 	github.com/fclairamb/ftpserverlib => github.com/drakkan/ftpserverlib v0.0.0-20200814103339-511fcfd63dfe
 	github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20200730125632-b21eac28818c
+	github.com/pkg/sftp => github.com/drakkan/sftp v0.0.0-20200820090459-de8eb908f763
 	golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20200731130417-7674a892f9b1
 	golang.org/x/net => github.com/drakkan/net v0.0.0-20200807161257-daa5cda5ae27
 )