[release-branch.go1.12] syscall: avoid _getdirentries64 on darwin

Getdirentries is implemented with the __getdirentries64 function
in libSystem.dylib. That function works, but it's on Apple's
can't-be-used-in-an-app-store-application list.

Implement Getdirentries using the underlying fdopendir/readdir_r/closedir.
The simulation isn't faithful, and could be slow, but it should handle
common cases.

Don't use Getdirentries in the stdlib, use fdopendir/readdir_r/closedir
instead (via (*os.File).readdirnames).

(Incorporates CL 170837 and CL 170698, which were small fixes to the
original tip CL.)

Fixes #31244

Update #28984

RELNOTE=yes

Change-Id: Ia6b5d003e5bfe43ba54b1e1d9cfa792cc6511717
Reviewed-on: https://go-review.googlesource.com/c/go/+/168479
Reviewed-by: Emmanuel Odeke <emm.odeke@gmail.com>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Run-TryBot: Emmanuel Odeke <emm.odeke@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
(cherry picked from commit 9da6530)
Reviewed-on: https://go-review.googlesource.com/c/go/+/170640
Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org>

src/internal/poll/fd_opendir_ios.go → src/internal/poll/fd_opendir_darwin.go

@@ -2,9 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build darwin
-// +build arm arm64
-
 package poll
 
 import (

src/os/dir_ios.go → src/os/dir_darwin.go

@@ -2,9 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build darwin
-// +build arm arm64
-
 package os
 
 import (
@@ -47,12 +44,12 @@ func (f *File) readdirnames(n int) (names []string, err error) {
 
 	names = make([]string, 0, size)
 	var dirent syscall.Dirent
-	var entptr uintptr
-	for len(names) < size {
-		if res := readdir_r(d.dir, uintptr(unsafe.Pointer(&dirent)), uintptr(unsafe.Pointer(&entptr))); res != 0 {
+	var entptr *syscall.Dirent
+	for len(names) < size || n == -1 {
+		if res := readdir_r(d.dir, &dirent, &entptr); res != 0 {
 			return names, wrapSyscallError("readdir", syscall.Errno(res))
 		}
-		if entptr == 0 { // EOF
+		if entptr == nil { // EOF
 			break
 		}
 		if dirent.Ino == 0 {
@@ -84,4 +81,4 @@ func (f *File) readdirnames(n int) (names []string, err error) {
 func closedir(dir uintptr) (err error)
 
 //go:linkname readdir_r syscall.readdir_r
-func readdir_r(dir, entry, result uintptr) (res int)
+func readdir_r(dir uintptr, entry *syscall.Dirent, result **syscall.Dirent) (res int)

src/os/dir_unix.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build aix darwin,!arm,!arm64 dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris
+// +build aix dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris
 
 package os
 

src/runtime/sys_darwin.go

@@ -89,6 +89,17 @@ func syscall_syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr)
 }
 func syscall6X()
 
+//go:linkname syscall_syscallPtr syscall.syscallPtr
+//go:nosplit
+//go:cgo_unsafe_args
+func syscall_syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) {
+	entersyscallblock()
+	libcCall(unsafe.Pointer(funcPC(syscallPtr)), unsafe.Pointer(&fn))
+	exitsyscall()
+	return
+}
+func syscallPtr()
+
 //go:linkname syscall_rawSyscall syscall.rawSyscall
 //go:nosplit
 //go:cgo_unsafe_args

src/runtime/sys_darwin_32.go

@@ -19,14 +19,3 @@ func syscall_syscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2, e
 	return
 }
 func syscall9()
-
-//go:linkname syscall_syscallPtr syscall.syscallPtr
-//go:nosplit
-//go:cgo_unsafe_args
-func syscall_syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) {
-	entersyscallblock()
-	libcCall(unsafe.Pointer(funcPC(syscallPtr)), unsafe.Pointer(&fn))
-	exitsyscall()
-	return
-}
-func syscallPtr()

src/runtime/sys_darwin_386.s

@@ -675,9 +675,42 @@ ok:
 	POPL	BP
 	RET
 
-// Not used on 386.
+// syscallPtr is like syscall except the libc function reports an
+// error by returning NULL and setting errno.
 TEXT runtime·syscallPtr(SB),NOSPLIT,$0
-	MOVL	$0xf1, 0xf1  // crash
+	PUSHL	BP
+	MOVL	SP, BP
+	SUBL	$24, SP
+	MOVL	32(SP), CX
+	MOVL	(0*4)(CX), AX // fn
+	MOVL	(1*4)(CX), DX // a1
+	MOVL	DX, 0(SP)
+	MOVL	(2*4)(CX), DX // a2
+	MOVL	DX, 4(SP)
+	MOVL	(3*4)(CX), DX // a3
+	MOVL	DX, 8(SP)
+
+	CALL	AX
+
+	MOVL	32(SP), CX
+	MOVL	AX, (4*4)(CX) // r1
+	MOVL	DX, (5*4)(CX) // r2
+
+	// syscallPtr libc functions return NULL on error
+	// and set errno.
+	TESTL	AX, AX
+	JNE	ok
+
+	// Get error code from libc.
+	CALL	libc_error(SB)
+	MOVL	(AX), AX
+	MOVL	32(SP), CX
+	MOVL	AX, (6*4)(CX) // err
+
+ok:
+	XORL	AX, AX        // no error (it's ignored anyway)
+	MOVL	BP, SP
+	POPL	BP
 	RET
 
 // syscall6 calls a function in libc on behalf of the syscall package.

src/runtime/sys_darwin_64.go

@@ -19,14 +19,3 @@ func syscall_syscallX(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) {
 	return
 }
 func syscallX()
-
-//go:linkname syscall_syscallXPtr syscall.syscallXPtr
-//go:nosplit
-//go:cgo_unsafe_args
-func syscall_syscallXPtr(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) {
-	entersyscallblock()
-	libcCall(unsafe.Pointer(funcPC(syscallXPtr)), unsafe.Pointer(&fn))
-	exitsyscall()
-	return
-}
-func syscallXPtr()

src/runtime/sys_darwin_amd64.s

@@ -637,9 +637,40 @@ ok:
 	POPQ	BP
 	RET
 
-// Not used on amd64.
-TEXT runtime·syscallXPtr(SB),NOSPLIT,$0
-	MOVL	$0xf1, 0xf1  // crash
+// syscallPtr is like syscallX except that the libc function reports an
+// error by returning NULL and setting errno.
+TEXT runtime·syscallPtr(SB),NOSPLIT,$0
+	PUSHQ	BP
+	MOVQ	SP, BP
+	SUBQ	$16, SP
+	MOVQ	(0*8)(DI), CX // fn
+	MOVQ	(2*8)(DI), SI // a2
+	MOVQ	(3*8)(DI), DX // a3
+	MOVQ	DI, (SP)
+	MOVQ	(1*8)(DI), DI // a1
+	XORL	AX, AX	      // vararg: say "no float args"
+
+	CALL	CX
+
+	MOVQ	(SP), DI
+	MOVQ	AX, (4*8)(DI) // r1
+	MOVQ	DX, (5*8)(DI) // r2
+
+	// syscallPtr libc functions return NULL on error
+	// and set errno.
+	TESTQ	AX, AX
+	JNE	ok
+
+	// Get error code from libc.
+	CALL	libc_error(SB)
+	MOVLQSX	(AX), AX
+	MOVQ	(SP), DI
+	MOVQ	AX, (6*8)(DI) // err
+
+ok:
+	XORL	AX, AX        // no error (it's ignored anyway)
+	MOVQ	BP, SP
+	POPQ	BP
 	RET
 
 // syscall6 calls a function in libc on behalf of the syscall package.

src/runtime/sys_darwin_arm.s

@@ -418,7 +418,7 @@ ok:
 	RET
 
 // syscallPtr is like syscall except the libc function reports an
-// error by returning NULL.
+// error by returning NULL and setting errno.
 TEXT runtime·syscallPtr(SB),NOSPLIT,$0
 	MOVW.W	R0, -4(R13)	// push structure pointer
 	MOVW	0(R0), R12	// fn

src/runtime/sys_darwin_arm64.s

@@ -465,9 +465,9 @@ TEXT runtime·syscallX(SB),NOSPLIT,$0
 ok:
 	RET
 
-// syscallXPtr is like syscallX except that the libc function reports an
-// error by returning NULL.
-TEXT runtime·syscallXPtr(SB),NOSPLIT,$0
+// syscallPtr is like syscallX except that the libc function reports an
+// error by returning NULL and setting errno.
+TEXT runtime·syscallPtr(SB),NOSPLIT,$0
 	SUB	$16, RSP	// push structure pointer
 	MOVD	R0, (RSP)
 

src/syscall/dirent_bsd_test.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build darwin,!arm,!arm64 dragonfly freebsd netbsd openbsd
+// +build darwin dragonfly freebsd netbsd openbsd
 
 package syscall_test
 

src/syscall/mksyscall.pl

@@ -350,6 +350,10 @@ ($)
 			$text .= "//go:linkname $funcname $funcname\n";
 			# Tell the linker that funcname can be found in libSystem using varname without the libc_ prefix.
 			my $basename = substr $funcname, 5;
+			if($basename eq "readdir_r" && ($ENV{'GOARCH'} eq "386" || $ENV{'GOARCH'} eq "amd64")) {
+			    # Hack to make sure we get the 64-bit inode version on darwin/macOS.
+			    $basename .= "\$INODE64"
+			}
 			$text .= "//go:cgo_import_dynamic $funcname $basename \"/usr/lib/libSystem.B.dylib\"\n";
 		}
 	}

src/syscall/syscall_darwin.go

@@ -258,6 +258,7 @@ func Kill(pid int, signum Signal) (err error) { return kill(pid, int(signum), 1)
 //sys	Chown(path string, uid int, gid int) (err error)
 //sys	Chroot(path string) (err error)
 //sys	Close(fd int) (err error)
+//sys	closedir(dir uintptr) (err error)
 //sys	Dup(fd int) (nfd int, err error)
 //sys	Dup2(from int, to int) (err error)
 //sys	Exchangedata(path1 string, path2 string, options int) (err error)
@@ -301,6 +302,7 @@ func Kill(pid int, signum Signal) (err error) { return kill(pid, int(signum), 1)
 //sys	Pread(fd int, p []byte, offset int64) (n int, err error)
 //sys	Pwrite(fd int, p []byte, offset int64) (n int, err error)
 //sys	read(fd int, p []byte) (n int, err error)
+//sys	readdir_r(dir uintptr, entry *Dirent, result **Dirent) (res Errno)
 //sys	Readlink(path string, buf []byte) (n int, err error)
 //sys	Rename(from string, to string) (err error)
 //sys	Revoke(path string) (err error)
@@ -363,12 +365,70 @@ func writelen(fd int, buf *byte, nbuf int) (n int, err error) {
 	return
 }
 
+func Getdirentries(fd int, buf []byte, basep *uintptr) (n int, err error) {
+	// Simulate Getdirentries using fdopendir/readdir_r/closedir.
+	const ptrSize = unsafe.Sizeof(uintptr(0))
+	fd2, err := Dup(fd)
+	if err != nil {
+		return 0, err
+	}
+	d, err := fdopendir(fd2)
+	if err != nil {
+		Close(fd2)
+		return 0, err
+	}
+	defer closedir(d)
+	// We keep the number of records already returned in *basep.
+	// It's not the full required semantics, but should handle the case
+	// of calling Getdirentries repeatedly.
+	// It won't handle assigning the results of lseek to *basep, or handle
+	// the directory being edited underfoot.
+	skip := *basep
+	*basep = 0
+	for {
+		var entry Dirent
+		var entryp *Dirent
+		e := readdir_r(d, &entry, &entryp)
+		if e != 0 {
+			return n, errnoErr(e)
+		}
+		if entryp == nil {
+			break
+		}
+		if skip > 0 {
+			skip--
+			*basep++
+			continue
+		}
+		reclen := int(entry.Reclen)
+		if reclen > len(buf) {
+			// Not enough room. Return for now.
+			// *basep will let us know where we should start up again.
+			// Note: this strategy for suspending in the middle and
+			// restarting is O(n^2) in the length of the directory. Oh well.
+			break
+		}
+		// Copy entry into return buffer.
+		s := struct {
+			ptr unsafe.Pointer
+			siz int
+			cap int
+		}{ptr: unsafe.Pointer(&entry), siz: reclen, cap: reclen}
+		copy(buf, *(*[]byte)(unsafe.Pointer(&s)))
+		buf = buf[reclen:]
+		n += reclen
+		*basep++
+	}
+	return n, nil
+}
+
 // Implemented in the runtime package (runtime/sys_darwin.go)
 func syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno)
 func syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno)
 func syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno)
 func rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno)
 func rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno)
+func syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno)
 
 // Find the entry point for f. See comments in runtime/proc.go for the
 // function of the same name.

src/syscall/syscall_darwin_386.go

@@ -16,7 +16,6 @@ func setTimeval(sec, usec int64) Timeval {
 
 //sys	Fstat(fd int, stat *Stat_t) (err error) = SYS_fstat64
 //sys	Fstatfs(fd int, stat *Statfs_t) (err error) = SYS_fstatfs64
-//sys	Getdirentries(fd int, buf []byte, basep *uintptr) (n int, err error) = SYS___getdirentries64
 //sysnb	Gettimeofday(tp *Timeval) (err error)
 //sys	Lstat(path string, stat *Stat_t) (err error) = SYS_lstat64
 //sys	Stat(path string, stat *Stat_t) (err error) = SYS_stat64
@@ -59,6 +58,20 @@ func libc_sendfile_trampoline()
 //go:linkname libc_sendfile libc_sendfile
 //go:cgo_import_dynamic libc_sendfile sendfile "/usr/lib/libSystem.B.dylib"
 
+func fdopendir(fd int) (dir uintptr, err error) {
+	r0, _, e1 := syscallPtr(funcPC(libc_fdopendir_trampoline), uintptr(fd), 0, 0)
+	dir = uintptr(r0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func libc_fdopendir_trampoline()
+
+//go:linkname libc_fdopendir libc_fdopendir
+//go:cgo_import_dynamic libc_fdopendir fdopendir "/usr/lib/libSystem.B.dylib"
+
 // Implemented in the runtime package (runtime/sys_darwin_32.go)
 func syscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno)
 

src/syscall/syscall_darwin_amd64.go

@@ -16,7 +16,6 @@ func setTimeval(sec, usec int64) Timeval {
 
 //sys	Fstat(fd int, stat *Stat_t) (err error) = SYS_fstat64
 //sys	Fstatfs(fd int, stat *Statfs_t) (err error) = SYS_fstatfs64
-//sys	Getdirentries(fd int, buf []byte, basep *uintptr) (n int, err error) = SYS___getdirentries64
 //sysnb	Gettimeofday(tp *Timeval) (err error)
 //sys	Lstat(path string, stat *Stat_t) (err error) = SYS_lstat64
 //sys	Stat(path string, stat *Stat_t) (err error) = SYS_stat64
@@ -59,6 +58,20 @@ func libc_sendfile_trampoline()
 //go:linkname libc_sendfile libc_sendfile
 //go:cgo_import_dynamic libc_sendfile sendfile "/usr/lib/libSystem.B.dylib"
 
+func fdopendir(fd int) (dir uintptr, err error) {
+	r0, _, e1 := syscallPtr(funcPC(libc_fdopendir_trampoline), uintptr(fd), 0, 0)
+	dir = uintptr(r0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func libc_fdopendir_trampoline()
+
+//go:linkname libc_fdopendir libc_fdopendir
+//go:cgo_import_dynamic libc_fdopendir fdopendir$INODE64 "/usr/lib/libSystem.B.dylib"
+
 // Implemented in the runtime package (runtime/sys_darwin_64.go)
 func syscallX(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno)