Skip to content

Commit e50a6a5

Browse files
rfjakobrfjakob
committed
syscallcompat: implement Getdents()
The Readdir function provided by os is inherently slow because it calls Lstat on all files. Getdents gives us all the information we need, but does not have a proper wrapper in the stdlib. Implement the "Getdents()" wrapper function that calls syscall.Getdents() and parses the returned byte blob to a fuse.DirEntry slice.
1 parent affb1c2 commit e50a6a5

File tree

3 files changed

+232
-0
lines changed

3 files changed

+232
-0
lines changed

‎internal/syscallcompat/getdents_linux.go

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
// +build linux
2+
3+
package syscallcompat
4+
5+
// Other implementations of getdents in Go:
6+
// https://github.com/ericlagergren/go-gnulib/blob/cb7a6e136427e242099b2c29d661016c19458801/dirent/getdents_unix.go
7+
// https://github.com/golang/tools/blob/5831d16d18029819d39f99bdc2060b8eff410b6b/imports/fastwalk_unix.go
8+
9+
import (
10+
"bytes"
11+
"syscall"
12+
"unsafe"
13+
14+
"github.com/hanwen/go-fuse/fuse"
15+
16+
"github.com/rfjakob/gocryptfs/internal/tlog"
17+
)
18+
19+
// HaveGetdents is true if we have a working implementation of Getdents
20+
constHaveGetdents=true
21+
22+
constsizeofDirent=int(unsafe.Sizeof(syscall.Dirent{}))
23+
24+
// Getdents wraps syscall.Getdents and converts the result to []fuse.DirEntry.
25+
// The function takes a path instead of an fd because we need to be able to
26+
// call Lstat on files. Fstatat is not yet available in Go as of v1.9:
27+
// https://github.com/golang/go/issues/14216
28+
funcGetdents(dirstring) ([]fuse.DirEntry, error) {
29+
fd, err:=syscall.Open(dir, syscall.O_RDONLY, 0)
30+
iferr!=nil {
31+
returnnil, err
32+
}
33+
defersyscall.Close(fd)
34+
// Collect syscall result in smartBuf.
35+
// "bytes.Buffer" is smart about expanding the capacity and avoids the
36+
// exponential runtime of simple append().
37+
varsmartBuf bytes.Buffer
38+
tmp:=make([]byte, 10000)
39+
for {
40+
n, err:=syscall.Getdents(fd, tmp)
41+
iferr!=nil {
42+
returnnil, err
43+
}
44+
ifn==0 {
45+
break
46+
}
47+
smartBuf.Write(tmp[:n])
48+
}
49+
// Make sure we have at least Sizeof(Dirent) of zeros after the last
50+
// entry. This prevents a cast to Dirent from reading past the buffer.
51+
smartBuf.Grow(sizeofDirent)
52+
buf:=smartBuf.Bytes()
53+
// Count the number of directory entries in the buffer so we can allocate
54+
// a fuse.DirEntry slice of the correct size at once.
55+
varnumEntries, offsetint
56+
foroffset<len(buf) {
57+
s:=*(*syscall.Dirent)(unsafe.Pointer(&buf[offset]))
58+
ifs.Reclen==0 {
59+
tlog.Warn.Printf("Getdents: corrupt entry #%d: Reclen=0 at offset=%d. Returning EBADR",
60+
numEntries, offset)
61+
// EBADR = Invalid request descriptor
62+
returnnil, syscall.EBADR
63+
}
64+
ifint(s.Reclen) >sizeofDirent {
65+
tlog.Warn.Printf("Getdents: corrupt entry #%d: Reclen=%d > %d. Returning EBADR",
66+
numEntries, sizeofDirent, s.Reclen)
67+
returnnil, syscall.EBADR
68+
}
69+
offset+=int(s.Reclen)
70+
numEntries++
71+
}
72+
// Parse the buffer into entries
73+
entries:=make([]fuse.DirEntry, 0, numEntries)
74+
offset=0
75+
foroffset<len(buf) {
76+
s:=*(*syscall.Dirent)(unsafe.Pointer(&buf[offset]))
77+
name, err:=getdentsName(s)
78+
iferr!=nil {
79+
returnnil, err
80+
}
81+
offset+=int(s.Reclen)
82+
ifname=="."||name==".." {
83+
// os.File.Readdir() drops "." and "..". Let's be compatible.
84+
continue
85+
}
86+
mode, err:=convertDType(s.Type, dir+"/"+name)
87+
iferr!=nil {
88+
// The file may have been deleted in the meantime. Just skip it
89+
// and go on.
90+
continue
91+
}
92+
entries=append(entries, fuse.DirEntry{
93+
Ino: s.Ino,
94+
Mode: mode,
95+
Name: name,
96+
})
97+
}
98+
returnentries, nil
99+
}
100+
101+
// getdentsName extracts the filename from a Dirent struct and returns it as
102+
// a Go string.
103+
funcgetdentsName(s syscall.Dirent) (string, error) {
104+
// After the loop, l contains the index of the first '\0'.
105+
l:=0
106+
forl=ranges.Name {
107+
ifs.Name[l] ==0 {
108+
break
109+
}
110+
}
111+
ifl<1 {
112+
tlog.Warn.Printf("Getdents: invalid name length l=%d. Returning EBADR", l)
113+
// EBADR = Invalid request descriptor
114+
return"", syscall.EBADR
115+
}
116+
// Copy to byte slice.
117+
name:=make([]byte, l)
118+
fori:=rangename {
119+
name[i] =byte(s.Name[i])
120+
}
121+
returnstring(name), nil
122+
}
123+
124+
// convertDType converts a Dirent.Type to at Stat_t.Mode value.
125+
funcconvertDType(dtypeuint8, filestring) (uint32, error) {
126+
ifdtype!=syscall.DT_UNKNOWN {
127+
// Shift up by four octal digits = 12 bits
128+
returnuint32(dtype) <<12, nil
129+
}
130+
// DT_UNKNOWN: we have to call Lstat()
131+
varst syscall.Stat_t
132+
err:=syscall.Lstat(file, &st)
133+
iferr!=nil {
134+
return0, err
135+
}
136+
// The S_IFMT bit mask extracts the file type from the mode.
137+
returnst.Mode&syscall.S_IFMT, nil
138+
}

‎internal/syscallcompat/getdents_other.go

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// +build !linux
2+
3+
package syscallcompat
4+
5+
import (
6+
"log"
7+
8+
"github.com/hanwen/go-fuse/fuse"
9+
)
10+
11+
// HaveGetdents is true if we have a working implementation of Getdents
12+
constHaveGetdents=false
13+
14+
funcGetdents(dirstring) ([]fuse.DirEntry, error) {
15+
log.Panic("only implemented on Linux")
16+
returnnil, nil
17+
}

‎internal/syscallcompat/getdents_test.go

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
// +build linux
2+
3+
package syscallcompat
4+
5+
import (
6+
"io/ioutil"
7+
"os"
8+
"strings"
9+
"syscall"
10+
"testing"
11+
12+
"github.com/hanwen/go-fuse/fuse"
13+
)
14+
15+
funcTestGetdents(t*testing.T) {
16+
// Fill a directory with filenames of length 1 ... 255
17+
testDir, err:=ioutil.TempDir("", "TestGetdents")
18+
iferr!=nil {
19+
t.Fatal(err)
20+
}
21+
fori:=1; i<=syscall.NAME_MAX; i++ {
22+
n:=strings.Repeat("x", i)
23+
err=ioutil.WriteFile(testDir+"/"+n, nil, 0600)
24+
iferr!=nil {
25+
t.Fatal(err)
26+
}
27+
}
28+
// "/", "/dev" and "/proc" are good test cases because they contain many
29+
// different file types (block and char devices, symlinks, mountpoints)
30+
dirs:= []string{testDir, "/", "/dev", "/proc"}
31+
for_, dir:=rangedirs {
32+
// Read directory using stdlib Readdir()
33+
fd, err:=os.Open(dir)
34+
iferr!=nil {
35+
t.Fatal(err)
36+
}
37+
readdirEntries, err:=fd.Readdir(0)
38+
iferr!=nil {
39+
t.Fatal(err)
40+
}
41+
fd.Close()
42+
readdirMap:=make(map[string]*syscall.Stat_t)
43+
for_, v:=rangereaddirEntries {
44+
readdirMap[v.Name()] =fuse.ToStatT(v)
45+
}
46+
// Read using our Getdents()
47+
getdentsEntries, err:=Getdents(dir)
48+
iferr!=nil {
49+
t.Fatal(err)
50+
}
51+
getdentsMap:=make(map[string]fuse.DirEntry)
52+
for_, v:=rangegetdentsEntries {
53+
getdentsMap[v.Name] =v
54+
}
55+
// Compare results
56+
iflen(getdentsEntries) !=len(readdirEntries) {
57+
t.Fatalf("len(getdentsEntries)=%d, len(readdirEntries)=%d",
58+
len(getdentsEntries), len(readdirEntries))
59+
}
60+
forname:=rangereaddirMap {
61+
g:=getdentsMap[name]
62+
r:=readdirMap[name]
63+
rTyp:=r.Mode&syscall.S_IFMT
64+
ifg.Mode!=rTyp {
65+
t.Errorf("%q: g.Mode=%#o, r.Mode=%#o", name, g.Mode, rTyp)
66+
}
67+
ifg.Ino!=r.Ino {
68+
// The inode number of a directory that is reported by stat
69+
// and getdents is different when it is a mountpoint. Only
70+
// throw an error when we are NOT looking at a directory.
71+
ifg.Mode!=syscall.S_IFDIR {
72+
t.Errorf("%s: g.Ino=%d, r.Ino=%d", name, g.Ino, r.Ino)
73+
}
74+
}
75+
}
76+
}
77+
}

0 commit comments

Comments
 (0)
close