resist stuck index tasks

[hanwen]

hanwen@zoekt-60gb-16cpu-localssd-drawfork-20180530:~$ ps -ef|grep zoekt
hanwen   14979 14250  0 15:31 pts/0    00:00:00 grep --color=auto zoekt
root     21184 27089  0 Apr21 ?        00:09:40 zoekt-git-index -require_ctags -parallelism=8 -repo_cache /zoekt/repos -index /zoekt-serving/index/ -incremental /zoekt/repos/github.com/sirthias/borer.git
root     21192 21184  0 Apr21 ?        00:00:17 /zoekt/bin/universal-ctags --_interactive=sandbox --fields=*
root     27089     1  0 Feb13 ?        06:17:56 /zoekt/bin/zoekt-indexserver -data_dir /zoekt -index_dir /zoekt-serving/index/ -mirror_config https://raw.githubusercontent.com/hanwen/zoekt-config/master/mirror.json -mirror_duration 7h -cpu_fraction 0.5
root     27370     1  8 Apr22 ?        3-02:23:25 /zoekt/bin/zoekt-webserver -listen :80 -index /zoekt-serving/index -log_dir /zoekt/logs -host_customization cs.bazel.build=r:bazel -template_dir=/zoekt/etc

here the index server was stuck since Apr 21. This should have timed out and continued with another repo instead.

[hanwen]

regrettably, I didn't kill with -QUIT. Let's see if this problems resurfaces.

[hanwen]

019/05/29 16:51:39 command [zoekt-git-index -require_ctags -parallelism=8 -repo_cache /zoekt/repos -index /zoekt-serving/index/ -incremental /zoekt/repos/github.com/sirthias/borer.git] 
OUT:
ERR: SIGQUIT: quit
PC=0x571b51 m=0 sigcode=0
goroutine 0 [idle]:
runtime.futex(0xfcc1c8, 0x80, 0x0, 0x0, 0x0, 0xc000039320, 0x0, 0x0, 0x7ffe05a14288, 0x521631, ...)
        /usr/lib/google-golang/src/runtime/sys_linux_amd64.s:535 +0x21
runtime.futexsleep(0xfcc1c8, 0x7ffe00000000, 0xffffffffffffffff)
        /usr/lib/google-golang/src/runtime/os_linux.go:46 +0x4b
runtime.notesleep(0xfcc1c8)
        /usr/lib/google-golang/src/runtime/lock_futex.go:151 +0xa1
runtime.stopm()
        /usr/lib/google-golang/src/runtime/proc.go:1945 +0xc1
runtime.findrunnable(0xc000038000, 0x0)
        /usr/lib/google-golang/src/runtime/proc.go:2408 +0x54a
runtime.schedule()
        /usr/lib/google-golang/src/runtime/proc.go:2534 +0x21c
runtime.park_m(0xc00008e780)
        /usr/lib/google-golang/src/runtime/proc.go:2614 +0xa1
runtime.mcall(0x0)
        /usr/lib/google-golang/src/runtime/asm_amd64.s:299 +0x5b
goroutine 1 [semacquire, 626 minutes]:
sync.runtime_Semacquire(0xc000478458)
        /usr/lib/google-golang/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc000478450)
        /usr/lib/google-golang/src/sync/waitgroup.go:130 +0x65
github.com/google/zoekt/build.(*Builder).Finish(0xc000478300, 0xc000024940, 0xb)
        /usr/local/google/home/hanwen/go/src/github.com/google/zoekt/build/builder.go:278 +0x5a
github.com/google/zoekt/gitindex.IndexGitRepo(0x7ffe05a15f1c, 0x2a, 0x101, 0x7ffe05a15ee5, 0xc, 0x7ffe05a15ef9, 0x15, 0x20000, 0x8, 0x6400000, ...)
        /usr/local/google/home/hanwen/go/src/github.com/google/zoekt/gitindex/index.go:510 +0x114a
main.main()
        /usr/local/google/home/hanwen/go/src/github.com/google/zoekt/cmd/zoekt-git-index/main.go:86 +0x7ea
goroutine 66 [syscall, 625 minutes]:
syscall.Syscall6(0xf7, 0x1, 0x377, 0xc000c753a0, 0x1000004, 0x0, 0x0, 0x59e, 0x59e, 0x0)
        /usr/lib/google-golang/src/syscall/asm_linux_amd64.s:44 +0x5
os.(*Process).blockUntilWaitable(0xc000178f90, 0x10000, 0x500, 0x0)
        /usr/lib/google-golang/src/os/wait_waitid.go:31 +0x98
os.(*Process).wait(0xc000178f90, 0x203000, 0x5, 0xfffd)
        /usr/lib/google-golang/src/os/exec_unix.go:22 +0x39
os.(*Process).Wait(...)
        /usr/lib/google-golang/src/os/exec.go:125
os/exec.(*Cmd).Wait(0xc000334160, 0x0, 0xc000320180)
        /usr/lib/google-golang/src/os/exec/exec.go:474 +0x61
github.com/google/zoekt/ctags.(*ctagsProcess).read(0xc000399590, 0xc000320180, 0x0, 0x0)
        /usr/local/google/home/hanwen/go/src/github.com/google/zoekt/ctags/json.go:86 +0x2a1
github.com/google/zoekt/ctags.(*ctagsProcess).Parse(0xc000399590, 0xc000105130, 0x43, 0xc0007bc000, 0x186a0, 0x1fe00, 0x0, 0x0, 0x0, 0x0, ...)
        /usr/local/google/home/hanwen/go/src/github.com/google/zoekt/ctags/json.go:169 +0x162
github.com/google/zoekt/ctags.(*lockedParser).Parse(0xc00000e060, 0xc000105130, 0x43, 0xc0007bc000, 0x186a0, 0x1fe00, 0x0, 0x0, 0x0, 0x0, ...)
        /usr/local/google/home/hanwen/go/src/github.com/google/zoekt/ctags/json.go:202 +0xd7
github.com/google/zoekt/build.ctagsAddSymbolsParser(0xc0004e6000, 0x1b3, 0x200, 0xb88780, 0xc00000e060, 0x0, 0x0)
        /usr/local/google/home/hanwen/go/src/github.com/google/zoekt/build/ctags.go:148 +0xad
github.com/google/zoekt/build.ctagsAddSymbols(0xc0004e6000, 0x1b3, 0x200, 0xb88780, 0xc00000e060, 0xc0000ec2c0, 0x1a, 0x0, 0x0)
        /usr/local/google/home/hanwen/go/src/github.com/google/zoekt/build/ctags.go:169 +0x8d2
github.com/google/zoekt/build.(*Builder).buildShard(0xc000478300, 0xc0004e6000, 0x1b3, 0x200, 0x0, 0x0, 0x0, 0x0)
        /usr/local/google/home/hanwen/go/src/github.com/google/zoekt/build/builder.go:450 +0x2a7
github.com/google/zoekt/build.(*Builder).flush.func1(0xc000478300, 0xc000b2a000, 0x0)
        /usr/local/google/home/hanwen/go/src/github.com/google/zoekt/build/builder.go:332 +0x85
created by github.com/google/zoekt/build.(*Builder).flush
        /usr/local/google/home/hanwen/go/src/github.com/google/zoekt/build/builder.go:330 +0x2e8
rax    0xca
rbx    0xfcc080
rcx    0x571b53
rdx    0x0
rdi    0xfcc1c8
rsi    0x80

[hanwen]

looks like a crash/hang in ctags

[hanwen]

.jvm/src/test/resources/n_structure_100000_opening_arrays.json

is a sequence of 100k '[' chars. This kills the json parser.

[hanwen]

reported as universal-ctags/ctags#2107