Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Failing test: x-pack/filebeat/input/awss3 TestBenchmarkInputS3 #30123

Closed
matschaffer opened this issue Feb 1, 2022 · 4 comments · Fixed by #30131
Closed

Failing test: x-pack/filebeat/input/awss3 TestBenchmarkInputS3 #30123

matschaffer opened this issue Feb 1, 2022 · 4 comments · Fixed by #30131
Assignees
@aspacca
Labels
flaky-test Unstable or unreliable test cases. Team:Integrations Label for the Integrations team

Comments

@matschaffer
Copy link
Contributor

Flaky Test

This CI flow fired after I merged #30103, failed on the first attempt and passed when retried via the github UI "Re-run all jobs" button.

cc @kaiyan-sheng who requested I open this after me raising it on slack

Stack Trace

goroutine 61445 [running]:
runtime.throw({0x5c7b1ea, 0x5a34560})
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/runtime/panic.go:1198 +0x71 fp=0xc00057dc20 sp=0xc00057dbf0 pc=0x40388b1
sync.throw({0x5c7b1ea, 0x587aec0})
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/runtime/panic.go:1184 +0x1e fp=0xc00057dc40 sp=0xc00057dc20 pc=0x406687e
sync.(*Mutex).unlockSlow(0xc000294030, 0xffffffff)
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/sync/mutex.go:196 +0x3c fp=0xc00057dc68 sp=0xc00057dc40 pc=0x4075ebc
sync.(*Mutex).Unlock(...)
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/sync/mutex.go:190
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).handlePurgingLock(0xc000148420, {{0x5c4e514, 0x6}, {0xc00050d750, 0x10}, {0xc00050d770, 0x9}, {0xc0763d3de2bc0f30, 0x16dc6d4369, 0x73ca460}, ...}, ...)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:110 +0x1ce fp=0xc00057dd80 sp=0xc00057dc68 pc=0x571242e
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).ProcessObject(0xc000148420, 0xea7e848)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:136 +0x329 fp=0xc00057df30 sp=0xc00057dd80 pc=0x57127a9
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll.func2()
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:345 +0x87 fp=0xc00057dfe0 sp=0xc00057df30 pc=0x5714107
runtime.goexit()
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/runtime/asm_amd64.s:1581 +0x1 fp=0xc00057dfe8 sp=0xc00057dfe0 pc=0x406ba21
created by github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:340 +0x1e8

goroutine 1 [chan receive, 1 minutes]:
testing.(*T).Run(0xc0002dc1a0, {0x5c670fc, 0xce8770a11c}, 0x5dbb058)
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/testing/testing.go:1307 +0x375
testing.runTests.func1(0xc0005ff830)
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/testing/testing.go:1598 +0x6e
testing.tRunner(0xc0002dc1a0, 0xc00057bd18)
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/testing/testing.go:1259 +0x102
testing.runTests(0xc000524880, {0x73987c0, 0x15, 0x15}, {0x408b44d, 0x5c5caac, 0x73ca460})
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/testing/testing.go:1596 +0x43f
testing.(*M).Run(0xc000524880)
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/testing/testing.go:1504 +0x51d
main.main()
	_testmain.go:83 +0x14b

goroutine 21 [chan receive]:
k8s.io/klog/v2.(*loggingT).flushDaemon(0xc00018a380)
	/Users/runner/go/pkg/mod/k8s.io/klog/v2@v2.30.0/klog.go:1181 +0x6a
created by k8s.io/klog/v2.init.0
	/Users/runner/go/pkg/mod/k8s.io/klog/v2@v2.30.0/klog.go:420 +0xfb

goroutine 61433 [select]:
github.com/elastic/beats/v7/x-pack/libbeat/common/aws.(*Sem).AcquireContext(0xc0004080f0, 0x8, {0x60409f8, 0xc00067cdc0})
	/Users/runner/work/beats/beats/x-pack/libbeat/common/aws/semaphore.go:36 +0x10f
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll(0xc000148420, {0x60409f8, 0xc00067cdc0})
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:317 +0x71
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.benchmarkInputS3.func1(0xc00016e6c0)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/input_benchmark_test.go:294 +0x7bf
testing.(*B).runN(0xc00016e6c0, 0x1)
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/testing/benchmark.go:192 +0x126
testing.(*B).run1.func1()
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/testing/benchmark.go:232 +0x59
created by testing.(*B).run1
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/testing/benchmark.go:225 +0xa5

goroutine 61385 [chan receive]:
testing.(*B).run1(0xc00016e6c0)
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/testing/benchmark.go:234 +0xbb
testing.Benchmark(0xc001158258)
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/testing/benchmark.go:817 +0xeb
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.benchmarkInputS3(0xc011aca1a0, 0x8)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/input_benchmark_test.go:255 +0x96
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.TestBenchmarkInputS3(0xc011aca1a0)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/input_benchmark_test.go:330 +0x1f7
testing.tRunner(0xc011aca1a0, 0x5dbb058)
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/testing/testing.go:1259 +0x102
created by testing.(*T).Run
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/testing/testing.go:1306 +0x35a

goroutine 61440 [chan receive]:
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).ProcessObject(0xc000148420, 0x0)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:117 +0x85
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll.func2()
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:345 +0x87
created by github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:340 +0x1e8

goroutine 61441 [chan receive]:
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).ProcessObject(0xc000148420, 0x0)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:117 +0x85
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll.func2()
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:345 +0x87
created by github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:340 +0x1e8

goroutine 61439 [runnable]:
github.com/elastic/go-structform/gotype.(*unfoldCtx).OnUint64(0x4006029, 0xcd5547c861cd18f4)
	/Users/runner/go/pkg/mod/github.com/elastic/go-structform@v0.0.9/gotype/unfold.go:290
github.com/elastic/go-structform.extArrVisitor.OnUint64Array({{0x6093508, 0xc001150000}}, {0xc000543780, 0x2, 0xc000e4fc80})
	/Users/runner/go/pkg/mod/github.com/elastic/go-structform@v0.0.9/array.go:161 +0x8e
github.com/elastic/go-structform/gotype.foldArrUint64(0x5838ac0, {0x5838ac0, 0xc00053d5d8})
	/Users/runner/go/pkg/mod/github.com/elastic/go-structform@v0.0.9/gotype/fold_arr.go:70 +0x48
github.com/elastic/go-structform/gotype.foldInterfaceValue(0xc0010d8c60, {0x5838ac0, 0xc00053d5d8})
	/Users/runner/go/pkg/mod/github.com/elastic/go-structform@v0.0.9/gotype/fold.go:105 +0x31e
github.com/elastic/go-structform/gotype.foldMapInterface(0xc0010d8c60, {0x5948c60, 0xc0124e1410})
	/Users/runner/go/pkg/mod/github.com/elastic/go-structform@v0.0.9/gotype/fold_map.go:55 +0x145
github.com/elastic/go-structform/gotype.foldInterfaceValue(0xc0010d8c60, {0x5948c60, 0xc0124e1410})
	/Users/runner/go/pkg/mod/github.com/elastic/go-structform@v0.0.9/gotype/fold.go:105 +0x31e
github.com/elastic/go-structform/gotype.(*Iterator).Fold(...)
	/Users/runner/go/pkg/mod/github.com/elastic/go-structform@v0.0.9/gotype/fold.go:93
github.com/elastic/beats/v7/libbeat/common/transform/typeconv.(*Converter).Convert(0xc000e4fc70, {0x5c1bc40, 0xc00125c390}, {0x5948c60, 0xc0124e1410})
	/Users/runner/work/beats/beats/libbeat/common/transform/typeconv/typeconv.go:108 +0x13a
github.com/elastic/beats/v7/libbeat/common/transform/typeconv.Convert({0x5c1bc40, 0xc00125c390}, {0x5948c60, 0xc0124e1410})
	/Users/runner/work/beats/beats/libbeat/common/transform/typeconv/typeconv.go:129 +0xd7
github.com/elastic/beats/v7/libbeat/statestore/storetest.(*MapStore).Get(0xc0124e1380, {0xc000766030, 0x25}, {0x5c1bc40, 0xc00125c390})
	/Users/runner/work/beats/beats/libbeat/statestore/storetest/storetest.go:156 +0x12d
github.com/elastic/beats/v7/libbeat/statestore.(*Store).Get(0xc00067cd80, {0xc000766030, 0x25}, {0x5c1bc40, 0xc00125c390})
	/Users/runner/work/beats/beats/libbeat/statestore/store.go:101 +0x150
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*states).MustSkip(0x5c4e514, {{0xc0010fb740, 0x52}, {0x5c4e514, 0x6}, {0xc00051f670, 0x10}, {0xc00051f690, 0x9}, {0xc0763d3de2bc0f30, ...}, ...}, ...)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/states.go:71 +0x12c
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).GetS3Objects(0xc000148420, {0x60409f8, 0xc00067cdc0}, 0xc00030c060)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:181 +0x7b8
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll.func1()
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:334 +0x6d
created by github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:329 +0x145

goroutine 61446 [chan receive]:
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).ProcessObject(0xc000148420, 0x0)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:117 +0x85
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll.func2()
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:345 +0x87
created by github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:340 +0x1e8

goroutine 61447 [chan receive]:
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).ProcessObject(0xc000148420, 0x0)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:117 +0x85
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll.func2()
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:345 +0x87
created by github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:340 +0x1e8

goroutine 61437 [sleep]:
time.Sleep(0x4c4b40)
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/runtime/time.go:193 +0x12e
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.benchmarkInputS3.func1.2()
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/input_benchmark_test.go:287 +0x32
created by github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.benchmarkInputS3.func1
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/input_benchmark_test.go:285 +0x777

goroutine 61444 [chan receive]:
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).ProcessObject(0xc000148420, 0x222c225a37333a37)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:117 +0x85
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll.func2()
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:345 +0x87
created by github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:340 +0x1e8

goroutine 61443 [chan receive]:
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).ProcessObject(0xc000148420, 0x0)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:117 +0x85
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll.func2()
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:345 +0x87
created by github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:340 +0x1e8

goroutine 61423 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000408108, 0x0)
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x0)
	/Users/runner/hostedtoolcache/go/1.17.6/x64/src/sync/cond.go:56 +0x8c
github.com/elastic/beats/v7/x-pack/libbeat/common/aws.(*Sem).Acquire(0xc0004080f0, 0x8)
	/Users/runner/work/beats/beats/x-pack/libbeat/common/aws/semaphore.go:53 +0xab
github.com/elastic/beats/v7/x-pack/libbeat/common/aws.(*Sem).AcquireContext.func1()
	/Users/runner/work/beats/beats/x-pack/libbeat/common/aws/semaphore.go:33 +0x65
created by github.com/elastic/beats/v7/x-pack/libbeat/common/aws.(*Sem).AcquireContext
	/Users/runner/work/beats/beats/x-pack/libbeat/common/aws/semaphore.go:31 +0xb0

goroutine 61442 [chan receive]:
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).ProcessObject(0xc000148420, 0x0)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:117 +0x85
github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll.func2()
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:345 +0x87
created by github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/s3.go:340 +0x1e8
@matschaffer matschaffer added the flaky-test Unstable or unreliable test cases. label Feb 1, 2022
@botelastic botelastic bot added the needs_team Indicates that the issue/PR needs a Team:* label label Feb 1, 2022
@matschaffer matschaffer added the Team:Integrations Label for the Integrations team label Feb 1, 2022
@elasticmachine
Copy link
Collaborator

Pinging @elastic/integrations (Team:Integrations)

@botelastic botelastic bot removed the needs_team Indicates that the issue/PR needs a Team:* label label Feb 1, 2022
@elasticmachine
Copy link
Collaborator

Pinging @elastic/security-external-integrations (Team:Security-External Integrations)

@efd6
Copy link
Contributor

efd6 commented Feb 1, 2022

This looked to me like data corruption; there is some weirdness in one of the running goroutines' calls:

0x5c4e514 is the receiver a *states, but also the address of the string data bytes for state.bucket in the second slot of the second parameter...

github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*states).MustSkip(0x5c4e514, {{0xc0010fb740, 0x52}, {0x5c4e514, 0x6}, {0xc00051f670, 0x10}, {0xc00051f690, 0x9}, {0xc0763d3de2bc0f30, ...}, ...}, ...)
	/Users/runner/work/beats/beats/x-pack/filebeat/input/awss3/states.go:71 +0x12c

This is not in the crashing goroutine, but is suggestive of underlying memory corruption problems since a well behaved program should not have a pointer to a string's bytes also be a pointer to an *awss3.states. So I ran the test with the race detector and I think that this may show if not the problem, at least a problem that needs to be addressed.

==================
WARNING: DATA RACE
Write at 0x00c000453a88 by goroutine 365:
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*states).Update()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/states.go:158 +0x54f
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).handlePurgingLock()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:103 +0x19a
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).ProcessObject()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:136 +0x3a4
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll.func2()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:345 +0xdc

Previous read at 0x00c000453a88 by goroutine 50:
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*states).IsListingFullyStored()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/states.go:112 +0xb9
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).handlePurgingLock()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:107 +0x1ce
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).ProcessObject()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:136 +0x3a4
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll.func2()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:345 +0xdc

Goroutine 365 (running) created at:
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:340 +0x3ad
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.benchmarkInputS3.func1()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/input_benchmark_test.go:294 +0xd0f
  testing.(*B).runN()
      /Users/user/goroot/src/testing/benchmark.go:192 +0x1e1
  testing.(*B).run1.func1()
      /Users/user/goroot/src/testing/benchmark.go:232 +0x78

Goroutine 50 (running) created at:
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:340 +0x3ad
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.benchmarkInputS3.func1()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/input_benchmark_test.go:294 +0xd0f
  testing.(*B).runN()
      /Users/user/goroot/src/testing/benchmark.go:192 +0x1e1
  testing.(*B).run1.func1()
      /Users/user/goroot/src/testing/benchmark.go:232 +0x78
==================
==================
WARNING: DATA RACE
Read at 0x00c000453a98 by goroutine 365:
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*states).IsListingFullyStored()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/states.go:108 +0x93
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).handlePurgingLock()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:107 +0x1ce
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).ProcessObject()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:136 +0x3a4
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll.func2()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:345 +0xdc

Previous write at 0x00c000453a98 by goroutine 50:
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*states).IsListingFullyStored()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/states.go:112 +0x104
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).handlePurgingLock()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:107 +0x1ce
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).ProcessObject()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:136 +0x3a4
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll.func2()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:345 +0xdc

Goroutine 365 (running) created at:
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:340 +0x3ad
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.benchmarkInputS3.func1()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/input_benchmark_test.go:294 +0xd0f
  testing.(*B).runN()
      /Users/user/goroot/src/testing/benchmark.go:192 +0x1e1
  testing.(*B).run1.func1()
      /Users/user/goroot/src/testing/benchmark.go:232 +0x78

Goroutine 50 (running) created at:
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.(*s3Poller).Poll()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/s3.go:340 +0x3ad
  github.com/elastic/beats/v7/x-pack/filebeat/input/awss3.benchmarkInputS3.func1()
      /Users/user/go/src/github.com/elastic/beats/x-pack/filebeat/input/awss3/input_benchmark_test.go:294 +0xd0f
  testing.(*B).runN()
      /Users/user/goroot/src/testing/benchmark.go:192 +0x1e1
  testing.(*B).run1.func1()
      /Users/user/goroot/src/testing/benchmark.go:232 +0x78
==================

The lines involved are certainly suggestive of involvement.

@efd6
Copy link
Contributor

efd6 commented Feb 1, 2022

Yes, this would definitely explain it, both listingInfo.storedObjects and listingInfo.finalCheck should be protected by a mutex, and data races on those values could easily allow execution to successfully pass the condition on line 107 twice.

@aspacca aspacca mentioned this issue Feb 1, 2022
Merged
3 tasks
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees

@aspacca aspacca

Labels
flaky-test Unstable or unreliable test cases. Team:Integrations Label for the Integrations team
Projects
None yet
Milestone
No milestone
Development

Successfully merging a pull request may close this issue.

add mutex to listinfInfo
5 participants
@matschaffer @aspacca @kaiyan-sheng @elasticmachine @efd6