Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Sierra issues #81

Open
copumpkin opened this issue Oct 12, 2016 · 3 comments
Open

Sierra issues #81

copumpkin opened this issue Oct 12, 2016 · 3 comments

Comments

@copumpkin
Copy link
Contributor

copumpkin commented Oct 12, 2016 •
edited
Loading

#79 upgraded to Go 1.7.1, but that doesn't seem sufficient to fix our issues. It seemed to work at first, but after a while, I get stack traces in the agent along the lines of:

failed MSpanList_Insert 0x8a48c8 0x2f9cbb4ded2cb 0x0
fatal error: MSpanList_Insert

runtime stack:
runtime.MSpanList_Insert(0x7e7bf8, 0x8a48c8)
    /usr/src/go/src/runtime/mheap.c:692 +0x8f
runtime.MHeap_Alloc(0x7e7640, 0x1, 0x10000000025, 0xe8e9)
    /usr/src/go/src/runtime/mheap.c:240 +0x66
runtime.MCentral_CacheSpan(0x7f0918, 0x0)
    /usr/src/go/src/runtime/mcentral.c:85 +0x167
runtime.MCache_Refill(0x89f000, 0x25, 0x800000000)
    /usr/src/go/src/runtime/mcache.c:90 +0xa0

goroutine 37 [running]:
runtime.switchtoM()
    /usr/src/go/src/runtime/asm_amd64.s:198 fp=0xc2080de1f0 sp=0xc2080de1e8
runtime.mallocgc(0x800, 0x3abec0, 0x1, 0x0)
    /usr/src/go/src/runtime/malloc.go:178 +0x849 fp=0xc2080de2a0 sp=0xc2080de1f0
runtime.newarray(0x3abec0, 0x800, 0x0)
    /usr/src/go/src/runtime/malloc.go:365 +0xc1 fp=0xc2080de2d8 sp=0xc2080de2a0
runtime.makeslice(0x39d320, 0x3d1, 0x800, 0x0, 0x0, 0x0)
    /usr/src/go/src/runtime/slice.go:32 +0x15c fp=0xc2080de320 sp=0xc2080de2d8
crypto/tls.(*block).reserve(0xc208082cc0, 0x452)
    /usr/src/go/src/crypto/tls/conn.go:438 +0x74 fp=0xc2080de3a0 sp=0xc2080de320
crypto/tls.(*block).readFromUntil(0xc208082cc0, 0x8b2988, 0xc20803c0a0, 0x452, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/conn.go:452 +0x73 fp=0xc2080de408 sp=0xc2080de3a0
crypto/tls.(*Conn).readRecord(0xc20807a2c0, 0x16, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/conn.go:586 +0xc60 fp=0xc2080de950 sp=0xc2080de408
crypto/tls.(*Conn).readHandshake(0xc20807a2c0, 0x0, 0x0, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/conn.go:779 +0xe7 fp=0xc2080deb10 sp=0xc2080de950
crypto/tls.(*clientHandshakeState).doFullHandshake(0xc208070480, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/handshake_client.go:227 +0x68 fp=0xc2080df0a8 sp=0xc2080deb10
crypto/tls.(*Conn).clientHandshake(0xc20807a2c0, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/handshake_client.go:197 +0x1981 fp=0xc2080df528 sp=0xc2080df0a8
crypto/tls.(*Conn).Handshake(0xc20807a2c0, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/conn.go:977 +0xf1 fp=0xc2080df568 sp=0xc2080df528
crypto/tls.DialWithDialer(0xc208044800, 0x53cdf0, 0x3, 0xc2080587d0, 0x50, 0xc2080d07e0, 0x50dce0, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/tls.go:141 +0x495 fp=0xc2080df678 sp=0xc2080df568
crypto/tls.Dial(0x53cdf0, 0x3, 0xc2080587d0, 0x50, 0xc2080d06c0, 0x0, 0x0, 0x0)
    /usr/src/go/src/crypto/tls/tls.go:165 +0x82 fp=0xc2080df6c8 sp=0xc2080df678
github.com/AdRoll/hologram/transport/remote.NewClient(0xc2080587d0, 0x50, 0x0, 0x0, 0x0, 0x0)
    /go/src/github.com/AdRoll/hologram/transport/remote/client.go:44 +0x260 fp=0xc2080df780 sp=0xc2080df6c8
github.com/AdRoll/hologram/agent.(*client).requestCredentials(0xc20801ec80, 0xc208044580, 0xc208087760, 0x17, 0x0, 0x0)
    /go/src/github.com/AdRoll/hologram/agent/client.go:169 +0x62 fp=0xc2080df980 sp=0xc2080df780
github.com/AdRoll/hologram/agent.(*client).AssumeRole(0xc20801ec80, 0xc208087760, 0x17, 0x0, 0x0)
    /go/src/github.com/AdRoll/hologram/agent/client.go:157 +0x112 fp=0xc2080df9d0 sp=0xc2080df980
github.com/AdRoll/hologram/agent.(*credentialsExpirationManager).maybeRefreshCredentials(0xc208044480, 0x0, 0x0)
    /go/src/github.com/AdRoll/hologram/agent/credentials_expiration_manager.go:63 +0x1e6 fp=0xc2080dfa68 sp=0xc2080df9d0
github.com/AdRoll/hologram/agent.(*credentialsExpirationManager).GetCredentials(0xc208044480, 0xc200000000, 0x0, 0x0)
    /go/src/github.com/AdRoll/hologram/agent/credentials_expiration_manager.go:48 +0x120 fp=0xc2080dfab8 sp=0xc2080dfa68
github.com/AdRoll/hologram/agent.(*metadataService).getCredentials(0xc20801ec60, 0x8b2a80, 0xc208064500, 0xc20803ad00)
    /go/src/github.com/AdRoll/hologram/agent/metadata_service.go:128 +0x53 fp=0xc2080dfc10 sp=0xc2080dfab8
github.com/AdRoll/hologram/agent.*metadataService.(github.com/AdRoll/hologram/agent.getCredentials)·fm(0x8b2a80, 0xc208064500, 0xc20803ad00)
    /go/src/github.com/AdRoll/hologram/agent/metadata_service.go:62 +0x45 fp=0xc2080dfc38 sp=0xc2080dfc10
net/http.HandlerFunc.ServeHTTP(0xc20800aca0, 0x8b2a80, 0xc208064500, 0xc20803ad00)
    /usr/src/go/src/net/http/server.go:1265 +0x41 fp=0xc2080dfc58 sp=0xc2080dfc38
net/http.(*ServeMux).ServeHTTP(0xc20800d920, 0x8b2a80, 0xc208064500, 0xc20803ad00)
    /usr/src/go/src/net/http/server.go:1541 +0x17d fp=0xc2080dfcb0 sp=0xc2080dfc58
net/http.serverHandler.ServeHTTP(0xc208056300, 0x8b2a80, 0xc208064500, 0xc20803ad00)
    /usr/src/go/src/net/http/server.go:1703 +0x19a fp=0xc2080dfd08 sp=0xc2080dfcb0
net/http.(*conn).serve(0xc208064320)
    /usr/src/go/src/net/http/server.go:1204 +0xb57 fp=0xc2080dffd8 sp=0xc2080dfd08
runtime.goexit()
    /usr/src/go/src/runtime/asm_amd64.s:2232 +0x1 fp=0xc2080dffe0 sp=0xc2080dffd8
created by net/http.(*Server).Serve
    /usr/src/go/src/net/http/server.go:1751 +0x35e

goroutine 1 [select]:
main.main()
    /go/src/github.com/AdRoll/hologram/cmd/hologram-agent/main.go:122 +0x1376

goroutine 5 [syscall]:
os/signal.loop()
    /usr/src/go/src/os/signal/signal_unix.go:21 +0x1f
created by os/signal.init·1
    /usr/src/go/src/os/signal/signal_unix.go:27 +0x35

goroutine 6 [IO wait]:
net.(*pollDesc).Wait(0xc208011250, 0x72, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc208011250, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).accept(0xc2080111f0, 0x0, 0x8b0be0, 0xc208088018)
    /usr/src/go/src/net/fd_unix.go:419 +0x40b
net.(*TCPListener).AcceptTCP(0xc20803c078, 0xc20805edf8, 0x0, 0x0)
    /usr/src/go/src/net/tcpsock_posix.go:234 +0x4e
net.(*TCPListener).Accept(0xc20803c078, 0x0, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/tcpsock_posix.go:244 +0x4c
net/http.(*Server).Serve(0xc208056300, 0x8b2708, 0xc20803c078, 0x0, 0x0)
    /usr/src/go/src/net/http/server.go:1728 +0x92
net/http.Serve(0x8b2708, 0xc20803c078, 0x8b2888, 0xc20800d920, 0x0, 0x0)
    /usr/src/go/src/net/http/server.go:1606 +0xa1
github.com/AdRoll/hologram/agent.(*metadataService).listen(0xc20801ec60)
    /go/src/github.com/AdRoll/hologram/agent/metadata_service.go:67 +0x3be
created by github.com/AdRoll/hologram/agent.(*metadataService).Start
    /go/src/github.com/AdRoll/hologram/agent/metadata_service.go:50 +0x44

goroutine 7 [IO wait]:
net.(*pollDesc).Wait(0xc2080112c0, 0x72, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc2080112c0, 0x0, 0x0)
    /usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).accept(0xc208011260, 0x0, 0x8b0be0, 0xc208088748)
    /usr/src/go/src/net/fd_unix.go:419 +0x40b
net.(*UnixListener).AcceptUnix(0xc20801ed40, 0x36aa0, 0x0, 0x0)
    /usr/src/go/src/net/unixsock_posix.go:282 +0x56
net.(*UnixListener).Accept(0xc20801ed40, 0x0, 0x0, 0x0, 0x0)
    /usr/src/go/src/net/unixsock_posix.go:293 +0x4c
github.com/AdRoll/hologram/transport/local.(*server).listen(0xc20801ed60)
    /go/src/github.com/AdRoll/hologram/transport/local/server.go:33 +0x4f
created by github.com/AdRoll/hologram/transport/local.NewServer
    /go/src/github.com/AdRoll/hologram/transport/local/server.go:65 +0x121

goroutine 20 [sleep]:
net.func·019()
    /usr/src/go/src/net/dnsclient_unix.go:240 +0x5a
created by net.loadConfig
    /usr/src/go/src/net/dnsclient_unix.go:269 +0x20c
failed MSpanList_Insert 0x8a3000 0x2f9cbb56eb615 0x0
fatal error: MSpanList_Insert

runtime stack:
runtime.throw(0x7be56b)
    /usr/src/go/src/runtime/panic.go:491 +0xad fp=0x7fff5fbffa50 sp=0x7fff5fbffa20
runtime.MSpanList_Insert(0x7eaa48, 0x8a3000)
    /usr/src/go/src/runtime/mheap.c:692 +0x8f fp=0x7fff5fbffa78 sp=0x7fff5fbffa50
MHeap_FreeSpanLocked(0x7e7640, 0x8a3000, 0x100)
    /usr/src/go/src/runtime/mheap.c:583 +0x163 fp=0x7fff5fbffab8 sp=0x7fff5fbffa78
MHeap_Grow(0x7e7640, 0x8, 0x0)
    /usr/src/go/src/runtime/mheap.c:420 +0x1a8 fp=0x7fff5fbffaf8 sp=0x7fff5fbffab8
MHeap_AllocSpanLocked(0x7e7640, 0x1, 0x0)
    /usr/src/go/src/runtime/mheap.c:298 +0x365 fp=0x7fff5fbffb38 sp=0x7fff5fbffaf8
mheap_alloc(0x7e7640, 0x1, 0x12, 0x0)
    /usr/src/go/src/runtime/mheap.c:190 +0x121 fp=0x7fff5fbffb60 sp=0x7fff5fbffb38
runtime.MHeap_Alloc(0x7e7640, 0x1, 0x10000000012, 0xe8e9)
    /usr/src/go/src/runtime/mheap.c:240 +0x66 fp=0x7fff5fbffb98 sp=0x7fff5fbffb60
MCentral_Grow(0x7ef3b8, 0x0)
    /usr/src/go/src/runtime/mcentral.c:197 +0x8b fp=0x7fff5fbffc00 sp=0x7fff5fbffb98
runtime.MCentral_CacheSpan(0x7ef3b8, 0x0)
    /usr/src/go/src/runtime/mcentral.c:85 +0x167 fp=0x7fff5fbffc38 sp=0x7fff5fbffc00
runtime.MCache_Refill(0x89f000, 0x12, 0x0)
    /usr/src/go/src/runtime/mcache.c:90 +0xa0 fp=0x7fff5fbffc60 sp=0x7fff5fbffc38
runtime.mcacheRefill_m()
    /usr/src/go/src/runtime/malloc.c:368 +0x57 fp=0x7fff5fbffc80 sp=0x7fff5fbffc60
runtime.onM(0x632bf8)
    /usr/src/go/src/runtime/asm_amd64.s:273 +0x9a fp=0x7fff5fbffc88 sp=0x7fff5fbffc80
runtime.mallocgc(0x120, 0x518020, 0x0, 0x0)
    /usr/src/go/src/runtime/malloc.go:178 +0x849 fp=0x7fff5fbffd38 sp=0x7fff5fbffc88
runtime.newobject(0x518020, 0x89f000)
    /usr/src/go/src/runtime/malloc.go:353 +0x49 fp=0x7fff5fbffd60 sp=0x7fff5fbffd38
runtime.newG(0x283ba)
    /usr/src/go/src/runtime/proc.go:233 +0x2a fp=0x7fff5fbffd78 sp=0x7fff5fbffd60
allocg(0x7d7520)
    /usr/src/go/src/runtime/proc.c:925 +0x1f fp=0x7fff5fbffd88 sp=0x7fff5fbffd78
runtime.malg(0x8000, 0x7d76e0)
    /usr/src/go/src/runtime/proc.c:2106 +0x1f fp=0x7fff5fbffdb8 sp=0x7fff5fbffd88
runtime.mpreinit(0x7d7f20)
    /usr/src/go/src/runtime/os_darwin.c:137 +0x27 fp=0x7fff5fbffdd0 sp=0x7fff5fbffdb8
mcommoninit(0x7d7f20)
    /usr/src/go/src/runtime/proc.c:201 +0xc9 fp=0x7fff5fbffdf8 sp=0x7fff5fbffdd0
runtime.schedinit()
    /usr/src/go/src/runtime/proc.c:138 +0x55 fp=0x7fff5fbffe20 sp=0x7fff5fbffdf8
runtime.rt0_go(0x7fff5fbffe50, 0x1, 0x7fff5fbffe50, 0x0, 0x1, 0x7fff5fbffee0, 0x0, 0x7fff5fbffefe, 0x7fff5fbfff21, 0x7fff5fbfff46, ...)
    /usr/src/go/src/runtime/asm_amd64.s:95 +0x116 fp=0x7fff5fbffe28 sp=0x7fff5fbffe20
@copumpkin
Copy link
Contributor Author

Is nobody else seeing this behavior?

@copumpkin
Copy link
Contributor Author

Ah, so I don't really know how Docker is supposed to work, but something is wonky with the current setup.

I added a call to go version to start.sh and sure enough, it says "go1.4.2".

Furthermore, when I tinker with the Dockerfile, the hologram.sh script doesn't seem to notice, so the script doesn't appear to be using the Dockerfile in the repository.

When I ran docker rmi adroll/hologram_env and it redownloaded the image, all was well and I was on 1.7.1.

@walterking
Copy link
Contributor

does that solve the crash? I've been running in sierra with no issues, but I dont use the docker build

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
2 participants
@copumpkin @walterking