ecs-agent stacktrace

2016-02-27T00:32:16Z [CRITICAL] ====== STACKTRACE ======
2016-02-27 00:32:16.22684745 +0000 UTC
goroutine 13 [running]:
github.com/aws/amazon-ecs-agent/agent/sighandlers.func·001()
	/go/src/github.com/aws/amazon-ecs-agent/agent/sighandlers/debug_handler.go:33 +0xcc
created by github.com/aws/amazon-ecs-agent/agent/sighandlers.StartDebugHandler
	/go/src/github.com/aws/amazon-ecs-agent/agent/sighandlers/debug_handler.go:38 +0x18e

goroutine 1 [select, 19 minutes]:
github.com/aws/amazon-ecs-agent/agent/acs/handler.func·006(0x0, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/acs/handler/acs_handler.go:151 +0xbde
github.com/aws/amazon-ecs-agent/agent/acs/handler.StartSession(0x7f1a5bf50dd8, 0xc20800a220, 0xc2080c8fc0, 0x5d, 0xc20804c2d0, 0xc2080a0c60, 0x7f1a5bf50e18, 0xc208052480, 0x7f1a5bf51858, 0xc2080a2700, ...)
	/go/src/github.com/aws/amazon-ecs-agent/agent/acs/handler/acs_handler.go:160 +0x4b5
main._main(0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/agent.go:244 +0x2405
main.main()
	/go/src/github.com/aws/amazon-ecs-agent/agent/agent.go:54 +0x1f

goroutine 5 [semacquire, 2864 minutes]:
sync.(*Cond).Wait(0xc208038f80)
	/usr/src/go/src/sync/cond.go:62 +0x9e
github.com/cihub/seelog.(*asyncLoopLogger).processItem(0xc208046540, 0x421b00)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/cihub/seelog/behavior_asynclooplogger.go:50 +0xc2
github.com/cihub/seelog.(*asyncLoopLogger).processQueue(0xc208046540)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/cihub/seelog/behavior_asynclooplogger.go:63 +0x31
created by github.com/cihub/seelog.newAsyncLoopLogger
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/cihub/seelog/behavior_asynclooplogger.go:40 +0x8e

goroutine 6 [semacquire, 2864 minutes]:
sync.(*Cond).Wait(0xc208039140)
	/usr/src/go/src/sync/cond.go:62 +0x9e
github.com/cihub/seelog.(*asyncLoopLogger).processItem(0xc208046660, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/cihub/seelog/behavior_asynclooplogger.go:50 +0xc2
github.com/cihub/seelog.(*asyncLoopLogger).processQueue(0xc208046660)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/cihub/seelog/behavior_asynclooplogger.go:63 +0x31
created by github.com/cihub/seelog.newAsyncLoopLogger
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/cihub/seelog/behavior_asynclooplogger.go:40 +0x8e

goroutine 7 [semacquire]:
sync.(*Cond).Wait(0xc208038980)
	/usr/src/go/src/sync/cond.go:62 +0x9e
github.com/cihub/seelog.(*asyncLoopLogger).processItem(0xc208046180, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/cihub/seelog/behavior_asynclooplogger.go:50 +0xc2
github.com/cihub/seelog.(*asyncLoopLogger).processQueue(0xc208046180)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/cihub/seelog/behavior_asynclooplogger.go:63 +0x31
created by github.com/cihub/seelog.newAsyncLoopLogger
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/cihub/seelog/behavior_asynclooplogger.go:40 +0x8e

goroutine 8 [syscall]:
os/signal.loop()
	/usr/src/go/src/os/signal/signal_unix.go:21 +0x1f
created by os/signal.init·1
	/usr/src/go/src/os/signal/signal_unix.go:27 +0x35

goroutine 40 [semacquire, 9 minutes]:
sync.(*RWMutex).Lock(0xc2080922a0)
	/usr/src/go/src/sync/rwmutex.go:87 +0xa4
github.com/fsouza/go-dockerclient.(*eventMonitoringState).updateLastSeen(0xc2080922a0, 0xc208109980)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/event.go:242 +0x33
github.com/fsouza/go-dockerclient.(*eventMonitoringState).monitorEvents(0xc2080922a0, 0xc20809e180)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/event.go:185 +0x242
created by github.com/fsouza/go-dockerclient.(*eventMonitoringState).enableEventMonitoring
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/event.go:143 +0x16f

goroutine 41 [chan receive, 2864 minutes]:
github.com/aws/amazon-ecs-agent/agent/engine.func·011()
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_container_engine.go:554 +0x6e
created by github.com/aws/amazon-ecs-agent/agent/engine.(*dockerGoClient).ContainerEvents
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_container_engine.go:556 +0x3f7

goroutine 36 [select]:
net.func·019()
	/usr/src/go/src/net/dnsclient_unix.go:241 +0x2f4
created by net.loadConfig
	/usr/src/go/src/net/dnsclient_unix.go:269 +0x20c

goroutine 42 [chan send, 34 minutes]:
github.com/aws/amazon-ecs-agent/agent/engine.func·012()
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_container_engine.go:636 +0x44b
created by github.com/aws/amazon-ecs-agent/agent/engine.(*dockerGoClient).ContainerEvents
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_container_engine.go:638 +0x4e4

goroutine 44 [chan send, 80 minutes]:
github.com/aws/amazon-ecs-agent/agent/engine.(*DockerTaskEngine).handleDockerEvents(0xc208052480, 0x7f1a5bf56838, 0xc20825f180)
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_task_engine.go:383 +0x819
created by github.com/aws/amazon-ecs-agent/agent/engine.(*DockerTaskEngine).Init
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_task_engine.go:131 +0x17a

goroutine 45 [chan receive, 2864 minutes]:
github.com/aws/amazon-ecs-agent/agent/sighandlers.StartTerminationHandler(0x7f1a5bf567d8, 0xc208046300, 0x7f1a5bf50e18, 0xc208052480)
	/go/src/github.com/aws/amazon-ecs-agent/agent/sighandlers/termination_handler.go:41 +0x18f
created by main._main
	/go/src/github.com/aws/amazon-ecs-agent/agent/agent.go:215 +0x208f

goroutine 46 [IO wait, 2863 minutes]:
net.(*pollDesc).Wait(0xc208296140, 0x72, 0x0, 0x0)
	/usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc208296140, 0x0, 0x0)
	/usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).accept(0xc2082960e0, 0x0, 0x7f1a5bf4ebe0, 0xc20829a138)
	/usr/src/go/src/net/fd_unix.go:419 +0x40b
net.(*TCPListener).AcceptTCP(0xc20804a070, 0x56c1a4, 0x0, 0x0)
	/usr/src/go/src/net/tcpsock_posix.go:234 +0x4e
net/http.tcpKeepAliveListener.Accept(0xc20804a070, 0x0, 0x0, 0x0, 0x0)
	/usr/src/go/src/net/http/server.go:1976 +0x4c
net/http.(*Server).Serve(0xc2080c9bc0, 0x7f1a5bf57978, 0xc20804a070, 0x0, 0x0)
	/usr/src/go/src/net/http/server.go:1728 +0x92
net/http.(*Server).ListenAndServe(0xc2080c9bc0, 0x0, 0x0)
	/usr/src/go/src/net/http/server.go:1718 +0x154
github.com/aws/amazon-ecs-agent/agent/handlers.func·005(0x0, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/handlers/v1_handlers.go:217 +0x42
github.com/aws/amazon-ecs-agent/agent/utils.RetryWithBackoff(0x7f1a5bf4f980, 0xc20828c210, 0xc2080aaf40, 0x0, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/utils/utils.go:110 +0x48
github.com/aws/amazon-ecs-agent/agent/handlers.ServeHttp(0xc2080a5190, 0x7f1a5bf50e18, 0xc208052480, 0xc2080a0c60)
	/go/src/github.com/aws/amazon-ecs-agent/agent/handlers/v1_handlers.go:222 +0x29b
created by main._main
	/go/src/github.com/aws/amazon-ecs-agent/agent/agent.go:218 +0x20d3

goroutine 47 [select, 257 minutes]:
github.com/aws/amazon-ecs-agent/agent/eventhandler.HandleEngineEvents(0x7f1a5bf50e18, 0xc208052480, 0x7f1a5bf51858, 0xc2080a2700, 0x7f1a5bf567d8, 0xc208046300)
	/go/src/github.com/aws/amazon-ecs-agent/agent/eventhandler/handler.go:35 +0x2f1
created by main._main
	/go/src/github.com/aws/amazon-ecs-agent/agent/agent.go:221 +0x2165

goroutine 48 [IO wait]:
net.(*pollDesc).Wait(0xc2082c9480, 0x72, 0x0, 0x0)
	/usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc2082c9480, 0x0, 0x0)
	/usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc2082c9420, 0xc2080e6000, 0x400, 0x400, 0x0, 0x7f1a5bf4ebe0, 0xc20821a4e8)
	/usr/src/go/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc20804a2f8, 0xc2080e6000, 0x400, 0x400, 0x0, 0x0, 0x0)
	/usr/src/go/src/net/net.go:121 +0xdc
crypto/tls.(*block).readFromUntil(0xc2082dd260, 0x7f1a5bf50aa0, 0xc20804a2f8, 0x5, 0x0, 0x0)
	/usr/src/go/src/crypto/tls/conn.go:454 +0xe6
crypto/tls.(*Conn).readRecord(0xc2080b5080, 0x17, 0x0, 0x0)
	/usr/src/go/src/crypto/tls/conn.go:539 +0x2da
crypto/tls.(*Conn).Read(0xc2080b5080, 0xc208320000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/src/go/src/crypto/tls/conn.go:904 +0x166
bufio.(*Reader).fill(0xc20833f740)
	/usr/src/go/src/bufio/bufio.go:97 +0x1ce
bufio.(*Reader).Read(0xc20833f740, 0xc20821a4e0, 0x2, 0x8, 0x0, 0x0, 0x0)
	/usr/src/go/src/bufio/bufio.go:174 +0x26c
github.com/gorilla/websocket.(*Conn).readFull(0xc2080c6780, 0xc20821a4e0, 0x2, 0x8, 0x0, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/gorilla/websocket/conn.go:524 +0xab
github.com/gorilla/websocket.(*Conn).advanceFrame(0xc2080c6780, 0xc208203700, 0x0, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/gorilla/websocket/conn.go:548 +0x162
github.com/gorilla/websocket.(*Conn).NextReader(0xc2080c6780, 0xc20804a3c8, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/gorilla/websocket/conn.go:688 +0x7e
github.com/gorilla/websocket.(*Conn).ReadMessage(0xc2080c6780, 0xc208229200, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/gorilla/websocket/conn.go:751 +0x6a
github.com/aws/amazon-ecs-agent/agent/wsclient.(*ClientServerImpl).ConsumeMessages(0xc20831a9c8, 0x0, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/wsclient/client.go:212 +0x7a
github.com/aws/amazon-ecs-agent/agent/tcs/client.(*clientServer).Serve(0xc20831a9a0, 0x0, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/tcs/client/client.go:86 +0x1cc
github.com/aws/amazon-ecs-agent/agent/tcs/handler.startSession(0xc2082c42a0, 0xd4, 0xc2080a4930, 0xc, 0xc20804c2d0, 0x0, 0x7f1a5bf579d8, 0xc208248aa0, 0x4a817c800, 0x0, ...)
	/go/src/github.com/aws/amazon-ecs-agent/agent/tcs/handler/handler.go:119 +0x3ce
github.com/aws/amazon-ecs-agent/agent/tcs/handler.startTelemetrySession(0xc2080c8fc0, 0x5d, 0xc20804c2d0, 0xc2080a0c60, 0x0, 0x7f1a5bf51858, 0xc2080a2700, 0x7f1a5bf50e18, 0xc208052480, 0x7f1a5bf579d8, ...)
	/go/src/github.com/aws/amazon-ecs-agent/agent/tcs/handler/handler.go:94 +0x379
github.com/aws/amazon-ecs-agent/agent/tcs/handler.StartSession(0xc2080c8fc0, 0x5d, 0xc20804c2d0, 0xc2080a0c60, 0x0, 0x7f1a5bf51858, 0xc2080a2700, 0x7f1a5bf50e18, 0xc208052480, 0x7f1a5bf579d8, ...)
	/go/src/github.com/aws/amazon-ecs-agent/agent/tcs/handler/handler.go:76 +0xbf
github.com/aws/amazon-ecs-agent/agent/tcs/handler.StartMetricsSession(0xc2080c8fc0, 0x5d, 0xc20804c2d0, 0xc2080a0c60, 0x0, 0x7f1a5bf51858, 0xc2080a2700, 0x7f1a5bf50e18, 0xc208052480)
	/go/src/github.com/aws/amazon-ecs-agent/agent/tcs/handler/handler.go:59 +0x5a2
created by main._main
	/go/src/github.com/aws/amazon-ecs-agent/agent/agent.go:233 +0x222d

goroutine 49 [semacquire, 41 minutes]:
sync.(*Mutex).Lock(0xc2080524f8)
	/usr/src/go/src/sync/mutex.go:66 +0xd3
sync.(*RWMutex).Lock(0xc2080524f8)
	/usr/src/go/src/sync/rwmutex.go:82 +0x33
github.com/aws/amazon-ecs-agent/agent/engine.(*DockerTaskEngine).AddTask(0xc208052480, 0xc20831a580, 0x0, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_task_engine.go:400 +0x58
github.com/aws/amazon-ecs-agent/agent/acs/handler.addStoppedTasks(0xc20804a568, 0x1, 0x1, 0x7f1a5bf50e18, 0xc208052480, 0x1)
	/go/src/github.com/aws/amazon-ecs-agent/agent/acs/handler/acs_handler.go:262 +0xad
github.com/aws/amazon-ecs-agent/agent/acs/handler.addPayloadTasks(0x7f1a5bf51858, 0xc2080a2700, 0x7ffc9bfa8e8e, 0x22, 0xc2080c8fc0, 0x5d, 0xc2082ec4c0, 0x7f1a5bf50e18, 0xc208052480, 0x1)
	/go/src/github.com/aws/amazon-ecs-agent/agent/acs/handler/acs_handler.go:248 +0x2ba
github.com/aws/amazon-ecs-agent/agent/acs/handler.handlePayloadMessage(0xc2080c9560, 0x7ffc9bfa8e8e, 0x22, 0xc2080c8fc0, 0x5d, 0xc2082ec4c0, 0x7f1a5bf50e18, 0xc208052480, 0x7f1a5bf51858, 0xc2080a2700, ...)
	/go/src/github.com/aws/amazon-ecs-agent/agent/acs/handler/acs_handler.go:202 +0x24e
github.com/aws/amazon-ecs-agent/agent/acs/handler.func·001()
	/go/src/github.com/aws/amazon-ecs-agent/agent/acs/handler/acs_handler.go:91 +0x1ce
created by github.com/aws/amazon-ecs-agent/agent/acs/handler.StartSession
	/go/src/github.com/aws/amazon-ecs-agent/agent/acs/handler/acs_handler.go:96 +0x367

goroutine 50 [select]:
github.com/fsouza/go-dockerclient.(*eventMonitoringState).monitorEvents(0xc2082961c0, 0xc20826a000)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/event.go:176 +0x419
created by github.com/fsouza/go-dockerclient.(*eventMonitoringState).enableEventMonitoring
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/event.go:143 +0x16f

goroutine 51 [chan receive, 2863 minutes]:
github.com/aws/amazon-ecs-agent/agent/engine.func·011()
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_container_engine.go:554 +0x6e
created by github.com/aws/amazon-ecs-agent/agent/engine.(*dockerGoClient).ContainerEvents
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_container_engine.go:556 +0x3f7

goroutine 52 [chan receive, 9 minutes]:
github.com/aws/amazon-ecs-agent/agent/engine.func·012()
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_container_engine.go:561 +0x7c
created by github.com/aws/amazon-ecs-agent/agent/engine.(*dockerGoClient).ContainerEvents
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_container_engine.go:638 +0x4e4

goroutine 5695 [chan receive, 7 minutes]:
github.com/aws/amazon-ecs-agent/agent/engine.func·017()
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/task_manager.go:128 +0x48
created by github.com/aws/amazon-ecs-agent/agent/engine.(*managedTask).overseeTask
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/task_manager.go:130 +0x7de

goroutine 55 [semacquire, 9 minutes]:
sync.(*RWMutex).RLock(0xc2080922a0)
	/usr/src/go/src/sync/rwmutex.go:36 +0x5f
github.com/fsouza/go-dockerclient.(*eventMonitoringState).isEnabled(0xc2080922a0, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/event.go:219 +0x34
github.com/fsouza/go-dockerclient.func·008(0xc208053dd0, 0xc208264b00)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/event.go:298 +0x353
created by github.com/fsouza/go-dockerclient.(*Client).eventHijack
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/event.go:303 +0x625

goroutine 56 [runnable]:
net.(*pollDesc).Wait(0xc208296d10, 0x72, 0x0, 0x0)
	/usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc208296d10, 0x0, 0x0)
	/usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc208296cb0, 0xc208206000, 0x1000, 0x1000, 0x0, 0x7f1a5bf4ebe0, 0xc20829a788)
	/usr/src/go/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc20804a1b8, 0xc208206000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/src/go/src/net/net.go:121 +0xdc
bufio.(*Reader).fill(0xc2080c8cc0)
	/usr/src/go/src/bufio/bufio.go:97 +0x1ce
bufio.(*Reader).ReadSlice(0xc2080c8cc0, 0x7f1a5bf4a90a, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/src/go/src/bufio/bufio.go:295 +0x257
net/http/internal.readLine(0xc2080c8cc0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/src/go/src/net/http/internal/chunked.go:110 +0x5a
net/http/internal.(*chunkedReader).beginChunk(0xc20828cff0)
	/usr/src/go/src/net/http/internal/chunked.go:47 +0x46
net/http/internal.(*chunkedReader).Read(0xc20828cff0, 0xc208254001, 0x5ff, 0x5ff, 0x0, 0x0, 0x0)
	/usr/src/go/src/net/http/internal/chunked.go:77 +0xbb
net/http.(*body).readLocked(0xc20825f080, 0xc208254001, 0x5ff, 0x5ff, 0x5, 0x0, 0x0)
	/usr/src/go/src/net/http/transfer.go:619 +0x7a
net/http.(*body).Read(0xc20825f080, 0xc208254001, 0x5ff, 0x5ff, 0x0, 0x0, 0x0)
	/usr/src/go/src/net/http/transfer.go:614 +0x115
encoding/json.(*Decoder).readValue(0xc208076480, 0xc207fcdda3, 0x0, 0x0)
	/usr/src/go/src/encoding/json/stream.go:124 +0x5e1
encoding/json.(*Decoder).Decode(0xc208076480, 0x7f99e0, 0xc2083225c0, 0x0, 0x0)
	/usr/src/go/src/encoding/json/stream.go:44 +0x7b
github.com/fsouza/go-dockerclient.func·008(0xc208053950, 0xc208264630)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/event.go:285 +0x1b7
created by github.com/fsouza/go-dockerclient.(*Client).eventHijack
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/event.go:303 +0x625

goroutine 5670 [semacquire, 17 minutes]:
sync.(*RWMutex).RLock(0xc2080524f8)
	/usr/src/go/src/sync/rwmutex.go:36 +0x5f
github.com/aws/amazon-ecs-agent/agent/engine.(*DockerTaskEngine).CheckTaskState(0xc208052480, 0xc2082649a0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_task_engine.go:262 +0x32a
created by github.com/aws/amazon-ecs-agent/agent/engine.(*managedTask).overseeTask
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/task_manager.go:135 +0x87c

goroutine 5591 [semacquire, 27 minutes]:
sync.(*RWMutex).RLock(0xc2080524f8)
	/usr/src/go/src/sync/rwmutex.go:36 +0x5f
github.com/aws/amazon-ecs-agent/agent/engine.(*DockerTaskEngine).CheckTaskState(0xc208052480, 0xc2082649a0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_task_engine.go:262 +0x32a
created by github.com/aws/amazon-ecs-agent/agent/engine.(*managedTask).overseeTask
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/task_manager.go:135 +0x87c

goroutine 61 [select, 9 minutes]:
github.com/aws/amazon-ecs-agent/agent/stats.(*DockerStatsEngine).handleDockerEvents(0xc208248aa0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/stats/engine.go:248 +0x689
created by github.com/aws/amazon-ecs-agent/agent/stats.(*DockerStatsEngine).listContainersAndStartEventHandler
	/go/src/github.com/aws/amazon-ecs-agent/agent/stats/engine.go:142 +0x1b2

goroutine 5667 [chan send, 19 minutes]:
github.com/aws/amazon-ecs-agent/agent/acs/handler.func·008(0xc2083620c0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/acs/handler/acs_handler.go:192 +0x4a
reflect.Value.call(0x827ee0, 0xc208262950, 0x13, 0x9b4470, 0x4, 0xc208119df8, 0x1, 0x1, 0x0, 0x0, ...)
	/usr/src/go/src/reflect/value.go:419 +0x10e5
reflect.Value.Call(0x827ee0, 0xc208262950, 0x13, 0xc208119df8, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/src/go/src/reflect/value.go:296 +0xbc
github.com/aws/amazon-ecs-agent/agent/wsclient.(*ClientServerImpl).handleMessage(0xc2080f6120, 0xc208346000, 0x4af, 0xe00)
	/go/src/github.com/aws/amazon-ecs-agent/agent/wsclient/client.go:277 +0x53c
github.com/aws/amazon-ecs-agent/agent/wsclient.(*ClientServerImpl).ConsumeMessages(0xc2080f6120, 0x0, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/wsclient/client.go:229 +0x80b
github.com/aws/amazon-ecs-agent/agent/acs/client.(*clientServer).Serve(0xc2080f6120, 0x0, 0x0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/acs/client/acs_client.go:59 +0x162
github.com/aws/amazon-ecs-agent/agent/acs/handler.func·005()
	/go/src/github.com/aws/amazon-ecs-agent/agent/acs/handler/acs_handler.go:147 +0x42
created by github.com/aws/amazon-ecs-agent/agent/acs/handler.func·006
	/go/src/github.com/aws/amazon-ecs-agent/agent/acs/handler/acs_handler.go:148 +0x977

goroutine 5694 [semacquire, 7 minutes]:
sync.(*RWMutex).RLock(0xc2080524f8)
	/usr/src/go/src/sync/rwmutex.go:36 +0x5f
github.com/aws/amazon-ecs-agent/agent/engine.(*DockerTaskEngine).CheckTaskState(0xc208052480, 0xc2082649a0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_task_engine.go:262 +0x32a
created by github.com/aws/amazon-ecs-agent/agent/engine.(*managedTask).overseeTask
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/task_manager.go:135 +0x87c

goroutine 5448 [semacquire, 77 minutes]:
sync.(*RWMutex).RLock(0xc2080524f8)
	/usr/src/go/src/sync/rwmutex.go:36 +0x5f
github.com/aws/amazon-ecs-agent/agent/engine.(*DockerTaskEngine).CheckTaskState(0xc208052480, 0xc2082649a0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_task_engine.go:262 +0x32a
created by github.com/aws/amazon-ecs-agent/agent/engine.(*managedTask).overseeTask
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/task_manager.go:135 +0x87c

goroutine 5702 [IO wait]:
net.(*pollDesc).Wait(0xc2082c8450, 0x72, 0x0, 0x0)
	/usr/src/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc2082c8450, 0x0, 0x0)
	/usr/src/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc2082c83f0, 0xc20811b400, 0x400, 0x400, 0x0, 0x7f1a5bf4ebe0, 0xc20831e5c8)
	/usr/src/go/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc20804a190, 0xc20811b400, 0x400, 0x400, 0x0, 0x0, 0x0)
	/usr/src/go/src/net/net.go:121 +0xdc
crypto/tls.(*block).readFromUntil(0xc20828cf90, 0x7f1a5bf50aa0, 0xc20804a190, 0x5, 0x0, 0x0)
	/usr/src/go/src/crypto/tls/conn.go:454 +0xe6
crypto/tls.(*Conn).readRecord(0xc2080b4840, 0x17, 0x0, 0x0)
	/usr/src/go/src/crypto/tls/conn.go:539 +0x2da
crypto/tls.(*Conn).Read(0xc2080b4840, 0xc2080b3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/src/go/src/crypto/tls/conn.go:904 +0x166
net/http.noteEOFReader.Read(0x7f1a5bf566b0, 0xc2080b4840, 0xc20831a108, 0xc2080b3000, 0x1000, 0x1000, 0x8a28c0, 0x0, 0x0)
	/usr/src/go/src/net/http/transport.go:1270 +0x6e
net/http.(*noteEOFReader).Read(0xc20824ebe0, 0xc2080b3000, 0x1000, 0x1000, 0xc208012000, 0x0, 0x0)
	<autogenerated>:125 +0xd4
bufio.(*Reader).fill(0xc20833e3c0)
	/usr/src/go/src/bufio/bufio.go:97 +0x1ce
bufio.(*Reader).Peek(0xc20833e3c0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/src/go/src/bufio/bufio.go:132 +0xf0
net/http.(*persistConn).readLoop(0xc20831a0b0)
	/usr/src/go/src/net/http/transport.go:842 +0xa4
created by net/http.(*Transport).dialConn
	/usr/src/go/src/net/http/transport.go:660 +0xc9f

goroutine 5703 [select]:
net/http.(*persistConn).writeLoop(0xc20831a0b0)
	/usr/src/go/src/net/http/transport.go:945 +0x41d
created by net/http.(*Transport).dialConn
	/usr/src/go/src/net/http/transport.go:661 +0xcbc

goroutine 5083 [select, 7 minutes]:
github.com/aws/amazon-ecs-agent/agent/engine.(*managedTask).waitEvent(0xc208259110, 0xc2082c95e0, 0xc2082c95e0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/task_manager.go:284 +0x735
github.com/aws/amazon-ecs-agent/agent/engine.(*managedTask).overseeTask(0xc208259110)
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/task_manager.go:131 +0x7fe
created by github.com/aws/amazon-ecs-agent/agent/engine.(*DockerTaskEngine).startTask
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_task_engine.go:317 +0x199

goroutine 5451 [semacquire, 67 minutes]:
sync.(*RWMutex).RLock(0xc2080524f8)
	/usr/src/go/src/sync/rwmutex.go:36 +0x5f
github.com/aws/amazon-ecs-agent/agent/engine.(*DockerTaskEngine).CheckTaskState(0xc208052480, 0xc2082649a0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_task_engine.go:262 +0x32a
created by github.com/aws/amazon-ecs-agent/agent/engine.(*managedTask).overseeTask
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/task_manager.go:135 +0x87c

goroutine 5484 [semacquire, 47 minutes]:
sync.(*RWMutex).RLock(0xc2080524f8)
	/usr/src/go/src/sync/rwmutex.go:36 +0x5f
github.com/aws/amazon-ecs-agent/agent/engine.(*DockerTaskEngine).CheckTaskState(0xc208052480, 0xc2082649a0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_task_engine.go:262 +0x32a
created by github.com/aws/amazon-ecs-agent/agent/engine.(*managedTask).overseeTask
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/task_manager.go:135 +0x87c

goroutine 5481 [semacquire, 57 minutes]:
sync.(*RWMutex).RLock(0xc2080524f8)
	/usr/src/go/src/sync/rwmutex.go:36 +0x5f
github.com/aws/amazon-ecs-agent/agent/engine.(*DockerTaskEngine).CheckTaskState(0xc208052480, 0xc2082649a0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_task_engine.go:262 +0x32a
created by github.com/aws/amazon-ecs-agent/agent/engine.(*managedTask).overseeTask
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/task_manager.go:135 +0x87c

goroutine 5708 [select]:
github.com/aws/amazon-ecs-agent/agent/tcs/client.(*clientServer).publishMetrics(0xc20831a9a0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/tcs/client/client.go:148 +0x162
created by github.com/aws/amazon-ecs-agent/agent/tcs/client.(*clientServer).Serve
	/go/src/github.com/aws/amazon-ecs-agent/agent/tcs/client/client.go:84 +0x1b0

goroutine 414 [semacquire, 80 minutes]:
sync.(*RWMutex).Lock(0xc2080524f8)
	/usr/src/go/src/sync/rwmutex.go:87 +0xa4
github.com/aws/amazon-ecs-agent/agent/engine.(*managedTask).cleanupTask(0xc208258450, 0x9d29229e000)
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/task_manager.go:441 +0x467
github.com/aws/amazon-ecs-agent/agent/engine.(*managedTask).overseeTask(0xc208258450)
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/task_manager.go:164 +0xd13
created by github.com/aws/amazon-ecs-agent/agent/engine.(*DockerTaskEngine).startTask
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_task_engine.go:317 +0x199

goroutine 5513 [semacquire, 37 minutes]:
sync.(*RWMutex).RLock(0xc2080524f8)
	/usr/src/go/src/sync/rwmutex.go:36 +0x5f
github.com/aws/amazon-ecs-agent/agent/engine.(*DockerTaskEngine).CheckTaskState(0xc208052480, 0xc2082649a0)
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/docker_task_engine.go:262 +0x32a
created by github.com/aws/amazon-ecs-agent/agent/engine.(*managedTask).overseeTask
	/go/src/github.com/aws/amazon-ecs-agent/agent/engine/task_manager.go:135 +0x87c

goroutine 5548 [chan send, 34 minutes]:
github.com/fsouza/go-dockerclient.(*eventMonitoringState).sendEvent(0xc2080922a0, 0xc208108e40)
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/event.go:236 +0x16c
created by github.com/fsouza/go-dockerclient.(*eventMonitoringState).monitorEvents
	/go/src/github.com/aws/amazon-ecs-agent/agent/Godeps/_workspace/src/github.com/fsouza/go-dockerclient/event.go:186 +0x267

====== /STACKTRACE ======