Skip to content

Instantly share code, notes, and snippets.

@kumarmj

kumarmj/gist:89e9b213360cce1eade7b0b9bddddd71

Created August 3, 2018 05:48
Show Gist options
  • Save kumarmj/89e9b213360cce1eade7b0b9bddddd71 to your computer and use it in GitHub Desktop.
Save kumarmj/89e9b213360cce1eade7b0b9bddddd71 to your computer and use it in GitHub Desktop.
Download ZIP
cromwell log AWS backend behind proxy
Raw
gistfile1.txt
Manoj
```[2018-08-03 15:41:02,73] [error] Credentials are invalid: Unable to execute HTTP request: Connect to sts.amazonaws.com:443 [sts.amazonaws.com/54.239.29.25] failed: connect timed out
java.lang.RuntimeException: Credentials are invalid: Unable to execute HTTP request: Connect to sts.amazonaws.com:443 [sts.amazonaws.com/54.239.29.25] failed: connect timed out
at cromwell.cloudsupport.aws.auth.AwsAuthMode.validateCredential(AwsAuthMode.scala:87)
at cromwell.cloudsupport.aws.auth.AwsAuthMode.validateCredential$(AwsAuthMode.scala:85)
at cromwell.cloudsupport.aws.auth.DefaultMode.validateCredential(AwsAuthMode.scala:118)
at cromwell.cloudsupport.aws.auth.DefaultMode._credential$lzycompute(AwsAuthMode.scala:129)
at cromwell.cloudsupport.aws.auth.DefaultMode._credential(AwsAuthMode.scala:119)
at cromwell.cloudsupport.aws.auth.DefaultMode.credential(AwsAuthMode.scala:132)
at cromwell.filesystems.s3.S3PathBuilder$.fromAuthMode(S3PathBuilder.scala:119)
at cromwell.filesystems.s3.S3PathBuilderFactory.withOptions(S3PathBuilderFactory.scala:58)
at cromwell.core.path.PathBuilderFactory$.$anonfun$instantiatePathBuilders$2(PathBuilderFactory.scala:23)
at cats.instances.ListInstances$$anon$1.$anonfun$traverse$2(list.scala:73)
at cats.instances.ListInstances$$anon$1.loop$2(list.scala:63)
at cats.instances.ListInstances$$anon$1.$anonfun$foldRight$2(list.scala:65)
at cats.Eval$.advance(Eval.scala:272)
at cats.Eval$.loop$1(Eval.scala:354)
at cats.Eval$.cats$Eval$$evaluate(Eval.scala:372)
at cats.Eval$Defer.value(Eval.scala:258)
at cats.instances.ListInstances$$anon$1.traverse(list.scala:72)
at cats.instances.ListInstances$$anon$1.traverse(list.scala:12)
at cats.Traverse$Ops.traverse(Traverse.scala:19)
at cats.Traverse$Ops.traverse$(Traverse.scala:19)
at cats.Traverse$ToTraverseOps$$anon$3.traverse(Traverse.scala:19)
at cromwell.core.path.PathBuilderFactory$.instantiatePathBuilders(PathBuilderFactory.scala:23)
at cromwell.engine.EngineFilesystems$.pathBuildersForWorkflow(EngineFilesystems.scala:27)
at cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor.$anonfun$workflowOptionsAndPathBuilders$1(MaterializeWorkflowDescriptorActor.scala:209)
at cats.data.Validated.map(Validated.scala:194)
at cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor.cromwell$engine$workflow$lifecycle$materialization$MaterializeWorkflowDescriptorActor$$workflowOptionsAndPathBuilders(MaterializeWorkflowDescriptorActor.scala:208)
at cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor$$anonfun$1.applyOrElse(MaterializeWorkflowDescriptorActor.scala:142)
at cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor$$anonfun$1.applyOrElse(MaterializeWorkflowDescriptorActor.scala:138)
at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:34)
at akka.actor.FSM.processEvent(FSM.scala:670)
at akka.actor.FSM.processEvent$(FSM.scala:667)
at cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor.akka$actor$LoggingFSM$$super$processEvent(MaterializeWorkflowDescriptorActor.scala:123)
at akka.actor.LoggingFSM.processEvent(FSM.scala:806)
at akka.actor.LoggingFSM.processEvent$(FSM.scala:788)
at cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor.processEvent(MaterializeWorkflowDescriptorActor.scala:123)
at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:664)
at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:658)
at akka.actor.Actor.aroundReceive(Actor.scala:517)
at akka.actor.Actor.aroundReceive$(Actor.scala:515)
at cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor.aroundReceive(MaterializeWorkflowDescriptorActor.scala:123)
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:588)
at akka.actor.ActorCell.invoke(ActorCell.scala:557)
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:258)
at akka.dispatch.Mailbox.run(Mailbox.scala:225)
at akka.dispatch.Mailbox.exec(Mailbox.scala:235)
at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
Caused by: software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Connect to sts.amazonaws.com:443 [sts.amazonaws.com/54.239.29.25] failed: connect timed out
at software.amazon.awssdk.core.http.pipeline.stages.RetryableStage$RetryExecutor.handleThrownException(RetryableStage.java:160)
at software.amazon.awssdk.core.http.pipeline.stages.RetryableStage$RetryExecutor.execute(RetryableStage.java:117)
at software.amazon.awssdk.core.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:66)
at software.amazon.awssdk.core.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:47)
at software.amazon.awssdk.core.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:239)
at software.amazon.awssdk.core.http.StreamManagingStage.execute(StreamManagingStage.java:56)
at software.amazon.awssdk.core.http.StreamManagingStage.execute(StreamManagingStage.java:42)
at software.amazon.awssdk.core.http.pipeline.stages.ClientExecutionTimedStage.executeWithTimer(ClientExecutionTimedStage.java:71)
at software.amazon.awssdk.core.http.pipeline.stages.ClientExecutionTimedStage.execute(ClientExecutionTimedStage.java:55)
at software.amazon.awssdk.core.http.pipeline.stages.ClientExecutionTimedStage.execute(ClientExecutionTimedStage.java:39)
at software.amazon.awssdk.core.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:239)
at software.amazon.awssdk.core.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:239)
at software.amazon.awssdk.core.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:35)
at software.amazon.awssdk.core.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:24)
at software.amazon.awssdk.core.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:281)
at software.amazon.awssdk.core.client.SyncClientHandlerImpl.doInvoke(SyncClientHandlerImpl.java:149)
at software.amazon.awssdk.core.client.SyncClientHandlerImpl.invoke(SyncClientHandlerImpl.java:131)
at software.amazon.awssdk.core.client.SyncClientHandlerImpl.execute(SyncClientHandlerImpl.java:100)
at software.amazon.awssdk.core.client.SyncClientHandlerImpl.execute(SyncClientHandlerImpl.java:76)
at software.amazon.awssdk.core.client.SdkClientHandler.execute(SdkClientHandler.java:45)
at software.amazon.awssdk.services.sts.DefaultSTSClient.getCallerIdentity(DefaultSTSClient.java:662)
at cromwell.cloudsupport.aws.auth.AwsAuthMode.$anonfun$credentialValidation$1(AwsAuthMode.scala:81)
at cromwell.cloudsupport.aws.auth.AwsAuthMode.$anonfun$credentialValidation$1$adapted(AwsAuthMode.scala:75)
at cromwell.cloudsupport.aws.auth.AwsAuthMode.$anonfun$validateCredential$1(AwsAuthMode.scala:86)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
at scala.util.Try$.apply(Try.scala:209)
at cromwell.cloudsupport.aws.auth.AwsAuthMode.validateCredential(AwsAuthMode.scala:86)
... 48 common frames omitted
Caused by: org.apache.http.conn.ConnectTimeoutException: Connect to sts.amazonaws.com:443 [sts.amazonaws.com/54.239.29.25] failed: connect timed out
at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:151)
at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:359)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at software.amazon.awssdk.http.apache.internal.conn.ClientConnectionManagerFactory$Handler.invoke(ClientConnectionManagerFactory.java:78)
at com.sun.proxy.$Proxy9.connect(Unknown Source)
at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:381)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:237)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at software.amazon.awssdk.http.apache.internal.impl.ApacheSdkHttpClient.execute(ApacheSdkHttpClient.java:70)
at software.amazon.awssdk.http.apache.ApacheHttpClient.execute(ApacheHttpClient.java:90)
at software.amazon.awssdk.http.apache.ApacheHttpClient.access$000(ApacheHttpClient.java:47)
at software.amazon.awssdk.http.apache.ApacheHttpClient$1.call(ApacheHttpClient.java:68)
at software.amazon.awssdk.http.apache.ApacheHttpClient$1.call(ApacheHttpClient.java:65)
at software.amazon.awssdk.core.http.pipeline.stages.MakeHttpRequestStage.executeHttpRequest(MakeHttpRequestStage.java:56)
at software.amazon.awssdk.core.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:47)
at software.amazon.awssdk.core.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:32)
at software.amazon.awssdk.core.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:239)
at software.amazon.awssdk.core.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:239)
at software.amazon.awssdk.core.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:239)
at software.amazon.awssdk.core.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:239)
at software.amazon.awssdk.core.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:239)
at software.amazon.awssdk.core.http.pipeline.stages.TimerExceptionHandlingStage.execute(TimerExceptionHandlingStage.java:40)
at software.amazon.awssdk.core.http.pipeline.stages.TimerExceptionHandlingStage.execute(TimerExceptionHandlingStage.java:30)
at software.amazon.awssdk.core.http.pipeline.stages.RetryableStage$RetryExecutor.doExecute(RetryableStage.java:139)
at software.amazon.awssdk.core.http.pipeline.stages.RetryableStage$RetryExecutor.execute(RetryableStage.java:105)
... 73 common frames omitted
Caused by: java.net.SocketTimeoutException: connect timed out
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:339)
at software.amazon.awssdk.http.apache.internal.conn.SdkTlsSocketFactory.connectSocket(SdkTlsSocketFactory.java:118)
at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142)
... 103 common frames omitted
[2018-08-03 15:41:02,73] [error] WorkflowManagerActor Workflow 96867c0d-a7fd-4786-bbcb-469eb5503004 failed (during MaterializingWorkflowDescriptorState): java.lang.RuntimeException: Credentials are invalid: Unable to execute HTTP request: Connect to sts.amazonaws.com:443 [sts.amazonaws.com/54.239.29.25] failed: connect timed out
at cromwell.cloudsupport.aws.auth.AwsAuthMode.validateCredential(AwsAuthMode.scala:87)
at cromwell.cloudsupport.aws.auth.AwsAuthMode.validateCredential$(AwsAuthMode.scala:85)
at cromwell.cloudsupport.aws.auth.DefaultMode.validateCredential(AwsAuthMode.scala:118)
at cromwell.cloudsupport.aws.auth.DefaultMode._credential$lzycompute(AwsAuthMode.scala:129)
at cromwell.cloudsupport.aws.auth.DefaultMode._credential(AwsAuthMode.scala:119)
at cromwell.cloudsupport.aws.auth.DefaultMode.credential(AwsAuthMode.scala:132)
at cromwell.filesystems.s3.S3PathBuilder$.fromAuthMode(S3PathBuilder.scala:119)
at cromwell.filesystems.s3.S3PathBuilderFactory.withOptions(S3PathBuilderFactory.scala:58)
at cromwell.core.path.PathBuilderFactory$.$anonfun$instantiatePathBuilders$2(PathBuilderFactory.scala:23)
at cats.instances.ListInstances$$anon$1.$anonfun$traverse$2(list.scala:73)
at cats.instances.ListInstances$$anon$1.loop$2(list.scala:63)
at cats.instances.ListInstances$$anon$1.$anonfun$foldRight$2(list.scala:65)
at cats.Eval$.advance(Eval.scala:272)
at cats.Eval$.loop$1(Eval.scala:354)
at cats.Eval$.cats$Eval$$evaluate(Eval.scala:372)
at cats.Eval$Defer.value(Eval.scala:258)
at cats.instances.ListInstances$$anon$1.traverse(list.scala:72)
at cats.instances.ListInstances$$anon$1.traverse(list.scala:12)
at cats.Traverse$Ops.traverse(Traverse.scala:19)
at cats.Traverse$Ops.traverse$(Traverse.scala:19)
at cats.Traverse$ToTraverseOps$$anon$3.traverse(Traverse.scala:19)
at cromwell.core.path.PathBuilderFactory$.instantiatePathBuilders(PathBuilderFactory.scala:23)
at cromwell.engine.EngineFilesystems$.pathBuildersForWorkflow(EngineFilesystems.scala:27)
at cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor.$anonfun$workflowOptionsAndPathBuilders$1(MaterializeWorkflowDescriptorActor.scala:209)
at cats.data.Validated.map(Validated.scala:194)
at cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor.cromwell$engine$workflow$lifecycle$materialization$MaterializeWorkflowDescriptorActor$$workflowOptionsAndPathBuilders(MaterializeWorkflowDescriptorActor.scala:208)
at cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor$$anonfun$1.applyOrElse(MaterializeWorkflowDescriptorActor.scala:142)
at cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor$$anonfun$1.applyOrElse(MaterializeWorkflowDescriptorActor.scala:138)
at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:34)
at akka.actor.FSM.processEvent(FSM.scala:670)
at akka.actor.FSM.processEvent$(FSM.scala:667)
at cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor.akka$actor$LoggingFSM$$super$processEvent(MaterializeWorkflowDescriptorActor.scala:123)
at akka.actor.LoggingFSM.processEvent(FSM.scala:806)
at akka.actor.LoggingFSM.processEvent$(FSM.scala:788)
at cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor.processEvent(MaterializeWorkflowDescriptorActor.scala:123)
at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:664)
at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:658)
at akka.actor.Actor.aroundReceive(Actor.scala:517)
at akka.actor.Actor.aroundReceive$(Actor.scala:515)
at cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor.aroundReceive(MaterializeWorkflowDescriptorActor.scala:123)
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:588)
at akka.actor.ActorCell.invoke(ActorCell.scala:557)
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:258)
at akka.dispatch.Mailbox.run(Mailbox.scala:225)
at akka.dispatch.Mailbox.exec(Mailbox.scala:235)
at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
Caused by: software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Connect to sts.amazonaws.com:443 [sts.amazonaws.com/54.239.29.25] failed: connect timed out
at software.amazon.awssdk.core.http.pipeline.stages.RetryableStage$RetryExecutor.handleThrownException(RetryableStage.java:160)
at software.amazon.awssdk.core.http.pipeline.stages.RetryableStage$RetryExecutor.execute(RetryableStage.java:117)
at software.amazon.awssdk.core.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:66)
at software.amazon.awssdk.core.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:47)
at software.amazon.awssdk.core.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:239)
at software.amazon.awssdk.core.http.StreamManagingStage.execute(StreamManagingStage.java:56)
at software.amazon.awssdk.core.http.StreamManagingStage.execute(StreamManagingStage.java:42)
at software.amazon.awssdk.core.http.pipeline.stages.ClientExecutionTimedStage.executeWithTimer(ClientExecutionTimedStage.java:71)
at software.amazon.awssdk.core.http.pipeline.stages.ClientExecutionTimedStage.execute(ClientExecutionTimedStage.java:55)
at software.amazon.awssdk.core.http.pipeline.stages.ClientExecutionTimedStage.execute(ClientExecutionTimedStage.java:39)
at software.amazon.awssdk.core.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:239)
at software.amazon.awssdk.core.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:239)
at software.amazon.awssdk.core.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:35)
at software.amazon.awssdk.core.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:24)
at software.amazon.awssdk.core.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:281)
at software.amazon.awssdk.core.client.SyncClientHandlerImpl.doInvoke(SyncClientHandlerImpl.java:149)
at software.amazon.awssdk.core.client.SyncClientHandlerImpl.invoke(SyncClientHandlerImpl.java:131)
at software.amazon.awssdk.core.client.SyncClientHandlerImpl.execute(SyncClientHandlerImpl.java:100)
at software.amazon.awssdk.core.client.SyncClientHandlerImpl.execute(SyncClientHandlerImpl.java:76)
at software.amazon.awssdk.core.client.SdkClientHandler.execute(SdkClientHandler.java:45)
at software.amazon.awssdk.services.sts.DefaultSTSClient.getCallerIdentity(DefaultSTSClient.java:662)
at cromwell.cloudsupport.aws.auth.AwsAuthMode.$anonfun$credentialValidation$1(AwsAuthMode.scala:81)
at cromwell.cloudsupport.aws.auth.AwsAuthMode.$anonfun$credentialValidation$1$adapted(AwsAuthMode.scala:75)
at cromwell.cloudsupport.aws.auth.AwsAuthMode.$anonfun$validateCredential$1(AwsAuthMode.scala:86)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
at scala.util.Try$.apply(Try.scala:209)
at cromwell.cloudsupport.aws.auth.AwsAuthMode.validateCredential(AwsAuthMode.scala:86)
... 48 more
Caused by: org.apache.http.conn.ConnectTimeoutException: Connect to sts.amazonaws.com:443 [sts.amazonaws.com/54.239.29.25] failed: connect timed out
```
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment