diff --git a/src/host/rpc_connections.h b/src/host/rpc_connections.h
index e25c8dc7fe41..541ea198f910 100644
--- a/src/host/rpc_connections.h
+++ b/src/host/rpc_connections.h
@@ -345,6 +345,8 @@ namespace asynchost
       // Invalidating the TCP socket will result in the handle being closed. No
       // more messages will be read from or written to the TCP socket.
       sockets[id] = nullptr;
+      idle_times.erase(id);
+
       RINGBUFFER_WRITE_MESSAGE(::tcp::tcp_close, to_enclave, id);
 
       return true;