Fix handling of connection errors during startup (CloudBotIRC#234)

Author: linuxdaemon

cloudbot/bot.py

@@ -166,8 +166,11 @@ def run(self):
         # Initializes the bot, plugins and connections
         self.loop.run_until_complete(self._init_routine())
         # Wait till the bot stops. The stopped_future will be set to True to restart, False otherwise
+        logger.debug("Init done")
         restart = self.loop.run_until_complete(self.stopped_future)
+        logger.debug("Waiting for plugin unload")
         self.loop.run_until_complete(self.plugin_manager.unload_all())
+        logger.debug("Unload complete")
         self.loop.close()
         return restart
 
@@ -197,21 +200,34 @@ def stop(self, reason=None, *, restart=False):
 
         self.observer.stop()
 
+        logger.debug("Stopping connect loops and shutting down clients")
         for connection in self.connections.values():
+            connection.active = False
+            if not connection.cancelled_future.done():
+                connection.cancelled_future.set_result(None)
+
             if not connection.connected:
                 # Don't quit a connection that hasn't connected
                 continue
             logger.debug("[{}] Closing connection.".format(connection.name))
 
             connection.quit(reason)
 
+        logger.debug("Done.")
+
+        logger.debug("Sleeping to let clients quit")
         yield from asyncio.sleep(1.0)  # wait for 'QUIT' calls to take affect
+        logger.debug("Sleep done.")
 
+        logger.debug("Closing clients")
         for connection in self.connections.values():
             connection.close()
 
+        logger.debug("All clients closed")
+
         self.running = False
         # Give the stopped_future a result, so that run() will exit
+        logger.debug("Setting future result for shutdown")
         self.stopped_future.set_result(restart)
 
     @asyncio.coroutine
@@ -238,8 +254,12 @@ def _init_routine(self):
 
         self.observer.start()
 
+        for conn in self.connections.values():
+            conn.active = True
+
         # Connect to servers
-        yield from asyncio.gather(*[conn.connect() for conn in self.connections.values()], loop=self.loop)
+        yield from asyncio.gather(*[conn.try_connect() for conn in self.connections.values()], loop=self.loop)
+        logger.debug("Connections created.")
 
         # Activate web interface.
         if self.config.get("web", {}).get("enabled", False) and web_installed:

cloudbot/client.py

@@ -4,6 +4,7 @@
 import random
 
 from cloudbot.permissions import PermissionManager
+from cloudbot.util import async_util
 
 logger = logging.getLogger("cloudbot")
 
@@ -19,6 +20,13 @@ def _decorate(cls):
     return lambda cls: _decorate(cls)
 
 
+class ClientConnectError(Exception):
+    def __init__(self, client_name, server):
+        super().__init__("Unable to connect to client {} with server {}".format(client_name, server))
+        self.client_name = client_name
+        self.server = server
+
+
 class Client:
     """
     A Client representing each connection the bot makes to a single server
@@ -71,6 +79,8 @@ def __init__(self, bot, name, nick, *, channels=None, config=None):
 
         self._active = False
 
+        self.cancelled_future = async_util.create_future(self.loop)
+
     def describe_server(self):
         raise NotImplementedError
 
@@ -84,7 +94,7 @@ def auto_reconnect(self):
     @asyncio.coroutine
     def try_connect(self):
         timeout = 30
-        while not self.connected:
+        while self.active and not self.connected:
             try:
                 yield from self.connect(timeout)
             except Exception:
@@ -185,3 +195,7 @@ def type(self):
     @property
     def active(self):
         return self._active
+
+    @active.setter
+    def active(self, value):
+        self._active = value

cloudbot/clients/irc.py

@@ -2,12 +2,14 @@
 import logging
 import random
 import re
+import socket
 import ssl
+import traceback
 from _ssl import PROTOCOL_SSLv23
 from functools import partial
 from ssl import SSLContext
 
-from cloudbot.client import Client, client
+from cloudbot.client import Client, client, ClientConnectError
 from cloudbot.event import Event, EventType, IrcOutEvent
 from cloudbot.util import async_util
 from cloudbot.util.parsers.irc import Message
@@ -114,15 +116,30 @@ def auto_reconnect(self):
 
     @asyncio.coroutine
     def try_connect(self):
-        while not self.connected:
+        while self.active and not self.connected:
             try:
                 yield from self.connect(self._timeout)
-            except (asyncio.TimeoutError, OSError):
-                logger.exception("[%s] Error occurred while connecting", self.name)
+            except (TimeoutError, asyncio.TimeoutError):
+                logger.error("[%s] Timeout occurred while connecting to %s", self.name, self.describe_server())
+            except (socket.error, socket.gaierror, OSError, ssl.SSLError):
+                logger.error(
+                    "[%s] Error occurred while connecting to %s (%s)",
+                    self.name, self.describe_server(),
+                    traceback.format_exc().splitlines()[-1]
+                )
+            except Exception as e:
+                raise ClientConnectError(self.name, self.describe_server()) from e
             else:
                 break
 
-            yield from asyncio.sleep(random.randrange(self._timeout))
+            sleep_time = random.randrange(self._timeout)
+            canceller = asyncio.shield(self.cancelled_future)
+            try:
+                yield from asyncio.wait_for(
+                    canceller, timeout=sleep_time
+                )
+            except asyncio.CancelledError:
+                pass
 
     @asyncio.coroutine
     def connect(self, timeout=None):
@@ -182,7 +199,8 @@ def quit(self, reason=None, set_inactive=True):
 
     def close(self):
         self.quit()
-        self._protocol.close()
+        if self._protocol:
+            self._protocol.close()
 
     def message(self, target, *messages):
         for text in messages: