Merge pull request #20 from cybertec-postgresql/bugfix/stale-leader-observation

Fix race condition causing a spurious promote during a global DCS outage

Author: avandras

patroni/ha.py

@@ -300,6 +300,7 @@ def set_is_leader(self, value: bool) -> None:
 
         :param value: is the current node the leader.
         """
+        self.patroni.multisite.set_is_local_leader(value)
         with self._leader_expiry_lock:
             self._leader_expiry = time.time() + self.dcs.ttl if value else 0
             if not value:

patroni/multisite.py

@@ -64,6 +64,10 @@ def status(self) -> Dict[str, Any]:
     def should_failover(self) -> bool:
         return False
 
+    def set_is_local_leader(self, value: bool):
+        """Update multisite mechanisms view if this node is running as a site leader."""
+        pass
+
     def on_shutdown(self, checkpoint_location: int, prev_location: int):
         pass
 
@@ -111,6 +115,8 @@ def __init__(self, config: 'Config', on_change: Optional[Callable[..., None]] =
 
         self._dcs_error = None
 
+        self._is_local_leader = False
+
     @staticmethod
     def get_dcs_config(config: 'Config') -> Tuple[Dict[str, Any], AbstractDCS]:
         msconfig = config['multisite']
@@ -421,6 +427,10 @@ def touch_member(self):
         logger.info(f"Registering site {self.name} in DCS with address {address}")
         self.dcs.touch_member(data)
 
+    def set_is_local_leader(self, value: bool):
+        # Assumes setting a boolean flag from other threads without a lock is atomic
+        self._is_local_leader = value
+
     def run(self):
         self._observe_leader()
         while not self._heartbeat.wait(self.config['observe_interval']):
@@ -433,7 +443,8 @@ def run(self):
             if self._state_updater:
                 self._state_updater.store_updates()
             while not self._heartbeat.wait(self.config['observe_interval']):
-                self._observe_leader()
+                if not self._is_local_leader:
+                    self._observe_leader()
 
     def shutdown(self):
         self.stop_requested = True