From c6bc5978e2d8d93a35b49385c1ce387878878e78 Mon Sep 17 00:00:00 2001
From: Finn Stutzenstein <finn.stutzenstein@hotmail.de>
Date: Mon, 10 Aug 2020 13:24:14 +0200
Subject: [PATCH] Ping redis connection when popped from pool

---
 .gitignore                                |  1 +
 openslides/utils/redis_connection_pool.py | 35 +++++++++++++++++++++--
 2 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 2d3f5be56..19c7bd08b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,7 @@ bower_components/*
 
 # OS3+
 /server/
+/haproxy/
 
 # Local user data (settings, database, media, search index, static files)
 personal_data/*
diff --git a/openslides/utils/redis_connection_pool.py b/openslides/utils/redis_connection_pool.py
index 894211bf8..51e8a2e88 100644
--- a/openslides/utils/redis_connection_pool.py
+++ b/openslides/utils/redis_connection_pool.py
@@ -1,5 +1,5 @@
 import asyncio
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 import aioredis
 from channels_redis.core import ConnectionPool as ChannelRedisConnectionPool
@@ -13,6 +13,10 @@ connection_pool_limit = getattr(settings, "CONNECTION_POOL_LIMIT", 100)
 logger.info(f"CONNECTION_POOL_LIMIT={connection_pool_limit}")
 
 
+class InvalidConnection(Exception):
+    pass
+
+
 class ConnectionPool(ChannelRedisConnectionPool):
     """ Adds a trivial, soft limit for the pool """
 
@@ -27,7 +31,34 @@ class ConnectionPool(ChannelRedisConnectionPool):
             await asyncio.sleep(0.1)
         self.counter += 1
 
-        return await super().pop(*args, **kwargs)
+        return await self.pop_ensured_connection(*args, **kwargs)
+
+    async def pop_ensured_connection(
+        self, *args: List[Any], **kwargs: Dict[str, Any]
+    ) -> aioredis.commands.Redis:
+        redis: Optional[aioredis.commands.Redis] = None
+
+        while redis is None:
+            redis = await super().pop(*args, **kwargs)
+
+            try:
+                await self.try_ping(redis)
+            except InvalidConnection:
+                if redis is not None:
+                    super().conn_error(redis)
+                redis = None
+
+        return redis
+
+    async def try_ping(self, redis: aioredis.commands.Redis) -> None:
+        try:
+            pong = await redis.ping()
+            if pong != b"PONG":
+                logger.info("Redis connection invalid, did not recieve PONG")
+                raise InvalidConnection()
+        except (ConnectionRefusedError, ConnectionResetError):
+            logger.info("Redis connection invalid, connection is bad")
+            raise InvalidConnection()
 
     def push(self, conn: aioredis.commands.Redis) -> None:
         super().push(conn)