2018-11-01 17:30:18 +01:00
|
|
|
from typing import Any
|
|
|
|
|
|
|
|
from django.conf import settings
|
|
|
|
|
2019-08-29 14:25:02 +02:00
|
|
|
from . import logging
|
|
|
|
|
|
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
2018-11-01 17:30:18 +01:00
|
|
|
|
Redis: Wait for replication on writes
Since channels_redis does not support dedicated read-redis instances, the
autoupdate message may be received before the data was replicated. All workers
read the autoupdate message from the write host, so there is a race between
getting this message and a finished replication. For large payloads, the
replication is slower in the most cases (even more in a distributed setup, where
the master and replica are on different nodes). The easy way is to wait for
replication. But there is one difficulty: The number of replicas has to be
known. There is a new settings-variable "AMOUNT_REPLICAS" which defaults to 1.
It needs to be set correctly! If it is too high, every autoupdate will be
delayed by 1 seconds because of a timeout witing for non-existent replicas. If
it is too low, some autoupdates may be wrong (and not detectable by the client!)
becuase of reading from non-synchronised relicas.
The other possibility is to fork channel_redis and add the feature of a
read-only redis. This ould help, because on a single redis instance all commands
are ordered: First, the data is synced, then the autoupdate message. Attention:
This means, if redis-replicas are scaled up, one must make sure to read from the
same instance. I think this is not possible in the way how dockers overlay
networks work. The only way would be to open one connection and reuse the
connection from channels_redis in OpenSlides. This would mean a heavy
integration of channels_redis (meaning including the source code in our repo).
For the first fix, this one is easy and should work.
2020-04-01 09:26:50 +02:00
|
|
|
# Defaults
|
|
|
|
use_redis = False
|
|
|
|
use_read_only_redis = False
|
|
|
|
|
2018-11-01 17:30:18 +01:00
|
|
|
try:
|
|
|
|
import aioredis
|
|
|
|
except ImportError:
|
Redis: Wait for replication on writes
Since channels_redis does not support dedicated read-redis instances, the
autoupdate message may be received before the data was replicated. All workers
read the autoupdate message from the write host, so there is a race between
getting this message and a finished replication. For large payloads, the
replication is slower in the most cases (even more in a distributed setup, where
the master and replica are on different nodes). The easy way is to wait for
replication. But there is one difficulty: The number of replicas has to be
known. There is a new settings-variable "AMOUNT_REPLICAS" which defaults to 1.
It needs to be set correctly! If it is too high, every autoupdate will be
delayed by 1 seconds because of a timeout witing for non-existent replicas. If
it is too low, some autoupdates may be wrong (and not detectable by the client!)
becuase of reading from non-synchronised relicas.
The other possibility is to fork channel_redis and add the feature of a
read-only redis. This ould help, because on a single redis instance all commands
are ordered: First, the data is synced, then the autoupdate message. Attention:
This means, if redis-replicas are scaled up, one must make sure to read from the
same instance. I think this is not possible in the way how dockers overlay
networks work. The only way would be to open one connection and reuse the
connection from channels_redis in OpenSlides. This would mean a heavy
integration of channels_redis (meaning including the source code in our repo).
For the first fix, this one is easy and should work.
2020-04-01 09:26:50 +02:00
|
|
|
pass
|
2018-11-01 17:30:18 +01:00
|
|
|
else:
|
2020-05-28 11:40:41 +02:00
|
|
|
from .redis_connection_pool import ConnectionPool # type: ignore
|
2019-09-13 10:16:36 +02:00
|
|
|
|
2018-11-01 17:30:18 +01:00
|
|
|
# set use_redis to true, if there is a value for REDIS_ADDRESS in the settings
|
2019-01-06 16:22:33 +01:00
|
|
|
redis_address = getattr(settings, "REDIS_ADDRESS", "")
|
2018-11-01 17:30:18 +01:00
|
|
|
use_redis = bool(redis_address)
|
2019-10-16 11:47:02 +02:00
|
|
|
|
2019-08-29 14:25:02 +02:00
|
|
|
if use_redis:
|
|
|
|
logger.info(f"Redis address {redis_address}")
|
2019-10-16 11:47:02 +02:00
|
|
|
pool = ConnectionPool({"address": redis_address})
|
2018-11-01 17:30:18 +01:00
|
|
|
|
2019-10-16 11:47:02 +02:00
|
|
|
redis_read_only_address = getattr(settings, "REDIS_READ_ONLY_ADDRESS", "")
|
|
|
|
use_read_only_redis = bool(redis_read_only_address)
|
|
|
|
if use_read_only_redis:
|
|
|
|
logger.info(f"Redis read only address {redis_read_only_address}")
|
|
|
|
read_only_pool = ConnectionPool({"address": redis_read_only_address})
|
|
|
|
else:
|
|
|
|
logger.info("Redis is not configured.")
|
2019-03-27 22:51:33 +01:00
|
|
|
|
|
|
|
|
2019-10-16 11:47:02 +02:00
|
|
|
# TODO: contextlib.asynccontextmanager can be used in python 3.7
|
2018-11-01 17:30:18 +01:00
|
|
|
class RedisConnectionContextManager:
|
|
|
|
"""
|
|
|
|
Async context manager for connections
|
|
|
|
"""
|
2019-01-06 16:22:33 +01:00
|
|
|
|
2019-10-16 11:47:02 +02:00
|
|
|
def __init__(self, read_only: bool) -> None:
|
|
|
|
self.pool = read_only_pool if read_only and use_read_only_redis else pool
|
2018-11-01 17:30:18 +01:00
|
|
|
|
2019-01-06 16:22:33 +01:00
|
|
|
async def __aenter__(self) -> "aioredis.RedisConnection":
|
2019-10-16 11:47:02 +02:00
|
|
|
self.conn = await self.pool.pop()
|
2018-11-01 17:30:18 +01:00
|
|
|
return self.conn
|
|
|
|
|
|
|
|
async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
|
2019-08-29 14:25:02 +02:00
|
|
|
if exc:
|
|
|
|
logger.warn(f"Redis Exception: {exc}. Do not reuse connection...")
|
2019-10-16 11:47:02 +02:00
|
|
|
self.pool.conn_error(self.conn)
|
2019-08-29 14:25:02 +02:00
|
|
|
else:
|
2019-10-16 11:47:02 +02:00
|
|
|
self.pool.push(self.conn)
|
2019-08-29 14:25:02 +02:00
|
|
|
self.conn = None
|
|
|
|
|
2018-11-01 17:30:18 +01:00
|
|
|
|
2019-10-16 11:47:02 +02:00
|
|
|
def get_connection(read_only: bool = False) -> RedisConnectionContextManager:
|
2018-11-01 17:30:18 +01:00
|
|
|
"""
|
|
|
|
Returns contextmanager for a redis connection.
|
|
|
|
"""
|
2019-10-16 11:47:02 +02:00
|
|
|
return RedisConnectionContextManager(read_only)
|