Logging prefix and handling redis connection errors

This commit is contained in:
FinnStutzenstein 2019-08-29 14:25:02 +02:00
parent 53dbcca85a
commit 2aa0275dca
10 changed files with 52 additions and 10 deletions

View File

@ -1,4 +1,3 @@
import logging
import os import os
import sys import sys
from collections import OrderedDict from collections import OrderedDict
@ -9,6 +8,7 @@ from django.apps import AppConfig
from django.conf import settings from django.conf import settings
from django.db.models.signals import post_migrate, pre_delete from django.db.models.signals import post_migrate, pre_delete
from openslides.utils import logging
from openslides.utils.schema_version import schema_version_handler from openslides.utils.schema_version import schema_version_handler

View File

@ -1,4 +1,3 @@
import logging
import sys import sys
from collections import defaultdict from collections import defaultdict
from typing import Dict, List from typing import Dict, List
@ -9,6 +8,7 @@ from django.contrib.contenttypes.models import ContentType
from django.db.models import Q from django.db.models import Q
from django.dispatch import Signal from django.dispatch import Signal
from ..utils import logging
from ..utils.autoupdate import Element, inform_changed_elements from ..utils.autoupdate import Element, inform_changed_elements

View File

@ -1,5 +1,4 @@
import json import json
import logging
from collections import defaultdict from collections import defaultdict
from datetime import datetime from datetime import datetime
from time import sleep from time import sleep
@ -8,6 +7,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Type
from asgiref.sync import async_to_sync from asgiref.sync import async_to_sync
from django.apps import apps from django.apps import apps
from . import logging
from .cache_providers import ( from .cache_providers import (
Cachable, Cachable,
ElementCacheProvider, ElementCacheProvider,

View File

@ -1,6 +1,5 @@
import functools import functools
import hashlib import hashlib
import logging
from collections import defaultdict from collections import defaultdict
from textwrap import dedent from textwrap import dedent
from typing import Any, Callable, Coroutine, Dict, List, Optional, Set, Tuple from typing import Any, Callable, Coroutine, Dict, List, Optional, Set, Tuple
@ -8,6 +7,7 @@ from typing import Any, Callable, Coroutine, Dict, List, Optional, Set, Tuple
from django.core.exceptions import ImproperlyConfigured from django.core.exceptions import ImproperlyConfigured
from typing_extensions import Protocol from typing_extensions import Protocol
from . import logging
from .redis import use_redis from .redis import use_redis
from .schema_version import SchemaVersion from .schema_version import SchemaVersion
from .utils import split_element_id, str_dict_to_bytes from .utils import split_element_id, str_dict_to_bytes

View File

@ -1,10 +1,10 @@
import logging
import time import time
from collections import defaultdict from collections import defaultdict
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
from urllib.parse import parse_qs from urllib.parse import parse_qs
from ..utils.websocket import WEBSOCKET_CHANGE_ID_TOO_HIGH from ..utils.websocket import WEBSOCKET_CHANGE_ID_TOO_HIGH
from . import logging
from .auth import async_anonymous_is_enabled from .auth import async_anonymous_is_enabled
from .autoupdate import AutoupdateFormat from .autoupdate import AutoupdateFormat
from .cache import element_cache, split_element_id from .cache import element_cache, split_element_id

View File

@ -0,0 +1,29 @@
import logging as python_logging
from typing import Any, MutableMapping, Tuple
from .utils import get_worker_id
class LoggerAdapter(python_logging.LoggerAdapter):
"""
Custom adapter for adding a prefix given in the constructor to every log
statement.
"""
def __init__(self, prefix: str, logger: python_logging.Logger) -> None:
super().__init__(logger, {})
self.prefix = prefix
def process(
self, msg: str, kwargs: MutableMapping[str, Any]
) -> Tuple[str, MutableMapping[str, Any]]:
return f"[{self.prefix}] {msg}", kwargs
def getLogger(name: str) -> LoggerAdapter:
"""
This method is for a drop-in replacement for the loggging module:
Use `from openslides.utils import logging` instead of `import logging`
"""
logger = python_logging.getLogger(name)
return LoggerAdapter(get_worker_id(), logger)

View File

@ -1,10 +1,10 @@
import logging
import time import time
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
from django.core.exceptions import ImproperlyConfigured from django.core.exceptions import ImproperlyConfigured
from django.db import models from django.db import models
from . import logging
from .access_permissions import BaseAccessPermissions from .access_permissions import BaseAccessPermissions
from .autoupdate import Element, inform_changed_data, inform_changed_elements from .autoupdate import Element, inform_changed_data, inform_changed_elements
from .rest_api import model_serializer_classes from .rest_api import model_serializer_classes

View File

@ -4,6 +4,10 @@ from typing import Any
from channels_redis.core import ConnectionPool from channels_redis.core import ConnectionPool
from django.conf import settings from django.conf import settings
from . import logging
logger = logging.getLogger(__name__)
try: try:
import aioredis import aioredis
@ -13,7 +17,8 @@ else:
# set use_redis to true, if there is a value for REDIS_ADDRESS in the settings # set use_redis to true, if there is a value for REDIS_ADDRESS in the settings
redis_address = getattr(settings, "REDIS_ADDRESS", "") redis_address = getattr(settings, "REDIS_ADDRESS", "")
use_redis = bool(redis_address) use_redis = bool(redis_address)
if use_redis:
logger.info(f"Redis address {redis_address}")
pool = ConnectionPool({"address": redis_address}) pool = ConnectionPool({"address": redis_address})
semaphore = asyncio.Semaphore(100) semaphore = asyncio.Semaphore(100)
@ -32,7 +37,13 @@ class RedisConnectionContextManager:
return self.conn return self.conn
async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None: async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
if exc:
logger.warn(f"Redis Exception: {exc}. Do not reuse connection...")
pool.conn_error(self.conn)
else:
pool.push(self.conn) pool.push(self.conn)
self.conn = None
semaphore.release() semaphore.release()

View File

@ -1,9 +1,10 @@
import logging
from typing import Optional from typing import Optional
from django.db.models import Max from django.db.models import Max
from mypy_extensions import TypedDict from mypy_extensions import TypedDict
from . import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
SchemaVersion = TypedDict("SchemaVersion", {"migration": int, "config": int, "db": str}) SchemaVersion = TypedDict("SchemaVersion", {"migration": int, "config": int, "db": str})

View File

@ -1,8 +1,9 @@
import asyncio import asyncio
import logging
import time import time
from typing import List, Optional from typing import List, Optional
from . import logging
class WebsocketLatencyLogger: class WebsocketLatencyLogger:
""" """