From e3cc592f70022e911ea9954819711aa3c54ebda5 Mon Sep 17 00:00:00 2001 From: Oskar Hahn Date: Sun, 12 Feb 2017 12:42:20 +0100 Subject: [PATCH 1/2] Added the command getgeiss. Work on start script to use Geiss. --- .travis.yml | 2 +- CHANGELOG | 3 +- openslides/__main__.py | 72 ++++++++++++------ .../core/management/commands/getgeiss.py | 73 +++++++++++++++++++ openslides/utils/main.py | 25 +++++++ openslides/utils/utils.py | 2 +- 6 files changed, 152 insertions(+), 25 deletions(-) create mode 100644 openslides/core/management/commands/getgeiss.py diff --git a/.travis.yml b/.travis.yml index d9c90f81c..fccee8a66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,6 @@ script: - coverage report --fail-under=44 - DJANGO_SETTINGS_MODULE='tests.settings' coverage run ./manage.py test tests.integration - - coverage report --fail-under=73 + - coverage report --fail-under=70 - DJANGO_SETTINGS_MODULE='tests.old.settings' ./manage.py test tests.old diff --git a/CHANGELOG b/CHANGELOG index 1655036f4..6d1607f67 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -115,7 +115,8 @@ Other: - Added function utils.auth.anonymous_is_enabled which returns true, if it is. - Changed has_perm to support an user id or None (for anyonmous) as first argument. - Removed our AnonymousUser. Make sure not to use user.has_perm() anymore. -- Added Russia translation (Thanks to Andreas Engler). +- Add the command getgeiss to download the latest version of Geiss. +- Added Russian translation (Thanks to Andreas Engler). Version 2.0 (2016-04-18) diff --git a/openslides/__main__.py b/openslides/__main__.py index 38555f091..012d855f1 100644 --- a/openslides/__main__.py +++ b/openslides/__main__.py @@ -1,19 +1,22 @@ #!/usr/bin/env python import os +import subprocess import sys -from django.core.management import execute_from_command_line +import django +from django.core.management import call_command, execute_from_command_line from openslides import __version__ as openslides_version from openslides.utils.main import ( ExceptionArgumentParser, UnknownCommand, get_default_settings_path, + get_geiss_path, get_local_settings_path, is_local_installation, + open_browser, setup_django_settings_module, - start_browser, write_settings, ) @@ -177,29 +180,54 @@ def start(args): # Set the django setting module and run migrations # A manual given environment variable will be overwritten setup_django_settings_module(settings_path, local_installation=local_installation) + django.setup() + from django.conf import settings - execute_from_command_line(['manage.py', 'migrate']) + call_command('migrate') - # Open the browser - if not args.no_browser: - if args.host == '0.0.0.0': - # Windows does not support 0.0.0.0, so use 'localhost' instead - start_browser('http://localhost:%s' % args.port) - else: - start_browser('http://%s:%s' % (args.host, args.port)) + use_geiss = True + if use_geiss: + # Make sure redis is used. + if settings.CHANNEL_LAYERS['default']['BACKEND'] != 'asgi_redis.RedisChannelLayer': + raise RuntimeError("You have to use the asgi redis backend in the settings to use Geiss.") - # Start the webserver - # Use flag --noreload to tell Django not to reload the server. - # Use flag --insecure to serve static files even if DEBUG is False. - # Use flag --nothreading to tell Django Channels to run in single thread mode. - execute_from_command_line([ - 'manage.py', - 'runserver', - '{}:{}'.format(args.host, args.port), - '--noreload', - '--insecure', - '--nothreading', - ]) + # Download Geiss and collect the static files. + call_command('getgeiss') + call_command('collectstatic', interactive=False) + + # Open the browser + if not args.no_browser: + open_browser(args.host, args.port) + + # Start Geiss in its own thread + subprocess.Popen([ + get_geiss_path(), + '--host', args.host, + '--port', args.port, + '--static', '/static/:{}'.format(settings.STATIC_ROOT), + '--static', '/media/:{}'.format(settings.MEDIA_ROOT), + ]) + + # Start one worker in this thread. There can be only one worker as long + # as sqlite is used. + call_command('runworker') + + else: + # Open the browser + if not args.no_browser: + open_browser(args.host, args.port) + + # Start the webserver + # Use flag --noreload to tell Django not to reload the server. + # Use flag --insecure to serve static files even if DEBUG is False. + # Use flag --nothreading to tell Django Channels to run in single thread mode. + call_command( + 'runserver', + '{}:{}'.format(args.host, args.port), + noreload=True, + insecure=True, + nothreading=True, + ) def createsettings(args): diff --git a/openslides/core/management/commands/getgeiss.py b/openslides/core/management/commands/getgeiss.py new file mode 100644 index 000000000..cecc28ada --- /dev/null +++ b/openslides/core/management/commands/getgeiss.py @@ -0,0 +1,73 @@ +import json +import os +import stat +import sys +from urllib.request import urlopen, urlretrieve + +from django.core.management.base import BaseCommand, CommandError + +from openslides.utils.main import get_geiss_path + + +class Command(BaseCommand): + """ + Command to get the latest release of Geiss from GitHub. + """ + help = 'Get the latest Geiss release from GitHub.' + + def handle(self, *args, **options): + geiss_name = get_geiss_name() + download_file = get_geiss_path() + + if os.path.isfile(download_file): + # Geiss does probably exist. Do Nothing. + # TODO: Add an update flag, that downloads geiss anyway. + return + + response = urlopen(get_geiss_url()).read() + release = json.loads(response.decode()) + download_url = None + for asset in release['assets']: + if asset['name'] == geiss_name: + download_url = asset['browser_download_url'] + break + if download_url is None: + raise CommandError("Could not find download URL in release.") + + urlretrieve(download_url, download_file) + + # Set the executable bit on the file. This will do nothing on windows + st = os.stat(download_file) + os.chmod(download_file, st.st_mode | stat.S_IEXEC) + + +def get_geiss_url(): + """ + Returns the URL to the API which gives the information which geiss binary + has to be downloaded. + + Currently this is a static github-url to the repo where geiss is at the + moment. Could be changed to use a setting or a flag in the future. + """ + return 'https://api.github.com/repos/ostcar/geiss/releases/latest' + + +def get_geiss_name(): + """ + Returns the name of the Geiss executable for the current operating system. + + For example geiss_windows_64.exe on a windows64 platform. + """ + # This will be 32 if the current python interpreter has only + # 32 bit, even if it is run on a 64 bit operating sysem. + bits = '64' if sys.maxsize > 2**32 else '32' + + geiss_names = { + 'linux': 'geiss_linux_{bits}', + 'win32': 'geiss_windows_{bits}.exe', # Yes, it is win32, even on a win64 system! + 'darwin': 'geiss_mac_{bits}'} + + try: + return geiss_names[sys.platform].format(bits=bits) + except KeyError: + raise CommandError("Plattform {} is not supported by Geiss".format(sys.platform)) diff --git a/openslides/utils/main.py b/openslides/utils/main.py index 6b9f7bca7..b99329c32 100644 --- a/openslides/utils/main.py +++ b/openslides/utils/main.py @@ -281,6 +281,14 @@ def start_browser(browser_url): thread.start() +def open_browser(host, port): + if host == '0.0.0.0': + # Windows does not support 0.0.0.0, so use 'localhost' instead + start_browser('http://localhost:%s' % port) + else: + start_browser('http://%s:%s' % (host, port)) + + def get_database_path_from_settings(): """ Retrieves the database path out of the settings file. Returns None, @@ -323,3 +331,20 @@ def is_local_installation(): This is the case if manage.py is used, or when the --local-installation flag is set. """ return True if '--local-installation' in sys.argv or 'manage.py' in sys.argv[0] else False + + +def get_geiss_path(): + """ + Returns the path and file to the geis binary. + """ + from django.conf import settings + download_path = getattr(settings, 'OPENSLIDES_USER_DATA_PATH', '') + bin_name = 'geiss.exe' if is_windows() else 'geiss' + return os.path.join(download_path, bin_name) + + +def is_windows(): + """ + Returns True when the current system is windows. Returns False otherwise. + """ + return sys.platform == 'win32' diff --git a/openslides/utils/utils.py b/openslides/utils/utils.py index 3770046cb..6983d2f3f 100644 --- a/openslides/utils/utils.py +++ b/openslides/utils/utils.py @@ -1,6 +1,6 @@ import re -import roman +import roman CAMEL_CASE_TO_PSEUDO_SNAKE_CASE_CONVERSION_REGEX_1 = re.compile('(.)([A-Z][a-z]+)') CAMEL_CASE_TO_PSEUDO_SNAKE_CASE_CONVERSION_REGEX_2 = re.compile('([a-z0-9])([A-Z])') From 688b247bcc8dff496bfb152b5d034e996760c159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norman=20J=C3=A4ckel?= Date: Wed, 15 Feb 2017 13:22:08 +0100 Subject: [PATCH 2/2] Added flag to use Geiss. Small changes for the new getgeiss command. Changed path of collected statics. --- .travis.yml | 2 +- README.rst | 1 + openslides/__main__.py | 34 +++++++---- .../core/management/commands/getgeiss.py | 60 ++++++++++--------- openslides/global_settings.py | 3 +- openslides/utils/main.py | 24 ++++---- openslides/utils/settings.py.tpl | 2 + setup.cfg | 1 + 8 files changed, 74 insertions(+), 53 deletions(-) diff --git a/.travis.yml b/.travis.yml index fccee8a66..d9c90f81c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,6 @@ script: - coverage report --fail-under=44 - DJANGO_SETTINGS_MODULE='tests.settings' coverage run ./manage.py test tests.integration - - coverage report --fail-under=70 + - coverage report --fail-under=73 - DJANGO_SETTINGS_MODULE='tests.old.settings' ./manage.py test tests.old diff --git a/README.rst b/README.rst index 47a228795..f3133ce22 100644 --- a/README.rst +++ b/README.rst @@ -47,6 +47,7 @@ virtual environment:: $ cd OpenSlides $ python3 -m venv .virtualenv $ source .virtualenv/bin/activate + $ pip install -U setuptools c. Install OpenSlides diff --git a/openslides/__main__.py b/openslides/__main__.py index 012d855f1..6af8aa3ab 100644 --- a/openslides/__main__.py +++ b/openslides/__main__.py @@ -101,6 +101,7 @@ def get_parser(): 'start', description=start_help, help=start_help) + subcommand_start.set_defaults(callback=start) subcommand_start.add_argument( '--no-browser', action='store_true', @@ -120,11 +121,14 @@ def get_parser(): action='store', default=None, help='The used settings file. The file is created, if it does not exist.') - subcommand_start.set_defaults(callback=start) subcommand_start.add_argument( '--local-installation', action='store_true', help='Store settings and user files in a local directory.') + subcommand_start.add_argument( + '--use-geiss', + action='store_true', + help='Use Geiss instead of Daphne as ASGI protocol server.') # Subcommand createsettings createsettings_help = 'Creates the settings file.' @@ -183,13 +187,13 @@ def start(args): django.setup() from django.conf import settings + # Migrate database call_command('migrate') - use_geiss = True - if use_geiss: - # Make sure redis is used. + if args.use_geiss: + # Make sure Redis is used. if settings.CHANNEL_LAYERS['default']['BACKEND'] != 'asgi_redis.RedisChannelLayer': - raise RuntimeError("You have to use the asgi redis backend in the settings to use Geiss.") + raise RuntimeError("You have to use the ASGI Redis backend in the settings to use Geiss.") # Download Geiss and collect the static files. call_command('getgeiss') @@ -208,8 +212,8 @@ def start(args): '--static', '/media/:{}'.format(settings.MEDIA_ROOT), ]) - # Start one worker in this thread. There can be only one worker as long - # as sqlite is used. + # Start one worker in this thread. There can be only one worker as + # long as SQLite3 is used. call_command('runworker') else: @@ -217,16 +221,24 @@ def start(args): if not args.no_browser: open_browser(args.host, args.port) - # Start the webserver + # Start Daphne and one worker + # # Use flag --noreload to tell Django not to reload the server. + # Therefor we have to set the keyword noreload to False because Django + # parses this directly to the use_reloader keyword. + # # Use flag --insecure to serve static files even if DEBUG is False. - # Use flag --nothreading to tell Django Channels to run in single thread mode. + # + # Use flag --nothreading to tell Django Channels to run in single + # thread mode with one worker only. Therefor we have to set the keyword + # nothreading to False because Django parses this directly to + # use_threading keyword. call_command( 'runserver', '{}:{}'.format(args.host, args.port), - noreload=True, + noreload=False, # Means True, see above. insecure=True, - nothreading=True, + nothreading=False, # Means True, see above. ) diff --git a/openslides/core/management/commands/getgeiss.py b/openslides/core/management/commands/getgeiss.py index cecc28ada..afe4b45a4 100644 --- a/openslides/core/management/commands/getgeiss.py +++ b/openslides/core/management/commands/getgeiss.py @@ -16,19 +16,19 @@ class Command(BaseCommand): help = 'Get the latest Geiss release from GitHub.' def handle(self, *args, **options): - geiss_name = get_geiss_name() + geiss_github_name = self.get_geiss_github_name() download_file = get_geiss_path() if os.path.isfile(download_file): - # Geiss does probably exist. Do Nothing. + # Geiss does probably exist. Do nothing. # TODO: Add an update flag, that downloads geiss anyway. return - response = urlopen(get_geiss_url()).read() + response = urlopen(self.get_geiss_url()).read() release = json.loads(response.decode()) download_url = None for asset in release['assets']: - if asset['name'] == geiss_name: + if asset['name'] == geiss_github_name: download_url = asset['browser_download_url'] break if download_url is None: @@ -40,34 +40,36 @@ class Command(BaseCommand): st = os.stat(download_file) os.chmod(download_file, st.st_mode | stat.S_IEXEC) + self.stdout.write(self.style.SUCCESS('Geiss successfully downloaded.')) -def get_geiss_url(): - """ - Returns the URL to the API which gives the information which geiss binary - has to be downloaded. + def get_geiss_url(self): + """ + Returns the URL to the API which gives the information which Geiss + binary has to be downloaded. - Currently this is a static github-url to the repo where geiss is at the - moment. Could be changed to use a setting or a flag in the future. - """ - return 'https://api.github.com/repos/ostcar/geiss/releases/latest' + Currently this is a static GitHub URL to the repository where geiss + is hosted at the moment. + """ + # TODO: Use a settings variable or a command line flag in the future. + return 'https://api.github.com/repos/ostcar/geiss/releases/latest' + def get_geiss_github_name(self): + """ + Returns the name of the Geiss executable for the current operating + system. -def get_geiss_name(): - """ - Returns the name of the Geiss executable for the current operating system. + For example geiss_windows_64 on a windows64 platform. + """ + # This will be 32 if the current python interpreter has only + # 32 bit, even if it is run on a 64 bit operating sysem. + bits = '64' if sys.maxsize > 2**32 else '32' - For example geiss_windows_64.exe on a windows64 platform. - """ - # This will be 32 if the current python interpreter has only - # 32 bit, even if it is run on a 64 bit operating sysem. - bits = '64' if sys.maxsize > 2**32 else '32' + geiss_names = { + 'linux': 'geiss_linux_{bits}', + 'win32': 'geiss_windows_{bits}.exe', # Yes, it is win32, even on a win64 system! + 'darwin': 'geiss_mac_{bits}'} - geiss_names = { - 'linux': 'geiss_linux_{bits}', - 'win32': 'geiss_windows_{bits}.exe', # Yes, it is win32, even on a win64 system! - 'darwin': 'geiss_mac_{bits}'} - - try: - return geiss_names[sys.platform].format(bits=bits) - except KeyError: - raise CommandError("Plattform {} is not supported by Geiss".format(sys.platform)) + try: + return geiss_names[sys.platform].format(bits=bits) + except KeyError: + raise CommandError("Plattform {} is not supported by Geiss".format(sys.platform)) diff --git a/openslides/global_settings.py b/openslides/global_settings.py index 43d897270..e47f8fe8a 100644 --- a/openslides/global_settings.py +++ b/openslides/global_settings.py @@ -81,8 +81,6 @@ LOCALE_PATHS = [ STATIC_URL = '/static/' -STATIC_ROOT = os.path.join(MODULE_DIR, '..', 'collected-static') - STATICFILES_DIRS = [ os.path.join(MODULE_DIR, 'static'), ] @@ -134,6 +132,7 @@ CACHES = { # Django Channels # http://channels.readthedocs.io/en/latest/ +# https://github.com/ostcar/geiss CHANNEL_LAYERS = { 'default': { diff --git a/openslides/utils/main.py b/openslides/utils/main.py index b99329c32..da06fb616 100644 --- a/openslides/utils/main.py +++ b/openslides/utils/main.py @@ -261,6 +261,18 @@ def write_settings(settings_path=None, template=None, **context): return os.path.realpath(settings_path) +def open_browser(host, port): + """ + Launches the default web browser at the given host and port and opens + the webinterface. Uses start_browser internally. + """ + if host == '0.0.0.0': + # Windows does not support 0.0.0.0, so use 'localhost' instead + start_browser('http://localhost:%s' % port) + else: + start_browser('http://%s:%s' % (host, port)) + + def start_browser(browser_url): """ Launches the default web browser at the given url and opens the @@ -281,14 +293,6 @@ def start_browser(browser_url): thread.start() -def open_browser(host, port): - if host == '0.0.0.0': - # Windows does not support 0.0.0.0, so use 'localhost' instead - start_browser('http://localhost:%s' % port) - else: - start_browser('http://%s:%s' % (host, port)) - - def get_database_path_from_settings(): """ Retrieves the database path out of the settings file. Returns None, @@ -335,7 +339,7 @@ def is_local_installation(): def get_geiss_path(): """ - Returns the path and file to the geis binary. + Returns the path and file to the Geiss binary. """ from django.conf import settings download_path = getattr(settings, 'OPENSLIDES_USER_DATA_PATH', '') @@ -345,6 +349,6 @@ def get_geiss_path(): def is_windows(): """ - Returns True when the current system is windows. Returns False otherwise. + Returns True if the current system is Windows. Returns False otherwise. """ return sys.platform == 'win32' diff --git a/openslides/utils/settings.py.tpl b/openslides/utils/settings.py.tpl index f9b265146..b6bf1726b 100644 --- a/openslides/utils/settings.py.tpl +++ b/openslides/utils/settings.py.tpl @@ -123,6 +123,8 @@ TIME_ZONE = 'Europe/Berlin' STATICFILES_DIRS = [os.path.join(OPENSLIDES_USER_DATA_PATH, 'static')] + STATICFILES_DIRS +STATIC_ROOT = os.path.join(OPENSLIDES_USER_DATA_PATH, 'collected-static') + # Files # https://docs.djangoproject.com/en/1.10/topics/files/ diff --git a/setup.cfg b/setup.cfg index 1f36c66b8..39c01bcdc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,6 @@ [coverage:run] source = openslides +omit = openslides/core/management/commands/getgeiss.py [coverage:html] directory = personal_data/tmp/htmlcov