Merge pull request #5440 from FinnStutzenstein/restructure

WIP: Repository restructure
This commit is contained in:
Finn Stutzenstein 2020-08-21 08:36:27 +02:00 committed by GitHub
commit 216e4f00a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
391 changed files with 1573 additions and 545 deletions

112
.gitignore vendored
View File

@ -1,63 +1,9 @@
# General ## General
*.pyc *.pyc
*.swp *.swp
*.swo *.swo
*.log *.log
*~ *~
# Virtual Environment
.virtualenv*/*
.venv/*
# Javascript tools and libraries
node_modules/*
bower_components/*
# OS4-Submodules
/openslides-*
# OS3+
/server/
/haproxy/
# Local user data (settings, database, media, search index, static files)
personal_data/*
openslides/static/*
collected-static/*
# Package building/IDE
docs/_build/*
*.egg-info
build/*
dist/*
debug/*
.DS_Store
.idea
*.code-workspace
# Unit test and coverage reports
.coverage
tests/file/*
tests/db.sqlite3.test
.pytest_cache
# Plugin development
openslides_*
# Mypy cache for typechecking
.mypy_cache
# OpenSlides 3 Client
# compiled output
client/dist
client/tmp
client/out-tsc
client/documentation
# dependencies
client/node_modules
# IDEs and editors # IDEs and editors
/.idea /.idea
.project .project
@ -66,13 +12,53 @@ client/node_modules
*.launch *.launch
.settings/ .settings/
*.sublime-workspace *.sublime-workspace
# IDE - VSCode
.vscode/* .vscode/*
*.code-workspace
# System Files
.DS_Store
Thumbs.db
# Virtual Environment
.virtualenv*/*
.venv/*
# misc ## Compatibility
# OS4-Submodules
/openslides-*/
/haproxy/
# Plugin development
openslides_*
# Old OS3 stuff
/tests/
## Server
# Local user data (settings, database, media, search index, static files)
personal_data/*
server/personal_data/*
server/openslides/static/*
# Unit test and coverage reports
.coverage
server/tests/file/*
server/tests/db.sqlite3.test
.pytest_cache
# Package building
*.egg-info
# Mypy cache for typechecking
.mypy_cache
## OpenSlides 3 Client
# Javascript tools and libraries
**/node_modules/*
**/bower_components/*
# compiled output
client/dist
client/static
client/tmp
client/out-tsc
# docs
client/documentation
Compodoc Compodoc
Compodocmodules Compodocmodules
# build artifacts
client/.sass-cache client/.sass-cache
client/connect.lock client/connect.lock
client/coverage client/coverage
@ -85,8 +71,10 @@ client/yarn.lock
package-lock.json package-lock.json
client/package-lock.json client/package-lock.json
cypress.json cypress.json
*-version.txt
# System Files ## Deployment
client/.DS_Store # Docker build artifacts
client/Thumbs.db /docker/docker-compose.yml
*-version.txt
# secrets
docker/secrets/*.env

View File

@ -1,12 +1,12 @@
dist: xenial dist: xenial
sudo: true os: linux
cache: cache:
- directories: - directories:
- $HOME/.cache/pip - $HOME/.cache/pip
- client/node_modules - client/node_modules
matrix: jobs:
include: include:
- stage: "Dependencies" - stage: "Dependencies"
name: "Installing dependencies for python" name: "Installing dependencies for python"
@ -15,11 +15,12 @@ matrix:
- "3.6" - "3.6"
cache: cache:
pip: true pip: true
install: before_install:
- python --version - python --version
- pip install --upgrade setuptools pip - cd server
- pip install --upgrade --requirement requirements/development.txt install:
- pip install --upgrade .[big_mode] - pip install --upgrade pip
- pip install --upgrade --requirement requirements.txt
- pip freeze - pip freeze
script: skip script: skip
@ -36,15 +37,11 @@ matrix:
install: install:
- npm install - npm install
script: skip script: skip
- stage: "Run tests" - stage: "Run tests"
name: "Client: Testing" name: "Client: Testing"
language: node_js language: node_js
node_js: "12.18" node_js: "12.18"
apt:
sources:
- google-chrome
packages:
- google-chrome-stable
services: services:
- xvfb - xvfb
install: install:
@ -79,36 +76,6 @@ matrix:
- cd client - cd client
- npm run build-debug - npm run build-debug
- name: "Server: Tests Python 3.6"
language: python
python:
- "3.6"
script:
- mypy openslides/ tests/
- pytest --cov --cov-fail-under=75
- name: "Server: Tests Python 3.7"
language: python
python:
- "3.7"
script:
- flake8 openslides tests
- isort --check-only --diff --recursive openslides tests
- black --check --diff --target-version py36 openslides tests
- mypy openslides/ tests/
- pytest --cov --cov-fail-under=75
- name: "Server: Tests Python 3.8"
language: python
python:
- "3.8"
script:
- flake8 openslides tests
- isort --check-only --diff --recursive openslides tests
- black --check --diff --target-version py36 openslides tests
- mypy openslides/ tests/
- pytest --cov --cov-fail-under=75
- name: "Client: Linting" - name: "Client: Linting"
language: node_js language: node_js
node_js: "12.18" node_js: "12.18"
@ -123,13 +90,51 @@ matrix:
- cd client - cd client
- npm run prettify-check - npm run prettify-check
- name: "Server: Tests Python 3.6"
language: python
python:
- "3.6"
before_install:
- cd server
script:
- mypy openslides/ tests/
- pytest --cov --cov-fail-under=75
- name: "Server: Tests Python 3.7"
language: python
python:
- "3.7"
before_install:
- cd server
script:
- flake8 openslides tests
- isort --check-only --diff --recursive openslides tests
- black --check --diff --target-version py36 openslides tests
- mypy openslides/ tests/
- pytest --cov --cov-fail-under=75
- name: "Server: Tests Python 3.8"
language: python
python:
- "3.8"
before_install:
- cd server
script:
- flake8 openslides tests
- isort --check-only --diff --recursive openslides tests
- black --check --diff --target-version py36 openslides tests
- mypy openslides/ tests/
- pytest --cov --cov-fail-under=75
- name: "Server: Tests Startup Routine Python 3.7" - name: "Server: Tests Startup Routine Python 3.7"
language: python language: python
python: python:
- "3.7" - "3.7"
before_install:
- cd server
script: script:
- set -e - set -e
- python manage.py createsettings - python manage.py createsettings
- python manage.py migrate - python manage.py migrate
- python manage.py runserver --noreload & (sleep 15 && kill $(ps aux | grep 'manage.py runserver' | head -n -1 | awk '{print $2}')) - python manage.py runserver --noreload & (sleep 20 && kill $(ps aux | grep 'manage.py runserver' | head -n -1 | awk '{print $2}'))
- set +e - set +e

View File

@ -31,3 +31,4 @@ Authors of OpenSlides in chronological order of first contribution:
Fadi Abbud <fmfn13@hotmail.com> Fadi Abbud <fmfn13@hotmail.com>
Gabriel Meyer <meyergabriel@live.de> Gabriel Meyer <meyergabriel@live.de>
Joshua Sangmeister <joshua.sangmeister@gmail.com> Joshua Sangmeister <joshua.sangmeister@gmail.com>
Gernot Schulz <gernot@intevtion.de>

View File

@ -2,12 +2,12 @@
OpenSlides Development OpenSlides Development
======================== ========================
This instruction helps you to setup a development environment for OpenSlides. This instruction helps you to setup a development environment for OpenSlides. A
simple dev setup will be configured without the need of the docker-compose
setup. There are only the server running without a cache and a sqlite database
and the client as an development server.
Installation and start of the development version
=================================================
1. Installation on GNU/Linux or Mac OS X 1. Installation on GNU/Linux or Mac OS X
---------------------------------------- ----------------------------------------
@ -37,15 +37,30 @@ Clone current master version from `OpenSlides GitHub repository
c. Setup a virtual Python environment (optional) c. Setup a virtual Python environment (optional)
'''''''''''''''''''''''''''''''''''''''''''''''' ''''''''''''''''''''''''''''''''''''''''''''''''
See step 1. b. in the installation section in the `README.rst You can setup a virtual Python environment using the virtual environment
<https://github.com/OpenSlides/OpenSlides/blob/master/README.rst>`_. (venv) package for Python to install OpenSlides as non-root user. This will
allow for encapsulated dependencies. They will be installed in the virtual
environment and not globally on your system.
Setup and activate the virtual environment::
d. Finish the server $ python3 -m venv .virtualenv
'''''''''''''''''''' $ source .virtualenv/bin/activate
You can exit the environment with::
$ deactivate
d. Server
'''''''''
Go into the server's directory::
$ cd server/
Install all required Python packages:: Install all required Python packages::
$ pip install --upgrade setuptools pip
$ pip install --requirement requirements.txt $ pip install --requirement requirements.txt
Create a settings file, run migrations and start the server:: Create a settings file, run migrations and start the server::
@ -54,41 +69,29 @@ Create a settings file, run migrations and start the server::
$ python manage.py migrate $ python manage.py migrate
$ python manage.py runserver $ python manage.py runserver
All you data (database, config, mediafiles) are stored in ``personal_data/var``.
To get help on the command line options run:: To get help on the command line options run::
$ python manage.py --help $ python manage.py --help
Later you might want to restart the server with one of the following commands. Later you might want to restart the server with one of the following commands.
To start OpenSlides with this command and to avoid opening new browser windows To run the OpenSlides server execute::
run::
$ python manage.py start --no-browser $ python manage.py runserver
When debugging something email related change the email backend to console:: When debugging something email related change the email backend to console::
$ python manage.py start --debug-email $ python manage.py runserver --debug-email
The server is available under http://localhost:8000. Especially the rest interface
might be important during development: http://localhost:8000/rest/ (The trailing
slash is important!).
e. Debugging the server e. Client
''''''''''''''''''''''' '''''''''
If you wish to have even further debugging, enable `django-extensions Go in the client's directory::
<https://django-extensions.readthedocs.io/>`_ in the ``settings.py`` by adding
``django_extensions`` to the list of ``INSTALLED_PLLUGINS``. Make sure, you
install the following packages::
$ pip install Werkzeug pyparsing pydot django-extensions
You can start the enhanced debugging-server via::
$ python manage.py runserver_plus
f. Setup and start the client
'''''''''''''''''''''''''''''
Go in the client's directory in a second command-line interface::
$ cd client/ $ cd client/
@ -97,14 +100,7 @@ Install all dependencies and start the development server::
$ npm install $ npm install
$ npm start $ npm start
Now the client is available under ``localhost:4200``. After a while, the client is available under http://localhost:4200.
If you want to provide the client statically, you can build it via::
$ npm run build
The build client files are availible from the root directory in
``openslides/static`` and can be provided via NGINX.
2. Installation on Windows 2. Installation on Windows
@ -140,12 +136,6 @@ a. Running server tests
To run some server tests see `.travis.yml To run some server tests see `.travis.yml
<https://github.com/OpenSlides/OpenSlides/blob/master/.travis.yml>`_. <https://github.com/OpenSlides/OpenSlides/blob/master/.travis.yml>`_.
You can generate an class-structure image when having `django_extensions`
enabled (see above)::
$ python manage.py graph_models -a -g -o my_project_visualized.png
b. Client tests and commands b. Client tests and commands
'''''''''''''''''''''''''''' ''''''''''''''''''''''''''''
@ -169,53 +159,23 @@ README.md using following command::
$ npm run licenses $ npm run licenses
OpenSlides in big mode 4. Notes for running OpenSlides in larger setups
====================== ------------------------------------------------
To install OpenSlides for big assemblies (in 'big mode') you have to setup some For productive setups refer to the docker-compose setup described in the main
additional components and configurations. In the 'big mode' you should use a webserver `README<https://github.com/OpenSlides/OpenSlides/blob/master/README.rst>`_.
like NGINX to serve the static and media files as proxy server in front of your OpenSlides
interface server. You should also use a database like PostgreSQL. Use Redis as channels backend,
cache backend and session engine. Finally you should use gunicorn with uvicorn as interface server.
While develpment it might be handy to use a cache and another database.
PostgreSQL is recommended and Redis necessary as a cache. Both can be set up in
the ``settings.py``. Please consider reading the `OpenSlides configuration
<https://github.com/OpenSlides/OpenSlides/blob/master/server/SETTINGS.rst>`_ page
to find out about all configurations, especially when using OpenSlides for big
assemblies.
1. Install and configure PostgreSQL and Redis If you followed the instructions and installed the pip requirements form the
--------------------------------------------- ``requirements.py`` all needed dependencies for another worker are installed.
Instead of running ``python manage.py runserver`` you can use daphne or gunicorn
Install `PostgreSQL <https://www.postgresql.org/>`_ and `Redis (the latter is used in the prod setup)::
<https://redis.io/>`_. For Ubuntu 18.04 e. g. run::
$ sudo apt-get install postgresql libpq-dev redis-server
Be sure that database and redis server is running. For Ubuntu 18.04 e. g. this
was done automatically if you used the package manager.
Then add database user and database. For Ubuntu 18.04 e. g. run::
$ sudo -u postgres createuser --pwprompt --createdb openslides
$ sudo -u postgres createdb --owner=openslides openslides
2. Change OpenSlides settings
-----------------------------
Create OpenSlides settings file if it does not exist::
$ python manage.py createsettings
Change OpenSlides settings file (usually called settings.py): Setup
`DATABASES` entry as mentioned in the settings file. Set `use_redis` to
`True`.
Populate your new database::
$ python manage.py migrate
3. Run OpenSlides
-----------------
To start Daphne run::
$ export DJANGO_SETTINGS_MODULE=settings $ export DJANGO_SETTINGS_MODULE=settings
$ export PYTHONPATH=personal_data/var/ $ export PYTHONPATH=personal_data/var/
@ -226,57 +186,3 @@ server::
$ gunicorn -w 4 -b 0.0.0.0:8000 -k uvicorn.workers.UvicornWorker openslides.asgi:application $ gunicorn -w 4 -b 0.0.0.0:8000 -k uvicorn.workers.UvicornWorker openslides.asgi:application
4. Use NGINX (optional)
-----------------------
When using NGINX as a proxy for delivering static files the performance of the
setup will increase.
This is an example ``nginx.conf`` configuration for Daphne listing on port
8000::
worker_processes 1;
events {
worker_connections 1024;
}
http {
server {
listen 80;
server_name localhost;
root $YOUR_OS_ROOT_FOLDER/openslides/static;
index index.html index.htm;
include /etc/nginx/mime.types;
client_max_body_size 100M;
gzip on;
gzip_min_length 1000;
gzip_proxied expired no-cache no-store private auth;
gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript;
location / {
try_files $uri $uri/ /index.html;
}
location /apps {
proxy_pass http://localhost:8000;
}
location /media {
proxy_pass http://localhost:8000;
}
location /rest {
proxy_set_header Host $http_host;
proxy_pass http://localhost:8000;
}
location /ws {
proxy_pass http://localhost:8000;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
}
}
}

View File

@ -1,30 +0,0 @@
FROM python:3.7-slim
RUN mkdir /app
RUN apt -y update && \
apt -y upgrade && \
apt install -y libpq-dev curl wget xz-utils bzip2 git gcc gnupg2 make g++
RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
RUN apt -y install nodejs
RUN npm install -g @angular/cli@latest
RUN useradd -m openslides
RUN chown -R openslides /app
WORKDIR /app
COPY . /app
RUN rm -rf /app/.virtualenv* && \
rm -rf /app/client/node_modules
RUN chown -R openslides /app
# Installing python dependencies
RUN pip install -r requirements.txt
RUN rm -rf /var/lib/apt/lists/*
# installing client
USER openslides
RUN ng config -g cli.warnings.versionMismatch false && \
cd client && \
npm install
RUN cd client && \
npm run build && \
./node_modules/.bin/compodoc -t -p src/tsconfig.app.json -n 'OpenSlides Documentation' -d ../openslides/static/doc -e html

View File

@ -5,170 +5,137 @@
What is OpenSlides? What is OpenSlides?
=================== ===================
OpenSlides is a free, web based presentation and assembly system for OpenSlides is a free, Web-based presentation and assembly system for
managing and projecting agenda, motions and elections of an assembly. See managing and projecting agenda, motions, and elections of assemblies. See
https://openslides.com for more information. https://openslides.com for more information.
Installation Installation
============ ============
The OpenSlides server runs everywhere where Python is running (for example on The main deployment method is using Docker and docker-compose. You only need to
GNU/Linux, Mac or Windows). For the OpenSlides client a current web browser is required. have these tools installed and no further dependencies. If you want a simpler
setup or are interested in developing, please refer to `development
instructions
1. Installation on GNU/Linux or Mac OS X
----------------------------------------
a. Check requirements
'''''''''''''''''''''
Make sure that you have installed `Python (>= 3.6) <https://www.python.org/>`_
on your system.
Additional you need build-essential packages, header files and a static
library for Python and also the pyvenv-3 binary package for python3.
E.g. run on Debian/Ubuntu::
$ sudo apt-get install build-essential python3-dev python3-venv
b. Setup a virtual Python environment (optional)
''''''''''''''''''''''''''''''''''''''''''''''''
You can setup a virtual Python environment using the virtual environment
(venv) package for Python to install OpenSlides as non-root user.
Create your OpenSlides directory and change to it::
$ mkdir OpenSlides
$ cd OpenSlides
Setup and activate the virtual environment::
$ python3 -m venv .virtualenv
$ source .virtualenv/bin/activate
$ pip install --upgrade setuptools pip
c. Install OpenSlides
'''''''''''''''''''''
To install OpenSlides just run::
$ pip install openslides
This installs the latest stable version. To install a specific (beta)
version use ``openslides==x.y``.
You can also use the package from the `OpenSlides website
<https://openslides.com/>`_. Download latest OpenSlides release as
compressed tar archive and run::
$ pip install openslides-x.y.tar.gz
This will install all required Python packages (see
``requirements/production.txt``).
d. Start OpenSlides
'''''''''''''''''''
To start OpenSlides simply run::
$ openslides
If you run this command the first time, a new database and the admin account
(Username: ``admin``, Password: ``admin``) will be created. Please change the
password after first login!
OpenSlides will start a webserver. It will also try to open the webinterface in
your default webbrowser. The server will try to listen on the local ip address
on port 8000. That means that the server will be available to everyone on your
local network (at least for commonly used network configurations).
If you use a virtual environment (see step b.), do not forget to activate
the environment before restart after you closed the terminal::
$ source .virtualenv/bin/activate
To get help on the command line options run::
$ openslides --help
You can store settings, database and other personal files in a local
subdirectory and use these files e. g. if you want to run multiple
instances of OpenSlides::
$ openslides start --local-installation
2. Installation on Windows
--------------------------
Follow the instructions above (1. Installation on GNU/Linux or Mac OS X) but care
of the following variations.
To get Python download and run the latest `Python 3.7 32-bit (x86) executable
installer <https://www.python.org/downloads/windows/>`_. Note that the 32-bit
installer is required even on a 64-bit Windows system. If you use the 64-bit
installer, step 1c of the instruction might fail unless you installed some
packages manually.
In some cases you have to install `MS Visual C++ 2015 build tools
<https://www.microsoft.com/en-us/download/details.aspx?id=48159>`_ before you
install the required python packages for OpenSlides (unfortunately Twisted
needs it).
To setup and activate the virtual environment in step 1b use::
> .virtualenv\Scripts\activate.bat
All other commands are the same as for GNU/Linux and Mac OS X.
3. Installation with Docker
---------------------------
The installation instruction for (1) and (2) described a way to use OpenSlides in a
'small mode' with max 10 concurrent clients. To install OpenSlides for big assemblies
('big mode') you have to setup some additional components and configurations.
The easiest way to run the OpenSlides 'big mode' environment (with PostgreSQL, Redis
and NGINX) with Docker Compose: use our docker compose suite. Follow the instruction in
the `openslides-docker-compose Repository <https://github.com/OpenSlides/openslides-docker-compose>`_.
To install and configure all components of our 'big mode' manually you can read the
`big-mode-instruction <https://github.com/OpenSlides/OpenSlides/blob/master/DEVELOPMENT.rst#openslides-in-big-mode>`_
Configuration
=============
Please consider reading the `OpenSlides configuration
<https://github.com/OpenSlides/OpenSlides/blob/master/SETTINGS.rst>`_ page to
find out about all configurations, especially when using OpenSlides for big
assemblies.
Development
===========
To setup a development environment for OpenSlides follow the instruction of
`DEVELOPMENT.rst
<https://github.com/OpenSlides/OpenSlides/blob/master/DEVELOPMENT.rst>`_. <https://github.com/OpenSlides/OpenSlides/blob/master/DEVELOPMENT.rst>`_.
Note: This is temporary and will be replace with nice scripts...
First, you have to clone this repository::
$ git clone https://github.com/OpenSlides/OpenSlides.git
$ cd OpenSlides/docker/
You need to build the Docker images for the client and server with this
script::
$ ./build.sh all
You must define a Django secret key in ``secrets/django.env``, for example::
$ printf "DJANGO_SECRET_KEY='%s'\n" \
"$(tr -dc 'a-zA-Z0-9' < /dev/urandom | head -c 64)" > secrets/django.env
We also strongly recommend that you set a secure admin password but it is not
strictly required. If you do not set an admin password, the default login
credentials will be displayed on the login page. Setting the admin password::
$ cp secrets/admin.env.example secrets/admin.env
$ vi secrets/admin.env
Afterwards, generate the configuration file::
EXTERNAL_HTTP_PORT=8000 m4 docker-compose.yml.m4 > docker-compose.yml
Finally, you can start the instance using ``docker-compose``::
$ docker-compose up
$ # or:
$ docker-compose up -d
$ docker-compose logs
$ # ...
$ docker-compose down
Docker Swarm Mode
-----------------
OpenSlides may also be deployed in Swarm mode. Distributing instances over
multiple nodes may increase performance and offer failure resistance.
An example configuration file, ``docker-stack.yml.m4``, is provided. Unlike
the Docker Compose setup, this configuration will most likely need to be
customized, especially its placement constraints and database-related
preferences.
Before deploying an instance on Swarm, please see `Database Configuration`_ and
`Backups`_, and review your ``docker-stack.yml``
Database Configuration
----------------------
It is fairly easy to get an OpenSlides instance up an running; however, for
production setups it is strongly advised to review the database configuration.
By default, the primary database cluster will archive all WAL files in its
volume. Regularly pruning old data is left up to the host system, i.e., you.
Alternatively, you may disable WAL archiving by setting
``PGNODE_WAL_ARCHIVING=off`` in ``.env`` before starting the instance.
The provided ``docker-stack.yml.m4`` file includes additional database
services which can act as hot standby clusters with automatic failover
functionality. To take advantage of this setup, the database services need to
be configured with proper placement constraints. Before relying on this setup,
please familiarize yourself with `repmgr <https://repmgr.org/>`_.
Backups
-------
All important data is stored in the database. Additionally, the project
directory should be included in backups to ensure a smooth recovery.
The primary database usually runs in the ``pgnode1`` service (but see `Database
Configuration`_ above).
.
In some cases, it may be sufficient to generate SQL dumps with ``pg_dump``
through ``docker exec`` to create backups. However, for proper incremental
backups, the host system can backup the cluster's data directory and WAL
archives.
The cluster's data directory is available as a volume on the host system.
Additionally, the database archives its WAL files in the same volume by
default. This way, the host system can include the database volume in its
regular filesystem-based backup routine and create efficient database backups
suitable for point-in-time recovery.
The `former management repository
<https://github.com/OpenSlides/openslides-docker-compose/>`_ provides the
script `openslides-pg-mgr.sh` which can enable Postgres' backup mode in all
OpenSlides database containers.
In Swarm mode, the primary database cluster may get placed on a number of
nodes. It is, therefore, crucial to restrict the placement of database
services to nodes on which appropriate backups have been configured.
Bugs, features and development
================================
Feel free to open issues here on GitHub! Please use the right templates for
bugs and features, and use them correctly. Pull requests are also welcome; for
a general overview of the development setup refer the `development instructions
<https://github.com/OpenSlides/OpenSlides/blob/master/DEVELOPMENT.rst>`_.
Used software Used software
============= =============
OpenSlides uses the following projects or parts of them: OpenSlides uses the following projects or parts of them:
* Several Python packages (see ``requirements/production.txt`` and ``requirements/big_mode.txt``). * several Python packages (see ``server/requirements/production.txt`` and
``server/requirements/big_mode.txt``)
* Several JavaScript packages (see ``client/package.json``)
* several JavaScript packages (see ``client/package.json``)
License and authors License and authors
=================== ===================

2
client/.dockerignore Normal file
View File

@ -0,0 +1,2 @@
.git
**/node_modules

View File

@ -1,31 +1,5 @@
# OpenSlides 3 Client # OpenSlides 3 Client
Prototype application for OpenSlides 3.0 (Client).
Currently under constant heavy maintenance.
## Development Info
As an Angular project, Angular CLI is highly recommended to create components and services.
See https://angular.io/guide/quickstart for details.
### Contribution Info
Please respect the code-style defined in `.editorconf` and `.pretierrc`.
Code alignment should be automatically corrected by the pre-commit hooks.
Adjust your editor to the `.editorconfig` to avoid surprises.
See https://editorconfig.org/ for details.
### Pre-Commit Hooks
Before commiting, new code will automatically be aligned to the definitions set in the
`.prettierrc`.
Furthermore, new code has to pass linting.
Our pre-commit hooks are:
`pretty-quick --staged` and `lint`
See `package.json` for details.
### Documentation Info ### Documentation Info
The documentation can be generated by running `npm run compodoc`. The documentation can be generated by running `npm run compodoc`.
@ -38,16 +12,6 @@ command. If no port specified, it will try to use 8080.
Please document new code using JSDoc tags. Please document new code using JSDoc tags.
See https://compodoc.app/guides/jsdoc-tags.html for details. See https://compodoc.app/guides/jsdoc-tags.html for details.
### Development server
Run `npm start` for a development server. Navigate to `http://localhost:4200/`.
The app will automatically reload if you change any of the source files.
A running OpenSlides (2.2 or higher) instance is expected on port 8000.
Start OpenSlides as usual using
`python manage.py start --no-browser --host 0.0.0.0`
### Translation ### Translation
We are using ngx-translate for translation purposes. We are using ngx-translate for translation purposes.

View File

@ -17,7 +17,7 @@
"build": { "build": {
"builder": "@angular-devkit/build-angular:browser", "builder": "@angular-devkit/build-angular:browser",
"options": { "options": {
"outputPath": "../openslides/static", "outputPath": "static",
"index": "src/index.html", "index": "src/index.html",
"main": "src/main.ts", "main": "src/main.ts",
"polyfills": "src/polyfills.ts", "polyfills": "src/polyfills.ts",

1
client/build.sh Symbolic link
View File

@ -0,0 +1 @@
../server/build.sh

23
client/docker/Dockerfile Normal file
View File

@ -0,0 +1,23 @@
FROM node:13 AS nodejs
RUN mkdir -p /build/app
WORKDIR /build/app
RUN useradd -m openslides
RUN chown -R openslides /build/app
USER root
RUN npm install -g @angular/cli@^9
RUN ng config -g cli.warnings.versionMismatch false
USER openslides
COPY package.json .
RUN npm install
COPY browserslist *.json ./
COPY src ./src
RUN npm run build
COPY docker/client-version.txt static/
FROM nginx
COPY --from=nodejs /build/app/static /usr/share/nginx/html
COPY docker/nginx.conf /etc/nginx/nginx.conf

56
client/docker/nginx.conf Normal file
View File

@ -0,0 +1,56 @@
worker_processes auto;
events {
worker_connections 32000;
}
http {
server {
listen 80;
server_name localhost;
root /usr/share/nginx/html;
index index.html index.htm;
include /etc/nginx/mime.types;
# Optimizations for OpenSlides
client_max_body_size 100M;
proxy_connect_timeout 300s;
proxy_read_timeout 300s;
proxy_set_header Host $http_host;
gzip on;
gzip_min_length 1000;
gzip_proxied expired no-cache no-store private auth;
gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript;
location /apps {
proxy_pass http://server:8000;
}
location /media/ {
proxy_pass http://media:8000;
}
location /rest {
proxy_pass http://server:8000;
}
location /ws {
proxy_pass http://server:8000;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
}
location /server-version.txt {
proxy_pass http://server:8000;
}
location = /basic_status {
stub_status;
}
location / {
try_files $uri $uri/ /index.html;
}
}
}

78
docker/.env Normal file
View File

@ -0,0 +1,78 @@
# OpenSlides instance configuration
#
# As well as environment variables for various services, this file contains
# variables used to persist custom settings for docker-compose.yml or
# docker-stack.yml. See the preamble of a docker-compose.yml.m4 or
# docker-stack.yml.m4 template for more information.
#
# Most variables are listed here only to facilitate discovery of the available
# options. Empty values cause the template's defaults to be inserted.
# General
# -------
INSTANCE_DOMAIN=
PROJECT_STACK_NAME=
EXTERNAL_HTTP_PORT=
DEFAULT_DOCKER_REGISTRY=
# Docker Images
# -------------
DOCKER_OPENSLIDES_BACKEND_NAME=
DOCKER_OPENSLIDES_BACKEND_TAG=
DOCKER_OPENSLIDES_FRONTEND_NAME=
DOCKER_OPENSLIDES_FRONTEND_TAG=
# Database
# --------
PGNODE_2_ENABLED=
PGNODE_3_ENABLED=
PGNODE_REPMGR_PRIMARY=
PGNODE_WAL_ARCHIVING=
PGNODE_1_PLACEMENT_CONSTR=
PGNODE_2_PLACEMENT_CONSTR=
PGNODE_3_PLACEMENT_CONSTR=
PGBOUNCER_PLACEMENT_CONSTR=
# Service Replication
# -------------------
OPENSLIDES_BACKEND_SERVICE_REPLICAS=
OPENSLIDES_FRONTEND_SERVICE_REPLICAS=
REDIS_RO_SERVICE_REPLICAS=
MEDIA_SERVICE_REPLICAS=
# E-Mail
# ------
DEFAULT_FROM_EMAIL=
POSTFIX_MYHOSTNAME=
POSTFIX_RELAYHOST=
# OpenSlides Backend settings (settings.py)
# -----------------------------------------
# Features
ENABLE_SAML=
ENABLE_ELECTRONIC_VOTING=
# Connections
AUTOUPDATE_DELAY=
CONNECTION_POOL_LIMIT=
DATABASE_HOST=
DATABASE_PASSWORD=
DATABASE_PORT=
DATABASE_USER=
EMAIL_HOST=
EMAIL_HOST_PASSWORD=
EMAIL_HOST_USER=
EMAIL_PORT=
JITSI_DOMAIN=
JITSI_PASSWORD=
JITSI_ROOM_NAME=
REDIS_CHANNLES_HOST=
REDIS_CHANNLES_PORT=
REDIS_HOST=
REDIS_PORT=
REDIS_SLAVE_HOST=
REDIS_SLAVE_PORT=
REDIS_SLAVE_WAIT_TIMEOUT=
# Logging
DJANGO_LOG_LEVEL=
OPENSLIDES_LOG_LEVEL=
RESET_PASSWORD_VERBOSE_ERRORS=

134
docker/build.sh Executable file
View File

@ -0,0 +1,134 @@
#!/bin/bash
set -e
declare -A TARGETS
TARGETS=(
[client]="$(dirname "${BASH_SOURCE[0]}")/../client/docker/"
[server]="$(dirname "${BASH_SOURCE[0]}")/../server/docker/"
[media-service]="https://github.com/OpenSlides/openslides-media-service.git"
[pgbouncer]="https://github.com/OpenSlides/openslides-docker-compose.git#:pgbouncer"
[postfix]="https://github.com/OpenSlides/openslides-docker-compose.git#:postfix"
[repmgr]="https://github.com/OpenSlides/openslides-docker-compose.git#:repmgr"
)
DOCKER_REPOSITORY="openslides"
DOCKER_TAG="latest"
CONFIG="/etc/osinstancectl"
OPTIONS=()
BUILT_IMAGES=()
DEFAULT_TARGETS=(server client)
usage() {
cat << EOF
Usage: $(basename ${BASH_SOURCE[0]}) [<options>] <service>...
Options:
-D, --docker-repo Specify a Docker repository
(default: unspecified, i.e., system default)
-t, --tag Tag the Docker image (default: $DOCKER_TAG)
--ask-push Offer to push newly built images to registry
--no-cache Pass --no-cache to docker-build
EOF
}
# Config file
if [[ -f "$CONFIG" ]]; then
echo "Found ${CONFIG} file."
source "$CONFIG"
fi
shortopt="hr:D:t:"
longopt="help,docker-repo:,tag:,ask-push,no-cache"
ARGS=$(getopt -o "$shortopt" -l "$longopt" -n "$ME" -- "$@")
if [ $? -ne 0 ]; then usage; exit 1; fi
eval set -- "$ARGS";
unset ARGS
# Parse options
while true; do
case "$1" in
-D|--docker-repo)
DOCKER_REPOSITORY="$2"
shift 2
;;
-t|--tag)
DOCKER_TAG="$2"
shift 2
;;
--ask-push)
ASK_PUSH=1
shift 1
;;
--no-cache)
OPTIONS+="--no-cache"
shift 1
;;
-h|--help) usage; exit 0 ;;
--) shift ; break ;;
*) usage; exit 1 ;;
esac
done
SELECTED_TARGETS=($@)
[[ "${#SELECTED_TARGETS[@]}" -ge 1 ]] || SELECTED_TARGETS=("${DEFAULT_TARGETS[@]}")
[[ "${SELECTED_TARGETS[@]}" != "all" ]] || SELECTED_TARGETS=("${!TARGETS[@]}")
for i in "${SELECTED_TARGETS[@]}"; do
loc="${TARGETS[$i]}"
[[ -n "$loc" ]] || {
echo "ERROR: Cannot build ${i}: not configured."
continue
}
img_name="openslides-${i}"
img="${img_name}:${DOCKER_TAG}"
if [[ -n "$DOCKER_REPOSITORY" ]]; then
img="${DOCKER_REPOSITORY}/${img}"
fi
echo "Building $img..."
# Special instructions for local services
build_script="$(dirname "${BASH_SOURCE[0]}")/../${i}/build.sh"
if [[ -f "$build_script" ]]; then
. "$build_script"
else
docker build --tag "$img" --pull "${OPTIONS[@]}" "$loc"
fi
BUILT_IMAGES+=("$img ON")
done
if [[ "${#BUILT_IMAGES[@]}" -ge 1 ]]; then
printf "\nSuccessfully built images:\n\n"
for i in "${BUILT_IMAGES[@]}"; do
read -r img x <<< "$i"
printf " - $img\n"
done
else
echo "No images were built."
exit 3
fi
[[ "$ASK_PUSH" ]] || exit 0
if hash whiptail > /dev/null 2>&1; then
while read img; do
echo "Pushing ${img}."
docker push "$img"
done < <( whiptail --title "OpenSlides build script" \
--checklist "Select images to push to their registry." \
25 78 16 --separate-output --noitem --clear \
${BUILT_IMAGES[@]} \
3>&2 2>&1 1>&3 )
else
echo
for i in "${BUILT_IMAGES[@]}"; do
read -r img x <<< "$i"
read -p "Push image '$img' to repository? [Y/n] " REPL
case "$REPL" in
N|n|No|no|NO) exit 0;;
*) docker push "$img" ;;
esac
done
fi

View File

@ -0,0 +1,242 @@
dnl This is a YAML template file. Simply translate it with m4 to create
dnl a standard configuration. Customizations can and should be added in .env
dnl by setting the appropriate variables.
dnl
dnl Usage:
dnl m4 docker-compose.yml.m4 > docker-compose.yml
dnl ( set -a; source .env; m4 docker-compose.yml.m4 ) > docker-compose.yml
dnl
dnl ----------------------------------------
divert(-1)dnl
define(`read_env', `esyscmd(`printf "%s" "$$1"')')
define(`ifenvelse', `ifelse(read_env(`$1'),, `$2', read_env(`$1'))')
define(`BACKEND_IMAGE',
ifenvelse(`DOCKER_OPENSLIDES_BACKEND_NAME', openslides/openslides-server):dnl
ifenvelse(`DOCKER_OPENSLIDES_BACKEND_TAG', latest))
define(`FRONTEND_IMAGE',
ifenvelse(`DOCKER_OPENSLIDES_FRONTEND_NAME', openslides/openslides-client):dnl
ifenvelse(`DOCKER_OPENSLIDES_FRONTEND_TAG', latest))
define(`PRIMARY_DB', `ifenvelse(`PGNODE_REPMGR_PRIMARY', pgnode1)')
define(`PGBOUNCER_NODELIST',
`ifelse(read_env(`PGNODE_2_ENABLED'), 1, `,pgnode2')`'dnl
ifelse(read_env(`PGNODE_3_ENABLED'), 1, `,pgnode3')')
define(`PROJECT_DIR', ifdef(`PROJECT_DIR',PROJECT_DIR,.))
define(`ADMIN_SECRET_AVAILABLE', `syscmd(`test -f 'PROJECT_DIR`/secrets/adminsecret.env')sysval')
define(`USER_SECRET_AVAILABLE', `syscmd(`test -f 'PROJECT_DIR`/secrets/usersecret.env')sysval')
divert(0)dnl
dnl ----------------------------------------
# This configuration was created from a template file. Before making changes,
# please make sure that you do not have a process in place that would override
# your changes in the future. The accompanying .env file might be the correct
# place for customizations instead.
version: '3.4'
x-osserver:
&default-osserver
image: BACKEND_IMAGE
networks:
- front
- back
restart: always
x-osserver-env: &default-osserver-env
AMOUNT_REPLICAS: ifenvelse(`REDIS_RO_SERVICE_REPLICAS', 1)
AUTOUPDATE_DELAY: ifenvelse(`AUTOUPDATE_DELAY', 1)
CONNECTION_POOL_LIMIT: ifenvelse(`CONNECTION_POOL_LIMIT', 100)
DATABASE_HOST: "ifenvelse(`DATABASE_HOST', pgbouncer)"
DATABASE_PASSWORD: "ifenvelse(`DATABASE_PASSWORD', openslides)"
DATABASE_PORT: ifenvelse(`DATABASE_PORT', 5432)
DATABASE_USER: "ifenvelse(`DATABASE_USER', openslides)"
DEFAULT_FROM_EMAIL: "ifenvelse(`DEFAULT_FROM_EMAIL', noreply@example.com)"
DJANGO_LOG_LEVEL: "ifenvelse(`DJANGO_LOG_LEVEL', INFO)"
EMAIL_HOST: "ifenvelse(`EMAIL_HOST', postfix)"
EMAIL_HOST_PASSWORD: "ifenvelse(`EMAIL_HOST_PASSWORD',)"
EMAIL_HOST_USER: "ifenvelse(`EMAIL_HOST_USER',)"
EMAIL_PORT: ifenvelse(`EMAIL_PORT', 25)
ENABLE_ELECTRONIC_VOTING: "ifenvelse(`ENABLE_ELECTRONIC_VOTING', False)"
ENABLE_SAML: "ifenvelse(`ENABLE_SAML', False)"
INSTANCE_DOMAIN: "ifenvelse(`INSTANCE_DOMAIN', http://example.com:8000)"
JITSI_DOMAIN: "ifenvelse(`JITSI_DOMAIN', None)"
JITSI_PASSWORD: "ifenvelse(`JITSI_PASSWORD', None)"
JITSI_ROOM_NAME: "ifenvelse(`JITSI_ROOM_NAME', None)"
OPENSLIDES_LOG_LEVEL: "ifenvelse(`OPENSLIDES_LOG_LEVEL', INFO)"
REDIS_CHANNLES_HOST: "ifenvelse(`REDIS_CHANNLES_HOST', redis-channels)"
REDIS_CHANNLES_PORT: ifenvelse(`REDIS_CHANNLES_PORT', 6379)
REDIS_HOST: "ifenvelse(`REDIS_HOST', redis)"
REDIS_PORT: ifenvelse(`REDIS_PORT', 6379)
REDIS_SLAVE_HOST: "ifenvelse(`REDIS_SLAVE_HOST', redis-slave)"
REDIS_SLAVE_PORT: ifenvelse(`REDIS_SLAVE_PORT', 6379)
REDIS_SLAVE_WAIT_TIMEOUT: ifenvelse(`REDIS_SLAVE_WAIT_TIMEOUT', 10000)
RESET_PASSWORD_VERBOSE_ERRORS: "ifenvelse(`RESET_PASSWORD_VERBOSE_ERRORS', False)"
x-pgnode: &default-pgnode
image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-repmgr:latest
networks:
- dbnet
labels:
org.openslides.role: "postgres"
restart: always
x-pgnode-env: &default-pgnode-env
REPMGR_RECONNECT_ATTEMPTS: 30
REPMGR_RECONNECT_INTERVAL: 10
REPMGR_WAL_ARCHIVE: "ifenvelse(`PGNODE_WAL_ARCHIVING', on)"
services:
server:
<< : *default-osserver
# Below is the default command. You can uncomment it to override the
# number of workers, for example:
# command: "gunicorn -w 8 --preload -b 0.0.0.0:8000
# -k uvicorn.workers.UvicornWorker openslides.asgi:application"
#
# Uncomment the following line to use daphne instead of gunicorn:
# command: "daphne -b 0.0.0.0 -p 8000 openslides.asgi:application"
depends_on:
- server-setup
environment:
<< : *default-osserver-env
secrets:
- django
ifelse(read_env(`ENABLE_SAML'), `True',- saml_cert
- saml_key
- saml_config)
ifelse(read_env(`OPENSLIDES_BACKEND_SERVICE_REPLICAS'),,,deploy:
replicas: ifenvelse(`OPENSLIDES_BACKEND_SERVICE_REPLICAS', 1))
server-setup:
<< : *default-osserver
entrypoint: /usr/local/sbin/entrypoint-db-setup
environment:
<< : *default-osserver-env
secrets:
- django
ifelse(ADMIN_SECRET_AVAILABLE, 0,- os_admin)
ifelse(USER_SECRET_AVAILABLE, 0,- os_user)
ifelse(read_env(`ENABLE_SAML'), `True',- saml_cert
- saml_key
- saml_config)
depends_on:
- pgbouncer
- redis
- redis-slave
- redis-channels
client:
image: FRONTEND_IMAGE
restart: always
depends_on:
- server
networks:
- front
ports:
- "127.0.0.1:ifenvelse(`EXTERNAL_HTTP_PORT', 8000):80"
pgnode1:
<< : *default-pgnode
environment:
<< : *default-pgnode-env
REPMGR_NODE_ID: 1
REPMGR_PRIMARY: ifelse(PRIMARY_DB, pgnode1, `# This is the primary', PRIMARY_DB)
volumes:
- "dbdata1:/var/lib/postgresql"
ifelse(read_env(`PGNODE_2_ENABLED'), 1, `'
pgnode2:
<< : *default-pgnode
environment:
<< : *default-pgnode-env
REPMGR_NODE_ID: 2
REPMGR_PRIMARY: ifelse(PRIMARY_DB, pgnode2, `# This is the primary', PRIMARY_DB)
volumes:
- "dbdata2:/var/lib/postgresql")
ifelse(read_env(`PGNODE_3_ENABLED'), 1, `'
pgnode3:
<< : *default-pgnode
environment:
<< : *default-pgnode-env
REPMGR_NODE_ID: 3
REPMGR_PRIMARY: ifelse(PRIMARY_DB, pgnode3, `# This is the primary', PRIMARY_DB)
volumes:
- "dbdata3:/var/lib/postgresql")
pgbouncer:
environment:
- PG_NODE_LIST=pgnode1`'PGBOUNCER_NODELIST
image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-pgbouncer:latest
restart: always
networks:
back:
aliases:
- db
- postgres
dbnet:
postfix:
image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-postfix:latest
restart: always
environment:
MYHOSTNAME: "ifenvelse(`POSTFIX_MYHOSTNAME', localhost)"
RELAYHOST: "ifenvelse(`POSTFIX_RELAYHOST', localhost)"
networks:
- back
redis:
image: redis:alpine
restart: always
networks:
back:
aliases:
- rediscache
redis-slave:
image: redis:alpine
restart: always
command: ["redis-server", "--save", "", "--slaveof", "redis", "6379"]
depends_on:
- redis
networks:
back:
aliases:
- rediscache-slave
ifelse(read_env(`REDIS_RO_SERVICE_REPLICAS'),,,deploy:
replicas: ifenvelse(`REDIS_RO_SERVICE_REPLICAS', 1))
redis-channels:
image: redis:alpine
restart: always
networks:
back:
media:
image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-media-service:latest
environment:
- CHECK_REQUEST_URL=server:8000/check-media/
restart: always
networks:
front:
back:
# Override command to run more workers per task
# command: ["gunicorn", "-w", "4", "--preload", "-b",
# "0.0.0.0:8000", "src.mediaserver:app"]
volumes:
dbdata1:
ifelse(read_env(`PGNODE_2_ENABLED'), 1, ` dbdata2:')
ifelse(read_env(`PGNODE_3_ENABLED'), 1, ` dbdata3:')
networks:
front:
back:
dbnet:
secrets:
django:
file: ./secrets/django.env
ifelse(ADMIN_SECRET_AVAILABLE, 0,os_admin:
file: ./secrets/adminsecret.env)
ifelse(USER_SECRET_AVAILABLE, 0,os_user:
file: ./secrets/usersecret.env)
ifelse(read_env(`ENABLE_SAML'), `True', saml_cert:
file: ./secrets/saml/sp.crt
saml_key:
file: ./secrets/saml/sp.key
saml_config:
file: ./secrets/saml/saml_settings.json)
# vim: set sw=2 et:

276
docker/docker-stack.yml.m4 Normal file
View File

@ -0,0 +1,276 @@
dnl This is a YAML template file. Simply translate it with m4 to create
dnl a standard configuration. Customizations can and should be added in .env
dnl by setting the appropriate variables.
dnl
dnl Usage:
dnl m4 docker-stack.yml.m4 > docker-stack.yml
dnl ( set -a; source .env; m4 docker-stack.yml.m4 ) > docker-stack.yml
dnl
dnl ----------------------------------------
divert(-1)dnl
define(`read_env', `esyscmd(`printf "%s" "$$1"')')
define(`ifenvelse', `ifelse(read_env(`$1'),, `$2', read_env(`$1'))')
define(`BACKEND_IMAGE',
ifenvelse(`DOCKER_OPENSLIDES_BACKEND_NAME', openslides/openslides-server):dnl
ifenvelse(`DOCKER_OPENSLIDES_BACKEND_TAG', latest))
define(`FRONTEND_IMAGE',
ifenvelse(`DOCKER_OPENSLIDES_FRONTEND_NAME', openslides/openslides-client):dnl
ifenvelse(`DOCKER_OPENSLIDES_FRONTEND_TAG', latest))
define(`PRIMARY_DB', `ifenvelse(`PGNODE_REPMGR_PRIMARY', pgnode1)')
define(`PGBOUNCER_NODELIST',
`ifelse(read_env(`PGNODE_2_ENABLED'), 1, `,pgnode2')`'dnl
ifelse(read_env(`PGNODE_3_ENABLED'), 1, `,pgnode3')')
define(`PROJECT_DIR', ifdef(`PROJECT_DIR',PROJECT_DIR,.))
define(`ADMIN_SECRET_AVAILABLE', `syscmd(`test -f 'PROJECT_DIR`/secrets/adminsecret.env')sysval')
define(`USER_SECRET_AVAILABLE', `syscmd(`test -f 'PROJECT_DIR`/secrets/usersecret.env')sysval')
divert(0)dnl
dnl ----------------------------------------
# This configuration was created from a template file. Before making changes,
# please make sure that you do not have a process in place that would override
# your changes in the future. The accompanying .env file might be the correct
# place for customizations instead.
version: '3.4'
x-osserver:
&default-osserver
image: BACKEND_IMAGE
networks:
- front
- back
x-osserver-env: &default-osserver-env
AMOUNT_REPLICAS: ifenvelse(`REDIS_RO_SERVICE_REPLICAS', 3)
AUTOUPDATE_DELAY: ifenvelse(`AUTOUPDATE_DELAY', 1)
CONNECTION_POOL_LIMIT: ifenvelse(`CONNECTION_POOL_LIMIT', 100)
DATABASE_HOST: "ifenvelse(`DATABASE_HOST', pgbouncer)"
DATABASE_PASSWORD: "ifenvelse(`DATABASE_PASSWORD', openslides)"
DATABASE_PORT: ifenvelse(`DATABASE_PORT', 5432)
DATABASE_USER: "ifenvelse(`DATABASE_USER', openslides)"
DEFAULT_FROM_EMAIL: "ifenvelse(`DEFAULT_FROM_EMAIL', noreply@example.com)"
DJANGO_LOG_LEVEL: "ifenvelse(`DJANGO_LOG_LEVEL', INFO)"
EMAIL_HOST: "ifenvelse(`EMAIL_HOST', postfix)"
EMAIL_HOST_PASSWORD: "ifenvelse(`EMAIL_HOST_PASSWORD',)"
EMAIL_HOST_USER: "ifenvelse(`EMAIL_HOST_USER',)"
EMAIL_PORT: ifenvelse(`EMAIL_PORT', 25)
ENABLE_ELECTRONIC_VOTING: "ifenvelse(`ENABLE_ELECTRONIC_VOTING', False)"
ENABLE_SAML: "ifenvelse(`ENABLE_SAML', False)"
INSTANCE_DOMAIN: "ifenvelse(`INSTANCE_DOMAIN', http://example.com:8000)"
JITSI_DOMAIN: "ifenvelse(`JITSI_DOMAIN', None)"
JITSI_PASSWORD: "ifenvelse(`JITSI_PASSWORD', None)"
JITSI_ROOM_NAME: "ifenvelse(`JITSI_ROOM_NAME', None)"
OPENSLIDES_LOG_LEVEL: "ifenvelse(`OPENSLIDES_LOG_LEVEL', INFO)"
REDIS_CHANNLES_HOST: "ifenvelse(`REDIS_CHANNLES_HOST', redis-channels)"
REDIS_CHANNLES_PORT: ifenvelse(`REDIS_CHANNLES_PORT', 6379)
REDIS_HOST: "ifenvelse(`REDIS_HOST', redis)"
REDIS_PORT: ifenvelse(`REDIS_PORT', 6379)
REDIS_SLAVE_HOST: "ifenvelse(`REDIS_SLAVE_HOST', redis-slave)"
REDIS_SLAVE_PORT: ifenvelse(`REDIS_SLAVE_PORT', 6379)
REDIS_SLAVE_WAIT_TIMEOUT: ifenvelse(`REDIS_SLAVE_WAIT_TIMEOUT', 10000)
RESET_PASSWORD_VERBOSE_ERRORS: "ifenvelse(`RESET_PASSWORD_VERBOSE_ERRORS', False)"
x-pgnode: &default-pgnode
image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-repmgr:latest
networks:
- dbnet
labels:
org.openslides.role: "postgres"
deploy:
replicas: 1
x-pgnode-env: &default-pgnode-env
REPMGR_RECONNECT_ATTEMPTS: 30
REPMGR_RECONNECT_INTERVAL: 10
REPMGR_WAL_ARCHIVE: "ifenvelse(`PGNODE_WAL_ARCHIVING', on)"
services:
server:
<< : *default-osserver
# Below is the default command. You can uncomment it to override the
# number of workers, for example:
# command: "gunicorn -w 8 --preload -b 0.0.0.0:8000
# -k uvicorn.workers.UvicornWorker openslides.asgi:application"
#
# Uncomment the following line to use daphne instead of gunicorn:
# command: "daphne -b 0.0.0.0 -p 8000 openslides.asgi:application"
environment:
<< : *default-osserver-env
secrets:
- django
ifelse(read_env(`ENABLE_SAML'), `True',- saml_cert
- saml_key
- saml_config)
deploy:
restart_policy:
condition: on-failure
delay: 5s
replicas: ifenvelse(`OPENSLIDES_BACKEND_SERVICE_REPLICAS', 1)
server-setup:
<< : *default-osserver
entrypoint: /usr/local/sbin/entrypoint-db-setup
environment:
<< : *default-osserver-env
secrets:
- django
ifelse(ADMIN_SECRET_AVAILABLE, 0,- os_admin)
ifelse(USER_SECRET_AVAILABLE, 0,- os_user)
ifelse(read_env(`ENABLE_SAML'), `True',- saml_cert
- saml_key
- saml_config)
client:
image: FRONTEND_IMAGE
networks:
- front
ports:
- "0.0.0.0:ifenvelse(`EXTERNAL_HTTP_PORT', 8000):80"
deploy:
replicas: ifenvelse(`OPENSLIDES_FRONTEND_SERVICE_REPLICAS', 1)
restart_policy:
condition: on-failure
delay: 5s
pgnode1:
<< : *default-pgnode
environment:
<< : *default-pgnode-env
REPMGR_NODE_ID: 1
REPMGR_PRIMARY: ifelse(PRIMARY_DB, pgnode1, `# This is the primary', PRIMARY_DB)
deploy:
placement:
constraints: ifenvelse(`PGNODE_1_PLACEMENT_CONSTR', [node.labels.openslides-db == dbnode1])
volumes:
- "dbdata1:/var/lib/postgresql"
ifelse(read_env(`PGNODE_2_ENABLED'), 1, `'
pgnode2:
<< : *default-pgnode
environment:
<< : *default-pgnode-env
REPMGR_NODE_ID: 2
REPMGR_PRIMARY: ifelse(PRIMARY_DB, pgnode2, `# This is the primary', PRIMARY_DB)
deploy:
placement:
constraints: ifenvelse(`PGNODE_2_PLACEMENT_CONSTR', [node.labels.openslides-db == dbnode2])
volumes:
- "dbdata2:/var/lib/postgresql")
ifelse(read_env(`PGNODE_3_ENABLED'), 1, `'
pgnode3:
<< : *default-pgnode
environment:
<< : *default-pgnode-env
REPMGR_NODE_ID: 3
REPMGR_PRIMARY: ifelse(PRIMARY_DB, pgnode3, `# This is the primary', PRIMARY_DB)
deploy:
placement:
constraints: ifenvelse(`PGNODE_3_PLACEMENT_CONSTR', [node.labels.openslides-db == dbnode3])
volumes:
- "dbdata3:/var/lib/postgresql")
pgbouncer:
environment:
- PG_NODE_LIST=pgnode1`'PGBOUNCER_NODELIST
image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-pgbouncer:latest
networks:
back:
aliases:
- db
- postgres
dbnet:
deploy:
restart_policy:
condition: on-failure
delay: 10s
placement:
constraints: ifenvelse(`PGBOUNCER_PLACEMENT_CONSTR', [node.role == manager])
postfix:
image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-postfix:latest
environment:
MYHOSTNAME: "ifenvelse(`POSTFIX_MYHOSTNAME', localhost)"
RELAYHOST: "ifenvelse(`POSTFIX_RELAYHOST', localhost)"
networks:
- back
deploy:
restart_policy:
condition: on-failure
delay: 5s
replicas: 1
placement:
constraints: [node.role == manager]
redis:
image: redis:alpine
networks:
back:
aliases:
- rediscache
deploy:
replicas: 1
restart_policy:
condition: on-failure
delay: 5s
redis-slave:
image: redis:alpine
command: ["redis-server", "--save", "", "--slaveof", "redis", "6379"]
networks:
back:
aliases:
- rediscache-slave
deploy:
replicas: ifenvelse(`REDIS_RO_SERVICE_REPLICAS', 3)
restart_policy:
condition: on-failure
delay: 5s
redis-channels:
image: redis:alpine
networks:
back:
deploy:
replicas: 1
restart_policy:
condition: on-failure
delay: 5s
media:
image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-media-service:latest
environment:
- CHECK_REQUEST_URL=server:8000/check-media/
deploy:
replicas: ifenvelse(`MEDIA_SERVICE_REPLICAS', 8)
restart_policy:
condition: on-failure
delay: 10s
networks:
front:
back:
# Override command to run more workers per task
# command: ["gunicorn", "-w", "4", "--preload", "-b",
# "0.0.0.0:8000", "src.mediaserver:app"]
volumes:
dbdata1:
ifelse(read_env(`PGNODE_2_ENABLED'), 1, ` dbdata2:')
ifelse(read_env(`PGNODE_3_ENABLED'), 1, ` dbdata3:')
networks:
front:
back:
driver_opts:
encrypted: ""
dbnet:
driver_opts:
encrypted: ""
secrets:
django:
file: ./secrets/django.env
ifelse(ADMIN_SECRET_AVAILABLE, 0,os_admin:
file: ./secrets/adminsecret.env)
ifelse(USER_SECRET_AVAILABLE, 0,os_user:
file: ./secrets/usersecret.env)
ifelse(read_env(`ENABLE_SAML'), `True', saml_cert:
file: ./secrets/saml/sp.crt
saml_key:
file: ./secrets/saml/sp.key
saml_config:
file: ./secrets/saml/saml_settings.json)
# vim: set sw=2 et:

View File

@ -0,0 +1,3 @@
## secrets/adminsecret.env is sourced by the server container to set the initial
## admin user password.
# OPENSLIDES_ADMIN_PASSWORD="<securepassword>"

View File

@ -0,0 +1,3 @@
# Define a secret key for Django
# https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/#secret-key
DJANGO_SECRET_KEY=

View File

@ -0,0 +1,5 @@
## Example user credential configuration
# OPENSLIDES_USER_FIRSTNAME="John"
# OPENSLIDES_USER_LASTNAME="Doe"
# OPENSLIDES_USER_PASSWORD="<securepassword>"
# OPENSLIDES_USER_EMAIL="john@example.com"

View File

@ -1,32 +0,0 @@
from django.core.management.base import BaseCommand
from ...models import User
class Command(BaseCommand):
"""
Command to create an OpenSlides user.
"""
help = "Creates an OpenSlides user."
def add_arguments(self, parser):
parser.add_argument("first_name", help="The first name of the new user.")
parser.add_argument("last_name", help="The last name of the new user.")
parser.add_argument("username", help="The username of the new user.")
parser.add_argument("password", help="The password of the new user.")
parser.add_argument("groups_id", help="The group id of the new user.")
parser.add_argument("--email", help="The email address of the new user.")
def handle(self, *args, **options):
user_data = {
"first_name": options["first_name"],
"last_name": options["last_name"],
"default_password": options["password"],
"email": options["email"] or "",
}
user = User.objects.create_user(
options["username"], options["password"], skip_autoupdate=True, **user_data
)
if options["groups_id"].isdigit():
user.groups.add(int(options["groups_id"]))

2
server/.dockerignore Normal file
View File

@ -0,0 +1,2 @@
*.pyc
__pycache__/

19
server/build.sh Executable file
View File

@ -0,0 +1,19 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
service_name="$(basename "$PWD")"
version_file="docker/${service_name}-version.txt"
{
printf "${service_name^} built on %s:\n\n" "$(date)"
printf "Branch: %s\n" "$(git rev-parse --abbrev-ref HEAD)"
printf '\n'
git show -s --format=raw
} > "$version_file"
docker build --tag "${img:-openslides/openslides-${service_name}:latest}" \
--pull "${OPTIONS[@]}" -f docker/Dockerfile .
rm "$version_file" || true
unset version_file
unset service_name

72
server/docker/Dockerfile Normal file
View File

@ -0,0 +1,72 @@
FROM python:3.7-slim AS base
ENV DEBIAN_FRONTEND noninteractive
ENV DEBIAN_PRIORITY critical
ENV DEBCONF_NOWARNINGS yes
ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE 1
# Variables relevant for CMD
ENV DJANGO_SETTINGS_MODULE settings
ENV PYTHONPATH personal_data/var/
RUN mkdir -p /app
WORKDIR /app
RUN useradd -m openslides
RUN chown -R openslides /app
RUN apt-get -y update && apt-get -y upgrade && \
apt-get install --no-install-recommends -y \
apt-transport-https \
bzip2 \
curl \
g++ \
gcc \
git \
gnupg2 \
libpq-dev \
make \
postgresql-client \
rsync \
wait-for-it \
wget \
xz-utils
# Install additional tools
RUN apt-get install --no-install-recommends -y \
dnsutils \
iputils-ping \
netcat \
procps \
traceroute
# Install saml requirements
RUN apt-get install --no-install-recommends -y \
libxml2-dev \
libxmlsec1-dev \
libxmlsec1-openssl \
pkg-config
RUN rm -rf /var/lib/apt/lists/*
COPY requirements /app/requirements
RUN pip install -r requirements/production.txt -r requirements/big_mode.txt \
-r requirements/saml.txt && \
rm -rf /root/.cache/pip
# SAML
COPY docker/saml-setup.sh /usr/local/lib/
RUN mkdir -p /app/personal_data/var/certs/ && \
chown -R openslides:openslides /app/personal_data/var/
USER openslides
# the `empty` folder is used for the dummy http server für the migrate entrypoint to serve no files.
RUN mkdir /app/empty
COPY docker/entrypoint /usr/local/sbin/
COPY docker/entrypoint-db-setup /usr/local/sbin/
COPY docker/settings.py /app/personal_data/var/settings.py
COPY manage.py /app/
COPY openslides /app/openslides
COPY docker/server-version.txt /app/openslides/core/static/server-version.txt
ENTRYPOINT ["/usr/local/sbin/entrypoint"]
CMD ["gunicorn", "-w", "8", "--preload", "-b", "0.0.0.0:8000", "-k", \
"uvicorn.workers.UvicornWorker", "openslides.asgi:application"]

26
server/docker/entrypoint Executable file
View File

@ -0,0 +1,26 @@
#!/bin/bash
set -e
# Set DJANGO_SECRET_KEY variable
source /run/secrets/django
[[ -n "$DJANGO_SECRET_KEY" ]] || {
echo "ERROR: Django secret key undefined! Cannot continue."
sleep 5
exit 2
}
export SECRET_KEY="$DJANGO_SECRET_KEY"
# SAML setup
. /usr/local/lib/saml-setup.sh
# TODO: env variable for this host
wait-for-it -t 0 "server-setup:8000"
printf 'Executing server: "%s"\n' "$*"
# Expected commands are one of:
# - daphne -b 0.0.0.0 -p 8000 openslides.asgi:application
# - gunicorn -w 4 -b 0.0.0.0:8000 -k uvicorn.workers.UvicornWorker \
# openslides.asgi:application
exec "$@"

View File

@ -0,0 +1,89 @@
#!/bin/bash
set -e
warn_insecure_admin() {
cat <<-EOF
==============================================
WARNING
==============================================
WARNING: INSECURE ADMIN ACCOUNT CONFIGURATION!
EOF
sleep 10
}
# Set DJANGO_SECRET_KEY variable
source /run/secrets/django
[[ -n "$DJANGO_SECRET_KEY" ]] || {
echo "ERROR: Django secret key undefined! Cannot continue."
sleep 5
exit 2
}
export SECRET_KEY="$DJANGO_SECRET_KEY"
# Configure database
# TODO: env variables??
echo "postgres:5432:instancecfg:openslides:openslides" > "${HOME}/.pgpass"
chmod 600 "${HOME}/.pgpass"
until pg_isready -h db; do
echo "Waiting for Postgres cluster to become available..."
sleep 3
done
# Wait for redis
wait-for-it redis:6379
wait-for-it redis-slave:6379
wait-for-it redis-channels:6379
echo 'running migrations'
python manage.py migrate
# Admin
if [[ -f /run/secrets/os_admin ]]; then
echo "Retrieving secure admin password"
source /run/secrets/os_admin
if [[ -n "${OPENSLIDES_ADMIN_PASSWORD}" ]]; then
echo "Changing admin password"
python manage.py changedefaultadminpassword "${OPENSLIDES_ADMIN_PASSWORD}"
else
warn_insecure_admin
fi
else
warn_insecure_admin
fi
# Main user
if [[ -f /run/secrets/os_user ]]; then
echo "Retrieving secure user credentials"
source /run/secrets/os_user
if [[ -n "${OPENSLIDES_USER_FIRSTNAME}" ]] &&
[[ -n "${OPENSLIDES_USER_LASTNAME}" ]] &&
[[ -n "${OPENSLIDES_USER_PASSWORD}" ]]; then
user_name="${OPENSLIDES_USER_FIRSTNAME} ${OPENSLIDES_USER_LASTNAME}"
echo "Adding user: ${user_name}"
# createopenslidesuser: error: the following arguments are required:
# first_name, last_name, username, password, groups_id
# email is optional
# userid forces to to only create a user with this id, if it not exists before.
python manage.py createopenslidesuser \
--userid 2 \
--email "${OPENSLIDES_USER_EMAIL:-}" \
"${OPENSLIDES_USER_FIRSTNAME}" \
"${OPENSLIDES_USER_LASTNAME}" \
"${user_name}" \
"${OPENSLIDES_USER_PASSWORD}" \
2
else
echo "Incomplete user account data. Skipping account creation."
fi
fi
# SAML setup
. /usr/local/lib/saml-setup.sh
echo "Done migrating and setting up user accounts..."
python -m http.server --directory /app/empty --bind 0.0.0.0 8000

View File

@ -0,0 +1,10 @@
# SAML setup
if [[ "$ENABLE_SAML" = True ]]; then
echo "Setting up SAML"
for i in /run/secrets/saml_{cert,key,config}; do
[[ -f "$i" ]] || { echo "ERROR: $i not found!"; exit 3; }
done
ln -s /run/secrets/saml_cert /app/personal_data/var/certs/sp.crt
ln -s /run/secrets/saml_key /app/personal_data/var/certs/sp.key
ln -s /run/secrets/saml_config /app/personal_data/var/saml_settings.json
fi

170
server/docker/settings.py Normal file
View File

@ -0,0 +1,170 @@
"""
Settings file for OpenSlides.
For more information on this file, see
https://github.com/OpenSlides/OpenSlides/blob/master/SETTINGS.rst
"""
import os
from openslides.global_settings import *
class MissingEnvironmentVariable(Exception):
pass
undefined = object()
def get_env(name, default=undefined, cast=str):
env = os.environ.get(name)
default_extension = ""
if env is None:
env = default
default_extension = " (default)"
if env is undefined:
raise MissingEnvironmentVariable(name)
if cast is bool:
env = env in ("1", "true", "True")
else:
env = cast(env)
if env is None:
print(f"{name}={default_extension}", flush=True)
else:
print(f'{name}="{env}"{default_extension}', flush=True)
return env
# The directory for user specific data files
OPENSLIDES_USER_DATA_DIR = "/app/personal_data/var"
SECRET_KEY = get_env("SECRET_KEY")
DEBUG = False
# Controls the verbosity on errors during a reset password. If enabled, an error
# will be shown, if there does not exist a user with a given email address. So one
# can check, if a email is registered. If this is not wanted, disable verbose
# messages. An success message will always be shown.
RESET_PASSWORD_VERBOSE_ERRORS = get_env("RESET_PASSWORD_VERBOSE_ERRORS", True, bool)
AUTOUPDATE_DELAY = get_env("AUTOUPDATE_DELAY", 1, int)
# Email settings
# For SSL/TLS specific settings see https://docs.djangoproject.com/en/1.11/topics/email/#smtp-backend
EMAIL_HOST = get_env("EMAIL_HOST", "postfix")
EMAIL_PORT = get_env("EMAIL_PORT", 25, int)
EMAIL_HOST_USER = get_env("EMAIL_HOST_USER", "")
EMAIL_HOST_PASSWORD = get_env("EMAIL_HOST_PASSWORD", "")
DEFAULT_FROM_EMAIL = get_env("DEFAULT_FROM_EMAIL", "noreply@example.com")
# Increasing Upload size to 100mb (default is 2.5mb)
DATA_UPLOAD_MAX_MEMORY_SIZE = 104857600
# Database
# https://docs.djangoproject.com/en/1.10/ref/settings/#databases
DATABASES = {
"default": {
"ENGINE": "django.db.backends.postgresql",
"NAME": "openslides",
"USER": get_env("DATABASE_USER", "openslides"),
"PASSWORD": get_env("DATABASE_PASSWORD", "openslides"),
"HOST": get_env("DATABASE_HOST", "db"),
"PORT": get_env("DATABASE_PORT", "5432"),
},
"mediafiles": {
"ENGINE": "django.db.backends.postgresql",
"NAME": "mediafiledata",
"USER": get_env("DATABASE_USER", "openslides"),
"PASSWORD": get_env("DATABASE_PASSWORD", "openslides"),
"HOST": get_env("DATABASE_HOST", "db"),
"PORT": get_env("DATABASE_PORT", "5432"),
},
}
# Redis
REDIS_HOST = get_env("REDIS_HOST", "redis")
REDIS_PORT = get_env("REDIS_PORT", 6379, int)
REDIS_SLAVE_HOST = get_env("REDIS_SLAVE_HOST", "redis-slave")
REDIS_SLAVE_PORT = get_env("REDIS_SLAVE_PORT", 6379, int)
REDIS_CHANNLES_HOST = get_env("REDIS_CHANNLES_HOST", "redis-channels")
REDIS_CHANNLES_PORT = get_env("REDIS_CHANNLES_PORT", 6379, int)
REDIS_SLAVE_WAIT_TIMEOUT = get_env("REDIS_SLAVE_WAIT_TIMEOUT", 10000, int)
# Django Channels
CHANNEL_LAYERS = {
"default": {
"BACKEND": "channels_redis.core.RedisChannelLayer",
"CONFIG": {
"hosts": [(REDIS_CHANNLES_HOST, REDIS_CHANNLES_PORT)],
"capacity": 10000,
},
},
}
# Collection Cache
REDIS_ADDRESS = f"redis://{REDIS_HOST}:{REDIS_PORT}/0"
REDIS_READ_ONLY_ADDRESS = f"redis://{REDIS_SLAVE_HOST}:{REDIS_SLAVE_PORT}/0"
AMOUNT_REPLICAS = get_env("AMOUNT_REPLICAS", 1)
CONNECTION_POOL_LIMIT = get_env("CONNECTION_POOL_LIMIT", 100, int)
# Session backend
SESSION_ENGINE = "redis_sessions.session"
SESSION_REDIS = {
"host": REDIS_HOST,
"port": REDIS_PORT,
"db": 0,
"prefix": "session",
"socket_timeout": 2,
}
# SAML integration
ENABLE_SAML = get_env("ENABLE_SAML", False, bool)
if ENABLE_SAML:
INSTALLED_APPS += ["openslides.saml"]
# TODO: More saml stuff...
# Controls if electronic voting (means non-analog polls) are enabled.
ENABLE_ELECTRONIC_VOTING = get_env("ENABLE_ELECTRONIC_VOTING", False, bool)
# Jitsi integration
JITSI_DOMAIN = get_env("JITSI_DOMAIN", None)
JITSI_ROOM_NAME = get_env("JITSI_ROOM_NAME", None)
JITSI_PASSWORD = get_env("JITSI_PASSWORD", None)
TIME_ZONE = "Europe/Berlin"
STATICFILES_DIRS = [os.path.join(OPENSLIDES_USER_DATA_DIR, "static")] + STATICFILES_DIRS
STATIC_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, "collected-static")
MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, "media", "")
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"gunicorn": {
"format": "{asctime} [{process:d}] [{levelname}] {name} {message}",
"style": "{",
"datefmt": "[%Y-%m-%d %H:%M:%S %z]",
},
},
"handlers": {
"console": {"class": "logging.StreamHandler", "formatter": "gunicorn",},
},
"loggers": {
"django": {
"handlers": ["console"],
"level": get_env("DJANGO_LOG_LEVEL", "INFO"),
},
"openslides": {
"handlers": ["console"],
"level": get_env("OPENSLIDES_LOG_LEVEL", "INFO"),
},
},
}
SETTINGS_FILEPATH = __file__

Some files were not shown because too many files have changed in this diff Show More