Create portable win32 binary distribution

extras/win32-portable/create_portable.txt

@@ -0,0 +1,12 @@
+How to create a new portable windows distribution of openslides:
+----------------------------------------------------------------
+1.) Follow the openslides installation instructions for windows, but add
+    the option "-Z" when executing easy_install e.g.:
+      easy_install -Z django django-mptt reportlab pil
+
+2.) in the main directory of the openslides checkout execute:
+      python extras/win32-portable/prepare_portable.py
+
+
+3.) The portable openslides distribution is now ready as a zip in the dist/
+    directory

extras/win32-portable/openslides.c

@@ -0,0 +1,125 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#define _WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+#include <Python.h>
+
+static const char *site_code =
+    "import sys;"
+    "import os;"
+    "import site;"
+    "path = os.path.dirname(sys.executable);"
+    "site_dir = os.path.join(path, \"site-packages\");"
+    "site.addsitedir(site_dir);"
+    "sys.path.append(path)";
+
+static const char *run_openslides_code =
+    "import openslides.main;"
+    "openslides.main.main()";
+
+/* determine the path to the executable
+ * NOTE: Py_GetFullProgramPath() can't be used because
+ *       this would trigger pythons search-path initialization
+ *       But we need this to initialize PYTHONHOME before this happens
+ */
+static char *
+_get_module_name()
+{
+    size_t size = 1;
+    char *name = NULL;
+    int i;
+
+    /* a path > 40k would be insane, it is more likely something
+     * else has gone very wrong on the system
+     */
+    for (i = 0;i < 10; i++)
+    {
+	DWORD res;
+	char *n;
+
+	n = realloc(name, size);
+	if (!n)
+	{
+	    free(name);
+	    return NULL;
+	}
+	name = n;
+
+	res = GetModuleFileNameA(NULL, name, size);
+	if (res != 0 && res < size)
+	{
+	    return name;
+	}
+	else if (res == size)
+	{
+	    /* NOTE: Don't check GetLastError() == ERROR_INSUFFICIENT_BUFFER
+	     *       here, it isn't set consisntently across all platforms
+	     */
+
+	    size += 4096;
+	}
+	else
+	{
+	    DWORD err = GetLastError();
+	    fprintf(stderr, "WARNING: GetModuleFileName() failed "
+		"(res = %d, err = %d)", res, err);
+	    free(name);
+	    return NULL;
+
+	}
+    }
+
+    return NULL;
+}
+
+static int
+_run()
+{
+    if (PyRun_SimpleString(site_code) != 0)
+    {
+	fprintf(stderr, "ERROR: failed to initialize site path\n");
+	return 1;
+    }
+
+    if (PyRun_SimpleString(run_openslides_code) != 0)
+    {
+	fprintf(stderr, "ERROR: failed to execute openslides\n");
+	return 1;
+    }
+
+    return 0;
+}
+
+
+int
+main(int argc, char *argv[])
+{
+    int returncode;
+    char *py_home, *sep = NULL;
+
+    Py_SetProgramName(argv[0]);
+
+    py_home = _get_module_name();
+
+    if (py_home)
+	sep = strrchr(py_home, '\\');
+    /* should always be the true */
+    if (sep)
+    {
+	*sep = '\0';
+	Py_SetPythonHome(py_home);
+    }
+
+    Py_Initialize();
+    PySys_SetArgvEx(argc, argv, 0);
+
+    returncode = _run();
+
+    Py_Finalize();
+    free(py_home);
+
+    return returncode;
+}

extras/win32-portable/prepare_portable.py

@@ -0,0 +1,292 @@
+import errno
+import glob
+import os
+import re
+import shutil
+import sys
+import zlib
+zlib.Z_DEFAULT_COMPRESSION = zlib.Z_BEST_COMPRESSION
+import zipfile
+
+import distutils.ccompiler
+import distutils.sysconfig
+
+from contextlib import nested
+
+import pkg_resources
+
+sys.path.insert(0, os.getcwd())
+import openslides
+
+COMMON_EXCLUDE = [
+    r".pyc$",
+    r".pyo$",
+    r".po$",
+    r".egg-info",
+    r"\blocale/(?!de/|en/)[^/]+/"
+]
+
+LIBEXCLUDE = [
+    r"^site-packages/",
+    r"^test/",
+    r"^curses/",
+    r"^idlelib/",
+    r"^lib2to3/",
+    r"^lib-tk/",
+    r"^msilib/",
+    r"^multiprocessing/",
+    r"^unittest/",
+]
+
+OPENSLIDES_EXCLUDE = [
+    r"^openslides/settings.py"
+]
+
+
+SITE_PACKAGES = {
+    "django": {
+        "copy": ["django"],
+        "exclude": [
+            r"^contrib/admindocs/",
+            r"^contrib/comments/",
+            r"^contrib/databrowse/",
+            r"^contrib/flatpages/",
+            r"^contrib/formtools/",
+            r"^contrib/gis/",
+            r"^contrib/humanize/",
+            r"^contrib/localflavor/",
+            r"^contrib/markup/",
+            r"^contrib/redirects/",
+            r"^contrib/sitemaps/",
+            r"^contrib/syndication/",
+            r"^contrib/webdesign/",
+        ]
+    },
+    "django-mptt": {
+        "copy": ["mptt"],
+    },
+    "reportlab": {
+        "copy": [
+            "reportlab",
+            "_renderPM.pyd",
+            "_rl_accel.pyd",
+            "sgmlop.pyd",
+            "pyHnj.pyd",
+        ],
+    },
+    "pil": {
+        # NOTE: PIL is a special case, see copy_pil
+        "copy": [],
+    }
+}
+
+PY_DLLS = [
+    "unicodedata.pyd",
+    "sqlite3.dll",
+    "_sqlite3.pyd",
+    "_socket.pyd",
+    "select.pyd",
+]
+
+MSVCR_PUBLIC_KEY = "1fc8b3b9a1e18e3b"
+MSVCR_VERSION = "9.0.21022.8"
+MSVCR_NAME = "Microsoft.VC90.CRT"
+
+def compile_re_list(patterns):
+    expr = "|".join("(?:{0})".format(x) for x in patterns)
+    return re.compile(expr)
+
+def relpath(base, path, addslash = False):
+    b = os.path.normpath(os.path.abspath(base))
+    p = os.path.normpath(os.path.abspath(path))
+    if p == b:
+        p = "."
+        if addslash:
+            p += "/"
+        return p
+
+    b += os.sep
+    if not p.startswith(b):
+        raise ValueError("{0!r} is not relative to {1!r}".format(path, base))
+    p = p[len(b):].replace(os.sep, "/")
+    if addslash:
+        p += "/"
+
+    return p
+
+def filter_excluded_dirs(exclude_pattern, basedir, dirpath, dnames):
+    i, l = 0, len(dnames)
+    while i < l:
+        rp = relpath(basedir, os.path.join(dirpath, dnames[i]), True)
+        if exclude_pattern.search(rp):
+            del dnames[i]
+            l -= 1
+        else:
+            i += 1
+
+def copy_dir_exclude(exclude, basedir, srcdir, destdir):
+    for dp, dnames, fnames in os.walk(srcdir):
+        filter_excluded_dirs(exclude, basedir, dp, dnames)
+
+        rp = relpath(basedir, dp)
+        target_dir = os.path.join(destdir, rp)
+        if not os.path.exists(target_dir):
+            os.makedirs(target_dir)
+
+        for fn in fnames:
+            fp = os.path.join(dp, fn)
+            rp = relpath(basedir, fp)
+            if exclude.search(rp):
+                continue
+
+            shutil.copyfile(fp, os.path.join(destdir, rp))
+
+def collect_lib(libdir, odir):
+    exclude = compile_re_list(COMMON_EXCLUDE + LIBEXCLUDE)
+    copy_dir_exclude(exclude, libdir, libdir, os.path.join(odir, "Lib"))
+
+def get_pkg_exclude(name, extra = ()):
+    exclude = COMMON_EXCLUDE[:]
+    exclude.extend(SITE_PACKAGES.get(name, {}).get("exclude", []))
+    exclude.extend(extra)
+    return compile_re_list(exclude)
+
+def copy_package(name, info, odir):
+    dist = pkg_resources.get_distribution(name)
+    exclude = get_pkg_exclude(name)
+
+    site_dir = dist.location
+    for thing in info.get("copy", []):
+        fp = os.path.join(site_dir, thing)
+        if not os.path.isdir(fp):
+            rp = relpath(site_dir, fp)
+            ofp = os.path.join(odir, rp)
+            shutil.copyfile(fp, ofp)
+        else:
+            copy_dir_exclude(exclude, site_dir, fp, odir)
+
+def copy_pil(odir):
+    dist = pkg_resources.get_distribution("pil")
+    exclude = get_pkg_exclude("pil")
+
+    dest_dir = os.path.join(odir, "PIL")
+    copy_dir_exclude(exclude, dist.location, dist.location, dest_dir)
+    fp = os.path.join(dest_dir, "PIL.pth")
+    if os.path.isfile(fp):
+        os.rename(fp, os.path.join(odir, "PIL.pth"))
+    else:
+        fp = os.path.join(os.path.dirname(dist.location), "PIL.pth")
+        shutil.copyfile(fp, os.path.join(odir, "PIL.pth"))
+
+def collect_site_packages(sitedir, odir):
+    if not os.path.exists(odir):
+        os.makedirs(odir)
+
+    for name, info in SITE_PACKAGES.iteritems():
+        copy_package(name, info, odir)
+
+    assert "pil" in SITE_PACKAGES
+    copy_pil(odir)
+
+def compile_openslides_launcher():
+    cc = distutils.ccompiler.new_compiler()
+    cc.add_include_dir(distutils.sysconfig.get_python_inc())
+    cc.add_library_dir(os.path.join(sys.exec_prefix, "Libs"))
+
+    objs = cc.compile(["extras/win32-portable/openslides.c"])
+    cc.link_executable(objs, "extras/win32-portable/openslides")
+
+def copy_dlls(odir):
+    dll_src = os.path.join(sys.exec_prefix, "DLLs")
+    dll_dest = os.path.join(odir, "DLLs")
+    if not os.path.exists(dll_dest):
+        os.makedirs(dll_dest)
+
+    for dll_name in PY_DLLS:
+        src = os.path.join(dll_src, dll_name)
+        dest = os.path.join(dll_dest, dll_name)
+        shutil.copyfile(src, dest)
+
+    pydllname = "python{0}{1}.dll".format(*sys.version_info[:2])
+    src = os.path.join(os.environ["WINDIR"], "System32", pydllname)
+    dest = os.path.join(odir, pydllname)
+    shutil.copyfile(src, dest)
+
+def copy_msvcr(odir):
+    candidates = glob.glob("{0}/x86_*{1}_{2}*".format(
+        os.path.join(os.environ["WINDIR"], "winsxs"),
+        MSVCR_NAME, MSVCR_PUBLIC_KEY))
+
+    msvcr_local_name = None
+    msvcr_dll_dir = None
+    for dp in candidates:
+        bn = os.path.basename(dp)
+        if MSVCR_VERSION in bn:
+            msvcr_local_name = bn
+            msvcr_dll_dir = dp
+            break
+    else:
+        sys.stderr.write("Warning could not determine msvcr runtime location\n")
+        sys.stderr.write("Private asssembly for VC runtime must be added manually\n")
+        return
+
+
+    msvcr_dest_dir = os.path.join(odir, MSVCR_NAME)
+    if not os.path.exists(msvcr_dest_dir):
+        os.makedirs(msvcr_dest_dir)
+
+    for fn in os.listdir(msvcr_dll_dir):
+        src = os.path.join(msvcr_dll_dir, fn)
+        dest = os.path.join(msvcr_dest_dir, fn)
+        shutil.copyfile(src, dest)
+
+    src = os.path.join(os.environ["WINDIR"], "winsxs", "Manifests",
+        "{0}.manifest".format(msvcr_local_name))
+    dest = os.path.join(msvcr_dest_dir, "{0}.manifest".format(MSVCR_NAME))
+    shutil.copyfile(src, dest)
+
+def main():
+    prefix = os.path.dirname(sys.executable)
+    libdir = os.path.join(prefix, "Lib")
+    sitedir = os.path.join(libdir, "site-packages")
+    odir = "dist/openslides-{0}-portable".format(openslides.get_version())
+
+    try:
+        shutil.rmtree(odir)
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+
+    os.makedirs(odir)
+
+    collect_lib(libdir, odir)
+    collect_site_packages(sitedir, os.path.join(odir, "site-packages"))
+
+    exclude = get_pkg_exclude("openslides", OPENSLIDES_EXCLUDE)
+    copy_dir_exclude(exclude, ".", "openslides", odir)
+
+    try:
+        compile_openslides_launcher()
+    except distutils.errors.DistutilsError:
+        sys.stderr.write("openslides.exe could not be build, "
+            "trying to use existing file\n")
+
+    shutil.copyfile("extras/win32-portable/openslides.exe",
+        os.path.join(odir, "openslides.exe"))
+
+    copy_dlls(odir)
+    copy_msvcr(odir)
+
+    fp = os.path.join("dist", "openslides-{0}-portable.zip".format(
+        openslides.get_version()))
+
+    with zipfile.ZipFile(fp, "w", zipfile.ZIP_DEFLATED) as zf:
+        for dp, dnames, fnames in os.walk(odir):
+            for fn in fnames:
+                fp = os.path.join(dp, fn)
+                rp = relpath(odir, fp)
+                zf.write(fp, rp)
+
+
+if __name__ == "__main__":
+    main()

openslides/main.py

@@ -0,0 +1,152 @@
+from __future__ import with_statement
+
+import os
+import sys
+import optparse
+import socket
+import time
+import threading
+import webbrowser
+from contextlib import nested
+
+import django.conf
+from django.core.management import execute_from_command_line
+from django.utils.crypto import get_random_string
+
+import openslides
+
+
+def main(argv = None):
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = optparse.OptionParser(description = "Run openslides using "
+        "django's builtin webserver")
+    parser.add_option("-a", "--address", help = "IP Address to listen on")
+    parser.add_option("-p", "--port", type = "int", help = "Port to listen on")
+    parser.add_option("--nothread", action = "store_true",
+        help = "Do not use threading")
+    parser.add_option("--syncdb", action = "store_true",
+        help = "Update/create database before starting the server")
+    parser.add_option("--reset-admin", action = "store_true",
+        help = "Make sure the user 'admin' exists and uses 'admin' as password")
+
+    opts, args = parser.parse_args(argv)
+    if args:
+        sys.stderr.write("This command does not take arguments!\n\n")
+        parser.print_help()
+        sys.exit(1)
+
+    if not prepare_openslides(opts.syncdb):
+        sys.exit(1)
+
+    if opts.reset_admin:
+        create_or_reset_admin_user()
+
+    argv = ["", "runserver", "--noreload"]
+    if opts.nothread:
+        argv.append("--nothread")
+
+    addr, port = detect_listen_opts(opts.address, opts.port)
+    argv.append("%s:%d" % (addr, port))
+
+    start_browser(addr, port)
+    execute_from_command_line(argv)
+
+def detect_listen_opts(address, port):
+    if address is None:
+        address = socket.gethostbyname(socket.gethostname())
+
+    if port is None:
+        # test if we can use port 80
+        s = socket.socket()
+        port = 80
+        try:
+            s.bind((address, port))
+            s.listen(-1)
+        except socket.error:
+            port = 8000
+        finally:
+            s.close()
+
+    return address, port
+
+def start_browser(addr, port):
+    if port == 80:
+        url = "http://%s" % (addr, )
+    else:
+        url = "http://%s:%d" % (addr, port)
+    browser = webbrowser.get()
+    def f():
+        time.sleep(1)
+        browser.open(url)
+
+    t = threading.Thread(target = f)
+    t.start()
+
+def prepare_openslides(always_syncdb = False):
+    settings_module = os.environ.get(django.conf.ENVIRONMENT_VARIABLE)
+    if not settings_module:
+        os.environ[django.conf.ENVIRONMENT_VARIABLE] = "openslides.settings"
+        settings_module = "openslides.settings"
+
+    try:
+        # settings is a lazy object, force the settings module
+        # to be imported
+        dir(django.conf.settings)
+    except ImportError:
+        pass
+    else:
+        if always_syncdb:
+            run_syncdb()
+        return True # import worked, settings are already configured
+
+
+    if settings_module != "openslides.settings":
+        sys.stderr.write("Settings module '%s' cannot be imported.\n"
+            % (django.conf.ENVIRONMENT_VARIABLE, ))
+        return False
+
+    openslides_dir = os.path.dirname(openslides.__file__)
+    src_fp = os.path.join(openslides_dir, "default.settings.py")
+    dest_fp = os.path.join(openslides_dir, "settings.py")
+
+    with nested(open(dest_fp, "w"), open(src_fp, "r")) as (dest, src):
+        for l in src:
+            if l.startswith("SECRET_KEY ="):
+                l = "SECRET_KEY = '%s'\n" % (generate_secret_key(), )
+            dest.write(l)
+
+
+    run_syncdb()
+    create_or_reset_admin_user()
+    return True
+
+def run_syncdb():
+    # now initialize the database
+    argv = ["", "syncdb", "--noinput"]
+    execute_from_command_line(argv)
+
+def create_or_reset_admin_user():
+    from django.contrib.auth.models import User
+
+    try:
+        obj = User.objects.get(username = "admin")
+    except User.DoesNotExist:
+        User.objects.create_superuser(
+            username = "admin",
+            password = "admin",
+            email = "admin@example.com")
+        return
+
+    obj.set_password("admin")
+    obj.save()
+
+
+def generate_secret_key():
+    # same chars/ length as used in djangos startproject command
+    chars = "abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)"
+    return get_random_string(50, chars)
+
+if __name__ == "__main__":
+    main()

start.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import openslides.main
+
+if __name__ == "__main__":
+    openslides.main.main()