diff --git a/THANKS b/THANKS index 847d02a42..f7c44c3d4 100644 --- a/THANKS +++ b/THANKS @@ -9,7 +9,11 @@ OpenSlides uses parts of the following projects: License: BSD * Django mptt - + + License: BSD + +* Django haystack + License: BSD * jQuery @@ -32,7 +36,7 @@ OpenSlides uses parts of the following projects: License: MIT/GPLv2 - jQuery bsmSelect - + License: MIT/GPLv2 * jQuery UI @@ -68,11 +72,11 @@ OpenSlides uses parts of the following projects: License: BSD * Pillow - + License: Standard PIL License * qrcode - + License: BSD * ReportLab @@ -87,6 +91,10 @@ OpenSlides uses parts of the following projects: License: Ubuntu Font Licence 1.0 +* Whoosh + + License: BSD + * Sphinx License: BSD diff --git a/extras/win32-portable/create_portable.txt b/extras/win32-portable/create_portable.txt index dda2949a4..3172eddf5 100644 --- a/extras/win32-portable/create_portable.txt +++ b/extras/win32-portable/create_portable.txt @@ -4,7 +4,7 @@ How to create a new portable Windows distribution of OpenSlides: 1.) Follow the OpenSlides installation instructions for windows, but add the option "-Z" when executing easy_install, e.g.: - easy_install -Z django django-mptt beautifulsoup4 bleach pillow qrcode reportlab tornado + easy_install -Z django django-mptt beautifulsoup4 bleach pillow qrcode reportlab tornado django-haystack whoosh 2.) To update the version resource of the prebuild openslides.exe pywin32 should be installed (it is not strictly required but at diff --git a/extras/win32-portable/licenses/django-haystack b/extras/win32-portable/licenses/django-haystack new file mode 100644 index 000000000..0bb702eda --- /dev/null +++ b/extras/win32-portable/licenses/django-haystack @@ -0,0 +1,31 @@ +Copyright (c) 2009-2013, Daniel Lindsley. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of Haystack nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +Prior to April 17, 2009, this software was released under the MIT license. diff --git a/extras/win32-portable/licenses/django-mptt b/extras/win32-portable/licenses/django-mptt index 07ac9a98d..346aa6196 100644 --- a/extras/win32-portable/licenses/django-mptt +++ b/extras/win32-portable/licenses/django-mptt @@ -1,6 +1,3 @@ -Django MPTT ------------ - Copyright (c) 2007, Jonathan Buchanan Permission is hereby granted, free of charge, to any person obtaining a copy of diff --git a/extras/win32-portable/licenses/tornado b/extras/win32-portable/licenses/tornado index adf6341b6..f6e141e2d 100644 --- a/extras/win32-portable/licenses/tornado +++ b/extras/win32-portable/licenses/tornado @@ -1,4 +1,4 @@ Tornado is one of `Facebook's open source technologies `_. It is available under the `Apache License, Version 2.0 -`_. \ No newline at end of file +`_. diff --git a/extras/win32-portable/licenses/whoosh b/extras/win32-portable/licenses/whoosh new file mode 100644 index 000000000..b0266325e --- /dev/null +++ b/extras/win32-portable/licenses/whoosh @@ -0,0 +1,26 @@ +Copyright 2011 Matt Chaput. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are +those of the authors and should not be interpreted as representing official +policies, either expressed or implied, of Matt Chaput. diff --git a/extras/win32-portable/prepare_portable.py b/extras/win32-portable/prepare_portable.py index 593e5b051..9b89e0e06 100755 --- a/extras/win32-portable/prepare_portable.py +++ b/extras/win32-portable/prepare_portable.py @@ -50,7 +50,6 @@ SITE_PACKAGES = { "django": { "copy": ["django"], "exclude": [ - r"^django/contrib/admin/", r"^django/contrib/admindocs/", r"^django/contrib/comments/", r"^django/contrib/databrowse/", @@ -103,6 +102,12 @@ SITE_PACKAGES = { "html5lib": { "copy": ["html5lib"], }, + "django-haystack": { + "copy": ["haystack"], + }, + "whoosh": { + "copy": ["whoosh"], + }, "wx": { # NOTE: wxpython is a special case, see copy_wx "copy": [], diff --git a/openslides/agenda/search_indexes.py b/openslides/agenda/search_indexes.py new file mode 100644 index 000000000..48325529f --- /dev/null +++ b/openslides/agenda/search_indexes.py @@ -0,0 +1,11 @@ +from haystack import indexes +from .models import Item + + +class Index(indexes.SearchIndex, indexes.Indexable): + text = indexes.EdgeNgramField(document=True, use_template=True) + modelfilter_name = "Agenda" # verbose_name of model + modelfilter_value = "agenda.item" # 'app_name.model_name' + + def get_model(self): + return Item diff --git a/openslides/agenda/templates/search/agenda-results.html b/openslides/agenda/templates/search/agenda-results.html new file mode 100644 index 000000000..8397a60f1 --- /dev/null +++ b/openslides/agenda/templates/search/agenda-results.html @@ -0,0 +1,10 @@ +{% load i18n %} +{% load highlight %} + +{% if perms.agenda.can_see_agenda %} +
  • + {{ result.object }}
    + {% trans "Agenda" %}
    + {% highlight result.text with request.GET.q %} +
    • +{% endif %} diff --git a/openslides/agenda/templates/search/indexes/agenda/item_text.txt b/openslides/agenda/templates/search/indexes/agenda/item_text.txt new file mode 100644 index 000000000..61e3e6538 --- /dev/null +++ b/openslides/agenda/templates/search/indexes/agenda/item_text.txt @@ -0,0 +1,2 @@ +{{ object.title }} +{{ object.text }} diff --git a/openslides/assignment/search_indexes.py b/openslides/assignment/search_indexes.py new file mode 100644 index 000000000..f9d434645 --- /dev/null +++ b/openslides/assignment/search_indexes.py @@ -0,0 +1,11 @@ +from haystack import indexes +from .models import Assignment + + +class Index(indexes.SearchIndex, indexes.Indexable): + text = indexes.EdgeNgramField(document=True, use_template=True) + modelfilter_name = "Elections" # verbose_name of model + modelfilter_value = "assignment.assignment" # 'app_name.model_name' + + def get_model(self): + return Assignment diff --git a/openslides/assignment/templates/search/assignment-results.html b/openslides/assignment/templates/search/assignment-results.html new file mode 100644 index 000000000..7ea3ef147 --- /dev/null +++ b/openslides/assignment/templates/search/assignment-results.html @@ -0,0 +1,10 @@ +{% load i18n %} +{% load highlight %} + +{% if perms.assignment.can_see_assignment %} +
  • + {{ result.object }}
    + {% trans "Election" %}
    + {% highlight result.text with request.GET.q %} +
    • +{% endif %} diff --git a/openslides/assignment/templates/search/indexes/assignment/assignment_text.txt b/openslides/assignment/templates/search/indexes/assignment/assignment_text.txt new file mode 100644 index 000000000..26c4bab18 --- /dev/null +++ b/openslides/assignment/templates/search/indexes/assignment/assignment_text.txt @@ -0,0 +1,3 @@ +{{ object.name }} +{{ object.description }} +{{ object.candidates }} diff --git a/openslides/core/templates/core/search.html b/openslides/core/templates/core/search.html new file mode 100644 index 000000000..7d3d06491 --- /dev/null +++ b/openslides/core/templates/core/search.html @@ -0,0 +1,56 @@ +{% extends 'base.html' %} + +{% load i18n %} + +{% block title %}{{ block.super }} – {% trans "Search" %}{% endblock %} + +{% block content %} +

    {% trans 'Search results' %}

    +
    + {% for field in form %} +
    + {% if field.name == "q" %} + +
    + + +
    + {% elif field.name == "models" %} + {% trans 'Filter' %}: +
    + {% for model in models %} + + {% endfor %} +
    + {% endif %} +
    + {% endfor %} +
    + + {% if query %} + {% for result in page.object_list %} + {% if forloop.first %} +
      + {% endif %} + {% with result_template=result.app_label|add:"-results.html" %} + {% include "search/"|add:result_template %} + {% endwith %} + {% if forloop.last %} +
    + {% endif %} + {% empty %} +

    {% trans "No results found." %}

    + {% endfor %} + + {% if page.has_previous or page.has_next %} +
    + {% if page.has_previous %}{% endif %}« Previous{% if page.has_previous %}{% endif %} + | + {% if page.has_next %}{% endif %}Next »{% if page.has_next %}{% endif %} +
    + {% endif %} + {% endif %} +{% endblock %} diff --git a/openslides/core/urls.py b/openslides/core/urls.py index 6d73466f7..aa9b99e54 100644 --- a/openslides/core/urls.py +++ b/openslides/core/urls.py @@ -26,4 +26,8 @@ urlpatterns = patterns( url(r'^version/$', views.VersionView.as_view(), name='core_version',), + + url(r'^search/$', + views.SearchView(), + name='search',), ) diff --git a/openslides/core/views.py b/openslides/core/views.py index fd511d237..0338c1424 100644 --- a/openslides/core/views.py +++ b/openslides/core/views.py @@ -11,9 +11,12 @@ """ from django.conf import settings +from django.core.exceptions import PermissionDenied from django.utils.importlib import import_module +from haystack.views import SearchView as _SearchView from openslides import get_git_commit_id, get_version, RELEASE +from openslides.utils.signals import template_manipulation from openslides.utils.views import TemplateView @@ -64,3 +67,47 @@ class VersionView(TemplateView): context['versions'].append((plugin_name, plugin_version)) return context + + +class SearchView(_SearchView): + """ + Shows search result page. + """ + template = 'core/search.html' + + def __call__(self, request): + if not request.user.is_authenticated(): + raise PermissionDenied + return super(SearchView, self).__call__(request) + + def extra_context(self): + """ + Adds extra context variables to set navigation and search filter. + + Returns a context dictionary. + """ + context = {} + template_manipulation.send( + sender=self.__class__, request=self.request, context=context) + context['models'] = self.get_indexed_searchmodels() + context['get_values'] = self.request.GET.getlist('models') + return context + + def get_indexed_searchmodels(self): + """ + Iterate over all INSTALLED_APPS and return a list of models which are + indexed by haystack/whoosh for using in customized model search filter + in search template search.html. Each list entry contains a verbose name + of the model and a special form field value for haystack (app_name.model_name), + e.g. ['Agenda', 'agenda.item']. + """ + models = [] + # TODO: cache this query! + for app in settings.INSTALLED_APPS: + try: + module = import_module(app + '.search_indexes') + except ImportError: + pass + else: + models.append([module.Index.modelfilter_name, module.Index.modelfilter_value]) + return models diff --git a/openslides/global_settings.py b/openslides/global_settings.py index 9c2baea72..373643730 100644 --- a/openslides/global_settings.py +++ b/openslides/global_settings.py @@ -108,6 +108,7 @@ INSTALLED_APPS = ( 'django.contrib.staticfiles', 'django.contrib.humanize', 'mptt', + 'haystack', # full-text-search 'openslides.poll', 'openslides.core', 'openslides.account', @@ -142,3 +143,13 @@ TEST_DISCOVER_TOP_LEVEL = os.path.dirname(os.path.dirname(__file__)) # Hosts/domain names that are valid for this site; required if DEBUG is False # See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts ALLOWED_HOSTS = ['*'] + +# Use Haystack with Whoosh for full text search +HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine' + }, +} + +# Haystack updates search index after each save/delete action by apps +HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor' diff --git a/openslides/main.py b/openslides/main.py index fd7624eca..9cbabc18f 100644 --- a/openslides/main.py +++ b/openslides/main.py @@ -70,6 +70,8 @@ INSTALLED_APPS += INSTALLED_PLUGINS # Example: "/home/media/media.lawrence.com/" MEDIA_ROOT = %(media_root_path)s +# Path to Whoosh search index +HAYSTACK_CONNECTIONS['default']['PATH'] = %(whoosh_index_path)s """ KEY_LENGTH = 30 @@ -225,16 +227,19 @@ def create_settings(settings_path, database_path=None): database_path = get_portable_db_path() dbpath_value = 'openslides.main.get_portable_db_path()' media_root_path_value = 'openslides.main.get_portable_media_root_path()' + whoosh_index_path_value = 'openslides.main.get_portable_whoosh_index_path()' else: if database_path is None: database_path = get_user_data_path('openslides', 'database.sqlite') dbpath_value = repr(fs2unicode(database_path)) media_root_path_value = repr(fs2unicode(get_user_data_path('openslides', 'media', ''))) + whoosh_index_path_value = repr(fs2unicode(get_user_data_path('openslides', 'whoosh_index', ''))) settings_content = CONFIG_TEMPLATE % dict( default_key=base64.b64encode(os.urandom(KEY_LENGTH)), dbpath=dbpath_value, - media_root_path=media_root_path_value) + media_root_path=media_root_path_value, + whoosh_index_path=whoosh_index_path_value) if not os.path.exists(settings_module): os.makedirs(settings_module) @@ -388,6 +393,10 @@ def get_portable_media_root_path(): return get_portable_path('openslides', 'media', '') +def get_portable_whoosh_index_path(): + return get_portable_path('openslides', 'whoosh_index', '') + + def win32_get_app_data_path(*args): shell32 = ctypes.WinDLL("shell32.dll") SHGetFolderPath = shell32.SHGetFolderPathW diff --git a/openslides/mediafile/search_indexes.py b/openslides/mediafile/search_indexes.py new file mode 100644 index 000000000..894448997 --- /dev/null +++ b/openslides/mediafile/search_indexes.py @@ -0,0 +1,11 @@ +from haystack import indexes +from .models import Mediafile + + +class Index(indexes.SearchIndex, indexes.Indexable): + text = indexes.EdgeNgramField(document=True, use_template=True) + modelfilter_name = "Files" # verbose_name of model + modelfilter_value = "mediafile.mediafile" # 'app_name.model_name' + + def get_model(self): + return Mediafile diff --git a/openslides/mediafile/templates/search/indexes/mediafile/mediafile_text.txt b/openslides/mediafile/templates/search/indexes/mediafile/mediafile_text.txt new file mode 100644 index 000000000..1cebe9df6 --- /dev/null +++ b/openslides/mediafile/templates/search/indexes/mediafile/mediafile_text.txt @@ -0,0 +1 @@ +{{ object.title }} diff --git a/openslides/mediafile/templates/search/mediafile-results.html b/openslides/mediafile/templates/search/mediafile-results.html new file mode 100644 index 000000000..aceb903ca --- /dev/null +++ b/openslides/mediafile/templates/search/mediafile-results.html @@ -0,0 +1,10 @@ +{% load i18n %} +{% load highlight %} + +{% if perms.mediafile.can_see %} +
  • + {{ result.object }}
    + {% trans "File" %}
    + {% highlight result.text with request.GET.q %} +
    • +{% endif %} diff --git a/openslides/motion/search_indexes.py b/openslides/motion/search_indexes.py new file mode 100644 index 000000000..46a599d9f --- /dev/null +++ b/openslides/motion/search_indexes.py @@ -0,0 +1,11 @@ +from haystack import indexes +from .models import Motion + + +class Index(indexes.SearchIndex, indexes.Indexable): + text = indexes.EdgeNgramField(document=True, use_template=True) + modelfilter_name = "Motions" # verbose_name of model + modelfilter_value = "motion.motion" # 'app_name.model_name' + + def get_model(self): + return Motion diff --git a/openslides/motion/templates/search/indexes/motion/motion_text.txt b/openslides/motion/templates/search/indexes/motion/motion_text.txt new file mode 100644 index 000000000..cdd239c13 --- /dev/null +++ b/openslides/motion/templates/search/indexes/motion/motion_text.txt @@ -0,0 +1,7 @@ +{{ object.identifier }} +{{ object.title }} +{{ object.text }} +{{ object.reason }} +{{ object.submitters }} +{{ object.supporters }} +{{ object.category }} diff --git a/openslides/motion/templates/search/motion-results.html b/openslides/motion/templates/search/motion-results.html new file mode 100644 index 000000000..708d8c5b3 --- /dev/null +++ b/openslides/motion/templates/search/motion-results.html @@ -0,0 +1,10 @@ +{% load i18n %} +{% load highlight %} + +{% if perms.motion.can_see_motion %} +
  • + {{ result.object }}
    + {% trans "Motion" %}
    + {% highlight result.text with request.GET.q %} +
    • +{% endif %} diff --git a/openslides/participant/search_indexes.py b/openslides/participant/search_indexes.py new file mode 100644 index 000000000..f142f19a8 --- /dev/null +++ b/openslides/participant/search_indexes.py @@ -0,0 +1,12 @@ +from haystack import indexes +from .models import User + + +class Index(indexes.SearchIndex, indexes.Indexable): + text = indexes.EdgeNgramField(document=True, use_template=True) + text = indexes.EdgeNgramField(document=True, use_template=True) + modelfilter_name = "Participants" # verbose_name of model + modelfilter_value = "participant.user" # 'app_name.model_name' + + def get_model(self): + return User diff --git a/openslides/participant/templates/search/indexes/participant/user_text.txt b/openslides/participant/templates/search/indexes/participant/user_text.txt new file mode 100644 index 000000000..a702a7079 --- /dev/null +++ b/openslides/participant/templates/search/indexes/participant/user_text.txt @@ -0,0 +1,4 @@ +{{ object.django_user }} +{{ object.structure_level }} +{{ object.committee }} +{{ object.about_me }} diff --git a/openslides/participant/templates/search/participant-results.html b/openslides/participant/templates/search/participant-results.html new file mode 100644 index 000000000..8fb97371d --- /dev/null +++ b/openslides/participant/templates/search/participant-results.html @@ -0,0 +1,10 @@ +{% load i18n %} +{% load highlight %} + +{% if perms.participant.can_see_participant %} +
  • + {{ result.object }}
    + {% trans "Participant" %}
    + {% highlight result.text with request.GET.q %} +
    • +{% endif %} diff --git a/openslides/static/styles/base.css b/openslides/static/styles/base.css index ce8ea5b33..30892f00e 100644 --- a/openslides/static/styles/base.css +++ b/openslides/static/styles/base.css @@ -34,6 +34,10 @@ body { position: absolute; margin: 8px 0 0 50px; } +#header .navbar-search { + margin-top: 0px; +} + footer { margin-bottom: 20px; } @@ -180,6 +184,15 @@ legend + .control-group { #dataTable_wrapper .row-fluid:after { clear: none; } +.searchresults li { + margin-bottom: 15px; +} +.searchresults li .app { + color: #999999; +} +.highlighted { + font-weight: bold; +} /** Left sitebar navigation **/ diff --git a/openslides/templates/base.html b/openslides/templates/base.html index e424910e9..a529a7a0f 100644 --- a/openslides/templates/base.html +++ b/openslides/templates/base.html @@ -27,22 +27,32 @@ {% get_config 'event_name' %} – {% get_config 'event_description' %} {% block loginbutton %} -
    - {% if user.is_authenticated %} - - {{ user.username }} - - - - {% else %} - {% trans "Login" %} - {% endif %} +
    + +
    +
    + + +
    +
      + + +
    + {% if user.is_authenticated %} + + {{ user.username }} + + + + {% else %} + {% trans "Login" %} + {% endif %} +
    {% endblock %}
    diff --git a/openslides/utils/test.py b/openslides/utils/test.py index ce6438a83..8e0f63f42 100644 --- a/openslides/utils/test.py +++ b/openslides/utils/test.py @@ -10,7 +10,7 @@ :license: GNU GPL, see LICENSE for more details. """ - +from django.core.management import call_command from django.test import TestCase as _TestCase from openslides.config.api import config @@ -35,4 +35,6 @@ class TestCase(_TestCase): except AttributeError: # The cache has only to be deleted if it exists. pass + # Clear the whoosh search index + call_command('clear_index', interactive=False, verbosity=0) return return_value diff --git a/requirements_production.txt b/requirements_production.txt index d20ea17f8..5d7b35605 100644 --- a/requirements_production.txt +++ b/requirements_production.txt @@ -6,3 +6,5 @@ qrcode==2.7 tornado==3.0.1 bleach==1.2.2 beautifulsoup4==4.2.0 +django-haystack==2.1.0 +whoosh==2.5.4 diff --git a/tests/settings.py b/tests/settings.py index 26e64edd4..fcd226c16 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -37,3 +37,6 @@ INSTALLED_APPS += INSTALLED_PLUGINS # Absolute path to the directory that holds media. # Example: "/home/media/media.lawrence.com/" MEDIA_ROOT = '' + +# Use RAM storage for whoosh index +HAYSTACK_CONNECTIONS['default']['STORAGE'] = 'ram'