From 1230f4a29af90da9f75c1ca25dc347f6721f8831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Sch=C3=BCtze?= Date: Fri, 20 Jan 2017 11:35:29 +0100 Subject: [PATCH] New full text search on client-side (Fixed #2865). Removed server-side search by whoosh. --- CHANGELOG | 1 + README.rst | 2 - openslides/agenda/static/js/agenda/base.js | 17 ++ openslides/agenda/static/js/agenda/site.js | 13 ++ openslides/assignments/models.py | 11 -- .../assignments/static/js/assignments/base.js | 15 +- .../assignments/static/js/assignments/site.js | 13 ++ openslides/core/apps.py | 13 -- openslides/core/static/css/app.css | 8 +- openslides/core/static/js/core/site.js | 120 ++++++------ openslides/core/static/templates/search.html | 76 +++----- openslides/core/urls.py | 4 - openslides/core/views.py | 21 -- openslides/mediafiles/models.py | 10 - .../static/js/mediafiles/resources.js | 14 +- .../mediafiles/static/js/mediafiles/states.js | 13 ++ openslides/motions/models.py | 14 -- openslides/motions/static/js/motions/base.js | 25 ++- openslides/motions/static/js/motions/site.js | 13 ++ openslides/topics/models.py | 8 - openslides/topics/static/js/topics/base.js | 8 - openslides/users/models.py | 11 -- openslides/users/static/js/users/site.js | 12 ++ openslides/utils/main.py | 2 +- openslides/utils/search.py | 180 ------------------ openslides/utils/settings.py.tpl | 6 - requirements_production.txt | 1 - tests/old/settings.py | 6 - tests/settings.py | 6 - 29 files changed, 227 insertions(+), 416 deletions(-) delete mode 100644 openslides/utils/search.py diff --git a/CHANGELOG b/CHANGELOG index 30caafee6..982f50190 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -47,6 +47,7 @@ Core: - Validate HTML strings from CKEditor against XSS attacks. - Added success/error symbol to config to show if saving was successful. - Added UTF-8 byte order mark for every CSV export. +- Moved full-text search to client-side (removed the server-side search engine Whoosh). Motions: - Added adjustable line numbering mode (outside, inside, none) for each diff --git a/README.rst b/README.rst index d12ca407c..47a228795 100644 --- a/README.rst +++ b/README.rst @@ -179,8 +179,6 @@ OpenSlides uses the following projects or parts of them: * `txaio `_, License: MIT -* `Whoosh `_, License: BSD - * `zope.interface `, License: ZPL 2.1 diff --git a/openslides/agenda/static/js/agenda/base.js b/openslides/agenda/static/js/agenda/base.js index a01c7b150..b92a6495b 100644 --- a/openslides/agenda/static/js/agenda/base.js +++ b/openslides/agenda/static/js/agenda/base.js @@ -87,6 +87,23 @@ angular.module('OpenSlidesApp.agenda', ['OpenSlidesApp.users']) getAgendaTitle: function () { return this.title; }, + // link name which is shown in search result + getSearchResultName: function () { + return this.getAgendaTitle(); + }, + // return true if a specific relation matches for given searchquery + // (here: speakers) + hasSearchResult: function (results) { + var item = this; + // search for speakers (check if any user.id from already found users matches) + return _.some(results, function(result) { + if (result.getResourceName() === "users/user") { + if (_.some(item.speakers, {'user_id': result.id})) { + return true; + } + } + }); + }, getListViewTitle: function () { var title; try { diff --git a/openslides/agenda/static/js/agenda/site.js b/openslides/agenda/static/js/agenda/site.js index 9d274f43b..a373d890d 100644 --- a/openslides/agenda/static/js/agenda/site.js +++ b/openslides/agenda/static/js/agenda/site.js @@ -23,6 +23,19 @@ angular.module('OpenSlidesApp.agenda.site', [ } ]) +.config([ + 'SearchProvider', + 'gettext', + function (SearchProvider, gettext) { + SearchProvider.register({ + 'verboseName': gettext('Agenda'), + 'collectionName': 'agenda/item', + 'urlDetailState': 'agenda.item.detail', + 'weight': 200, + }); + } +]) + .config([ '$stateProvider', 'gettext', diff --git a/openslides/assignments/models.py b/openslides/assignments/models.py index 36249e90f..8f51dfb56 100644 --- a/openslides/assignments/models.py +++ b/openslides/assignments/models.py @@ -19,7 +19,6 @@ from openslides.poll.models import ( from openslides.utils.autoupdate import inform_changed_data from openslides.utils.exceptions import OpenSlidesError from openslides.utils.models import RESTModelMixin -from openslides.utils.search import user_name_helper from .access_permissions import AssignmentAccessPermissions @@ -354,16 +353,6 @@ class Assignment(RESTModelMixin, models.Model): """ return self.agenda_item.pk - def get_search_index_string(self): - """ - Returns a string that can be indexed for the search. - """ - return " ".join(( - self.title, - self.description, - user_name_helper(self.related_users.all()), - " ".join(tag.name for tag in self.tags.all()))) - class AssignmentVote(RESTModelMixin, BaseVote): option = models.ForeignKey( diff --git a/openslides/assignments/static/js/assignments/base.js b/openslides/assignments/static/js/assignments/base.js index 1dfde308f..31ac3d3fc 100644 --- a/openslides/assignments/static/js/assignments/base.js +++ b/openslides/assignments/static/js/assignments/base.js @@ -341,9 +341,18 @@ angular.module('OpenSlidesApp.assignments', []) getSearchResultName: function () { return this.getAgendaTitle(); }, - // subtitle of search result - getSearchResultSubtitle: function () { - return "Election"; + // return true if a specific relation matches for given searchquery + // (here: related_users/candidates) + hasSearchResult: function (results) { + var assignment = this; + // search for related users (check if any user.id from already found users matches) + return _.some(results, function(result) { + if (result.getResourceName() === "users/user") { + if (_.some(assignment.assignment_related_users, {'user_id': result.id})) { + return true; + } + } + }); }, // override project function of jsDataModel factory project: function (projectorId, pollId) { diff --git a/openslides/assignments/static/js/assignments/site.js b/openslides/assignments/static/js/assignments/site.js index 578f1b9a2..7418aa275 100644 --- a/openslides/assignments/static/js/assignments/site.js +++ b/openslides/assignments/static/js/assignments/site.js @@ -23,6 +23,19 @@ angular.module('OpenSlidesApp.assignments.site', [ } ]) +.config([ + 'SearchProvider', + 'gettext', + function (SearchProvider, gettext) { + SearchProvider.register({ + 'verboseName': gettext('Elections'), + 'collectionName': 'assignments/assignment', + 'urlDetailState': 'assignments.assignment.detail', + 'weight': 400, + }); + } +]) + .config([ '$stateProvider', 'gettext', diff --git a/openslides/core/apps.py b/openslides/core/apps.py index 6d6f2fdff..1758b94b6 100644 --- a/openslides/core/apps.py +++ b/openslides/core/apps.py @@ -14,11 +14,9 @@ class CoreAppConfig(AppConfig): from . import projector # noqa # Import all required stuff. - from django.db.models import signals from .config import config from .signals import post_permission_creation from ..utils.rest_api import router - from ..utils.search import index_add_instance, index_del_instance from .config_variables import get_config_variables from .signals import delete_django_app_permissions from .views import ( @@ -46,17 +44,6 @@ class CoreAppConfig(AppConfig): router.register(self.get_model('ProjectorMessage').get_collection_string(), ProjectorMessageViewSet) router.register(self.get_model('Countdown').get_collection_string(), CountdownViewSet) - # Update the search when a model is saved or deleted - signals.post_save.connect( - index_add_instance, - dispatch_uid='index_add_instance') - signals.post_delete.connect( - index_del_instance, - dispatch_uid='index_del_instance') - signals.m2m_changed.connect( - index_add_instance, - dispatch_uid='m2m_index_add_instance') - def get_startup_elements(self): from .config import config from ..utils.collection import Collection diff --git a/openslides/core/static/css/app.css b/openslides/core/static/css/app.css index 877f0bac4..89ca1993e 100644 --- a/openslides/core/static/css/app.css +++ b/openslides/core/static/css/app.css @@ -1406,8 +1406,14 @@ img { .searchresults li { margin-bottom: 12px; } -.searchresults li .subtitle { +.searchresults h3 { + margin-bottom: 3px; + padding-bottom: 0; +} +.searchresults .hits { + margin-bottom: 10px; color: #999999; + font-size: 85%; } /* ngDialog: override ngdialog-theme-default */ diff --git a/openslides/core/static/js/core/site.js b/openslides/core/static/js/core/site.js index 34a08b73d..783984b32 100644 --- a/openslides/core/static/js/core/site.js +++ b/openslides/core/static/js/core/site.js @@ -97,6 +97,27 @@ angular.module('OpenSlidesApp.core.site', [ } ]) +// Provider to register a searchable module/app. +.provider('Search', [ + function() { + var searchModules = []; + + this.register = function(module) { + searchModules.push(module); + }; + + this.$get = [ + function () { + return { + getAll: function () { + return searchModules; + } + }; + } + ]; + } +]) + .run([ 'editableOptions', 'gettext', @@ -952,76 +973,59 @@ angular.module('OpenSlidesApp.core.site', [ // Search Controller .controller('SearchCtrl', [ '$scope', - '$http', + '$filter', '$stateParams', - '$location', - '$sanitize', + 'Search', 'DS', - function ($scope, $http, $stateParams, $location, $sanitize, DS) { - $scope.fullword = false; - $scope.filterAgenda = true; - $scope.filterMotion = true; - $scope.filterAssignment = true; - $scope.filterUser = true; - $scope.filterMedia = true; + 'Motion', + function ($scope, $filter, $stateParams, Search, DS, Motion) { + $scope.searchresults = []; + var searchModules = Search.getAll(); // search function $scope.search = function() { - var query = _.escape($scope.query); - if (query !== '') { - var lastquery = query; - // attach asterisks if search is not for full words only - if (!$scope.fullword) { - if (query.charAt(0) != '*'){ - query = "*" + query; - } - if (query.charAt(query.length - 1) != '*'){ - query = query + "*"; - } - } - $scope.query = lastquery; - $http.get('/core/search_api/?q=' + query).then(function(success) { - $scope.results = []; - var elements = success.data.elements; - angular.forEach(elements, function(element) { - DS.find(element.collection, element.id).then(function(data) { - data.urlState = element.collection.replace('/','.')+'.detail'; - data.urlParam = {id: element.id}; - $scope.results.push(data); - }); - }); + $scope.results = []; + var foundObjects = []; + // search in rest properties of all defined searchModule + // (does not found any related objects, e.g. speakers of items) + _.forEach(searchModules, function(searchModule) { + var result = {}; + result.verboseName = searchModule.verboseName; + result.collectionName = searchModule.collectionName; + result.urlDetailState = searchModule.urlDetailState; + result.weight = searchModule.weight; + result.checked = true; + result.elements = $filter('filter')(DS.getAll(searchModule.collectionName), $scope.searchquery); + $scope.results.push(result); + _.forEach(result.elements, function(element) { + foundObjects.push(element); }); - $location.url('/search/?q=' + lastquery); - } + }); + // search additionally in specific releations of all defined searchModules + _.forEach(searchModules, function(searchModule) { + _.forEach(DS.getAll(searchModule.collectionName), function(object) { + if (_.isFunction(object.hasSearchResult)) { + if (object.hasSearchResult(foundObjects, $scope.searchquery)) { + // releation found, check if object is not yet in search results + _.forEach($scope.results, function(result) { + if ((object.getResourceName() === result.collectionName) && + _.findIndex(result.elements, {'id': object.id}) === -1) { + result.elements.push(object); + } + }); + } + } else { + return false; + } + }); + }); }; //get search string from parameters submitted from outside the scope if ($stateParams.q) { - $scope.query = $stateParams.q; + $scope.searchquery = $stateParams.q; $scope.search(); } - - // returns element if part of the current search selection - $scope.filterresult = function() { - return function(result) { - if ($scope.filterUser && result.urlState == 'users.user.detail') { - return result; - } - if ($scope.filterMotion && result.urlState == 'motions.motion.detail') { - return result; - } - if ($scope.filterAgenda && result.urlState == 'topics.topic.detail') { - return result; - } - if ($scope.filterAssignment && result.urlState == 'assignments.assignment.detail') { - return result; - } - if ($scope.filterMedia && result.urlState== 'mediafiles.mediafile.detail') { - return result; - } - return; - }; - }; } ]) diff --git a/openslides/core/static/templates/search.html b/openslides/core/static/templates/search.html index 69f590f78..564af7ea5 100644 --- a/openslides/core/static/templates/search.html +++ b/openslides/core/static/templates/search.html @@ -5,52 +5,38 @@
-
- - - - -
-
- - - - - -
- +
+ + + + +
+ +
+ +
+ +
+
+
+

{{ result.verboseName | translate }}

+
+ {{ result.elements.length}} results +
+
    +
  1. + + {{ element.getSearchResultName() }} + + + {{ element.getSearchResultName() }} + +
-
-
    -
  1. - - {{ result.getSearchResultName() }} - - - {{ result.getSearchResultName() }} - -
    - {{ result.getSearchResultSubtitle() | translate }} -

No results.

-
+
diff --git a/openslides/core/urls.py b/openslides/core/urls.py index 982ed8914..356dd642c 100644 --- a/openslides/core/urls.py +++ b/openslides/core/urls.py @@ -11,10 +11,6 @@ urlpatterns = [ views.VersionView.as_view(), name='core_version'), - url(r'^core/search_api/$', - views.SearchView.as_view(), - name='core_search'), - url(r'^core/encode_media/$', views.MediaEncoder.as_view(), name="core_mediaencoding"), diff --git a/openslides/core/views.py b/openslides/core/views.py index 23e04d20f..7ba51b2d3 100644 --- a/openslides/core/views.py +++ b/openslides/core/views.py @@ -5,7 +5,6 @@ import uuid from collections import OrderedDict from operator import attrgetter from textwrap import dedent -from urllib.parse import unquote from django.apps import apps from django.conf import settings @@ -34,7 +33,6 @@ from ..utils.rest_api import ( detail_route, list_route, ) -from ..utils.search import search from .access_permissions import ( ChatMessageAccessPermissions, ConfigAccessPermissions, @@ -814,25 +812,6 @@ class VersionView(utils_views.APIView): return result -class SearchView(utils_views.APIView): - """ - Accepts a search string and returns a list of objects where each object - is a dictonary with the keywords collection and id. - - This view expects a get argument 'q' with a search string. - - See: https://pythonhosted.org/Whoosh/querylang.html for the format of the - search string. - """ - http_method_names = ['get'] - - def get_context_data(self, **context): - query = self.request.GET.get('q', '') - return super().get_context_data( - elements=search(unquote(query)), - **context) - - class MediaEncoder(utils_views.APIView): """ MediaEncoder is a class based view to prepare encoded media for pdfMake diff --git a/openslides/mediafiles/models.py b/openslides/mediafiles/models.py index 60d7424bb..85667dc87 100644 --- a/openslides/mediafiles/models.py +++ b/openslides/mediafiles/models.py @@ -2,8 +2,6 @@ from django.conf import settings from django.db import models from django.utils.translation import ugettext as _ -from openslides.utils.search import user_name_helper - from ..utils.models import RESTModelMixin from .access_permissions import MediafileAccessPermissions @@ -73,11 +71,3 @@ class Mediafile(RESTModelMixin, models.Model): kB = size / 1024 size_string = '%d kB' % kB return size_string - - def get_search_index_string(self): - """ - Returns a string that can be indexed for the search. - """ - return " ".join(( - self.title, - user_name_helper(self.uploader))) diff --git a/openslides/mediafiles/static/js/mediafiles/resources.js b/openslides/mediafiles/static/js/mediafiles/resources.js index cfb53dc32..ee46eec0b 100644 --- a/openslides/mediafiles/static/js/mediafiles/resources.js +++ b/openslides/mediafiles/static/js/mediafiles/resources.js @@ -36,9 +36,17 @@ angular.module('OpenSlidesApp.mediafiles.resources', [ getSearchResultName: function () { return this.title; }, - // subtitle of search result - getSearchResultSubtitle: function () { - return "File"; + // return true if a specific relation matches for given searchquery + // (here: speakers) + hasSearchResult: function (results) { + var mediafile = this; + // search for speakers (check if any user.id from already found users matches) + return _.some(results, function(result) { + if ((result.getResourceName() === "users/user") && + (mediafile.uploader_id === result.id)) { + return true; + } + }); }, }, computed: { diff --git a/openslides/mediafiles/static/js/mediafiles/states.js b/openslides/mediafiles/static/js/mediafiles/states.js index 9be26fc83..c8c806781 100644 --- a/openslides/mediafiles/static/js/mediafiles/states.js +++ b/openslides/mediafiles/static/js/mediafiles/states.js @@ -22,6 +22,19 @@ angular.module('OpenSlidesApp.mediafiles.states', [ } ]) +.config([ + 'SearchProvider', + 'gettext', + function (SearchProvider, gettext) { + SearchProvider.register({ + 'verboseName': gettext('Files'), + 'collectionName': 'mediafiles/mediafile', + 'urlDetailState': 'mediafiles.mediafile.detail', + 'weight': 600, + }); + } +]) + .config([ 'gettext', '$stateProvider', diff --git a/openslides/motions/models.py b/openslides/motions/models.py index 3700ec01b..a939c962a 100644 --- a/openslides/motions/models.py +++ b/openslides/motions/models.py @@ -20,7 +20,6 @@ from openslides.poll.models import ( ) from openslides.utils.autoupdate import inform_changed_data from openslides.utils.models import RESTModelMixin -from openslides.utils.search import user_name_helper from .access_permissions import ( CategoryAccessPermissions, @@ -648,19 +647,6 @@ class Motion(RESTModelMixin, models.Model): yield amendment yield from amendment.get_amendments_deep() - def get_search_index_string(self): - """ - Returns a string that can be indexed for the search. - """ - return " ".join(( - self.title or '', - self.text or '', - self.reason or '', - str(self.category) if self.category else '', - user_name_helper(self.submitters.all()), - user_name_helper(self.supporters.all()), - " ".join(tag.name for tag in self.tags.all()))) - class MotionVersion(RESTModelMixin, models.Model): """ diff --git a/openslides/motions/static/js/motions/base.js b/openslides/motions/static/js/motions/base.js index 3ce5f05dc..4502e3793 100644 --- a/openslides/motions/static/js/motions/base.js +++ b/openslides/motions/static/js/motions/base.js @@ -192,6 +192,7 @@ angular.module('OpenSlidesApp.motions', [ .factory('Motion', [ 'DS', '$http', + '$filter', 'MotionPoll', 'MotionChangeRecommendation', 'MotionComment', @@ -204,7 +205,7 @@ angular.module('OpenSlidesApp.motions', [ 'OpenSlidesSettings', 'Projector', 'operator', - function(DS, $http, MotionPoll, MotionChangeRecommendation, MotionComment, jsDataModel, gettext, gettextCatalog, + function(DS, $http, $filter, MotionPoll, MotionChangeRecommendation, MotionComment, jsDataModel, gettext, gettextCatalog, Config, lineNumberingService, diffService, OpenSlidesSettings, Projector, operator) { var name = 'motions/motion'; return DS.defineResource({ @@ -385,9 +386,25 @@ angular.module('OpenSlidesApp.motions', [ getSearchResultName: function () { return this.getTitle(); }, - // subtitle of search result - getSearchResultSubtitle: function () { - return "Motion"; + // return true if a specific relation matches for given searchquery + // e.g. submitter, supporters or category + hasSearchResult: function (results, searchquery) { + var motion = this; + // search for submitters and supporters (check if any user.id from already found users matches) + var foundSomething = _.some(results, function(result) { + if (result.getResourceName() === "users/user") { + if (_.some(motion.submitters, {'id': result.id})) { + return true; + } else if (_.some(motion.supporters, { 'id': result.id })) { + return true; + } + } + }); + // search for category + if (!foundSomething && motion.category && motion.category.name.match(new RegExp(searchquery, 'i'))) { + foundSomething = true; + } + return foundSomething; }, getChangeRecommendations: function (versionId, order) { /* diff --git a/openslides/motions/static/js/motions/site.js b/openslides/motions/static/js/motions/site.js index e15087df7..a37b5fd80 100644 --- a/openslides/motions/static/js/motions/site.js +++ b/openslides/motions/static/js/motions/site.js @@ -26,6 +26,19 @@ angular.module('OpenSlidesApp.motions.site', [ } ]) +.config([ + 'SearchProvider', + 'gettext', + function (SearchProvider, gettext) { + SearchProvider.register({ + 'verboseName': gettext('Motions'), + 'collectionName': 'motions/motion', + 'urlDetailState': 'motions.motion.detail', + 'weight': 300, + }); + } +]) + .config([ '$stateProvider', 'gettext', diff --git a/openslides/topics/models.py b/openslides/topics/models.py index d60e162e7..58c59cb11 100644 --- a/openslides/topics/models.py +++ b/openslides/topics/models.py @@ -63,11 +63,3 @@ class Topic(RESTModelMixin, models.Model): def get_agenda_list_view_title(self): return self.title - - def get_search_index_string(self): - """ - Returns a string that can be indexed for the search. - """ - return " ".join(( - self.title, - self.text)) diff --git a/openslides/topics/static/js/topics/base.js b/openslides/topics/static/js/topics/base.js index 4cceb5b37..5cb6e5cd6 100644 --- a/openslides/topics/static/js/topics/base.js +++ b/openslides/topics/static/js/topics/base.js @@ -21,14 +21,6 @@ angular.module('OpenSlidesApp.topics', []) getAgendaTitle: function () { return this.title; }, - // link name which is shown in search result - getSearchResultName: function () { - return this.getAgendaTitle(); - }, - // subtitle of search result - getSearchResultSubtitle: function () { - return 'Topic'; - }, }, relations: { belongsTo: { diff --git a/openslides/users/models.py b/openslides/users/models.py index 51c8741cc..70a5cf5e0 100644 --- a/openslides/users/models.py +++ b/openslides/users/models.py @@ -12,8 +12,6 @@ from django.contrib.auth.models import ( from django.db import models from django.db.models import Prefetch, Q -from openslides.utils.search import user_name_helper - from ..utils.collection import CollectionElement from ..utils.models import RESTModelMixin from .access_permissions import GroupAccessPermissions, UserAccessPermissions @@ -207,15 +205,6 @@ class User(RESTModelMixin, PermissionsMixin, AbstractBaseUser): CollectionElement.from_instance(self) return super().save(*args, **kwargs) - def get_search_index_string(self): - """ - Returns a string that can be indexed for the search. - """ - return " ".join(( - user_name_helper(self), - self.structure_level, - self.about_me)) - def has_perm(self, perm): """ This method is closed. Do not use it but use openslides.utils.auth.has_perm. diff --git a/openslides/users/static/js/users/site.js b/openslides/users/static/js/users/site.js index 10dd09349..94a9e40e9 100644 --- a/openslides/users/static/js/users/site.js +++ b/openslides/users/static/js/users/site.js @@ -22,6 +22,18 @@ angular.module('OpenSlidesApp.users.site', [ }); } ]) +.config([ + 'SearchProvider', + 'gettext', + function (SearchProvider, gettext) { + SearchProvider.register({ + 'verboseName': gettext('Participants'), + 'collectionName': 'users/user', + 'urlDetailState': 'users.user.detail', + 'weight': 500, + }); + } +]) .config([ '$stateProvider', diff --git a/openslides/utils/main.py b/openslides/utils/main.py index 697837402..d47f68a40 100644 --- a/openslides/utils/main.py +++ b/openslides/utils/main.py @@ -137,7 +137,7 @@ def get_default_settings_context(user_data_path=None): The argument 'user_data_path' is a given path for user specific data or None. """ - # Setup path for user specific data (SQLite3 database, media, search index, ...): + # Setup path for user specific data (SQLite3 database, media, ...): # Take it either from command line or get default path default_context = {} if user_data_path: diff --git a/openslides/utils/search.py b/openslides/utils/search.py deleted file mode 100644 index 4193ac87f..000000000 --- a/openslides/utils/search.py +++ /dev/null @@ -1,180 +0,0 @@ -import os -import shutil - -from django.conf import settings -from django.contrib.auth import get_user_model -from django.core.exceptions import ImproperlyConfigured -from django.db.models import QuerySet -from whoosh import fields -from whoosh.filedb.filestore import RamStorage -from whoosh.index import create_in, exists_in, open_dir -from whoosh.qparser import QueryParser -from whoosh.writing import AsyncWriter - - -def get_schema(): - """ - This method creates the whoosh schema. It is only needed when the search - index is build. After this, the schema is saved and loaded with the index. - - When the schema is changed, then the index has to be recreated or the index - has to be altert. See: - https://pythonhosted.org/Whoosh/schema.html#modifying-the-schema-after-indexing - """ - return fields.Schema( - id=fields.ID(stored=True), - collection=fields.ID(stored=True), - id_collection=fields.ID(unique=True), - content=fields.TEXT) - - -class Index: - """ - Represents the whoosh index. - """ - - def get_index_path(self): - """ - Returns the index path. - - Raises ImproperlyConfigured if the path is not set in the settings. - """ - try: - return settings.SEARCH_INDEX - except AttributeError: - raise ImproperlyConfigured("Set SEARCH_INDEX into your settings.") - - def create_index(self): - """ - Creats the whoosh index. Delets an existing index if exists. - - Returns the index. - """ - path = self.get_index_path() - if path == 'ram': - self.storage = RamStorage().create_index(get_schema()) - else: - if os.path.exists(path): - shutil.rmtree(path) - os.mkdir(path) - self.storage = create_in(path, get_schema()) - return self.storage - - def get_or_create_index(self): - """ - Returns an index object. - - Creats the index if it does not exist - """ - # Try to return a storage object that was created before. - try: - return self.storage - except AttributeError: - pass - path = self.get_index_path() - if path != 'ram' and exists_in(path): - # Quick fix to bypass errors when many clients login. - # TODO: Solve this hack. - try: - return open_dir(path) - except FileNotFoundError: - pass - return self.create_index() - - -index = Index() - - -def combine_id_and_collection(instance): - """ - Returns a string where the id and the collection string of an instance - are combined. - """ - return "{}{}".format(instance.id, instance.get_collection_string()) - - -def user_name_helper(users): - """ - Helper to index a user or a list of users. - - Returns a string which contains the names of all users seperated by a space. - - users can be a list, a queryset or an user object. If it is something else - then the str(users) is returned. - """ - if isinstance(users, list) or isinstance(users, QuerySet): - user_string = " ".join(str(user) for user in users) - elif isinstance(users, get_user_model()): - user_string = str(users) - else: - user_string = str(users) - return user_string - - -def index_add_instance(sender, instance, **kwargs): - """ - Receiver that should be called by the post_save signal and the m2m_changed - signal. - - If the instance has an method get_search_string, then it is written - into the search index. The method has to return an dictonary that can be - used as keyword arguments to writer.add_document. - - This function uses whoosh.writing.AsyncWriter. - """ - try: - get_search_index_string = instance.get_search_index_string - except AttributeError: - # If the instance is not searchable, then exit this signal early. - return - - created = kwargs.get('created', False) - - writer_kwargs = { - 'id_collection': combine_id_and_collection(instance), - 'id': str(instance.pk), - 'collection': instance.get_collection_string(), - 'content': get_search_index_string()} - - with AsyncWriter(index.get_or_create_index()) as writer: - if created: - writer.add_document(**writer_kwargs) - else: - writer.update_document(**writer_kwargs) - - -def index_del_instance(sender, instance, **kwargs): - """ - Like index_add_instance but deletes the instance from the index. - - Should be called by the post_delete signal. - - This function uses whoosh.writing.AsyncWriter. - """ - try: - # Try to get the arrribute get_search_attributes. It is not needed - # in this method (and therefore not called) but it tells us if the - # instance is searchable. - instance.get_search_index_string - except AttributeError: - # If the instance is not searchable, then exit this signal early. - return - - with AsyncWriter(index.get_or_create_index()) as writer: - writer.delete_by_term('id_collection', combine_id_and_collection(instance)) - - -def search(query): - """ - Searchs elements. - - query has to be a query string. See: https://pythonhosted.org/Whoosh/querylang.html - - The return value is a list of dictonaries where each dictonary has the keys - id and collection. - """ - search_index = index.get_or_create_index() - parser = QueryParser("content", search_index.schema) - query = parser.parse(query) - result = search_index.searcher().search(query, limit=None) - return [dict(element) for element in result] diff --git a/openslides/utils/settings.py.tpl b/openslides/utils/settings.py.tpl index 00abad7b0..f9b265146 100644 --- a/openslides/utils/settings.py.tpl +++ b/openslides/utils/settings.py.tpl @@ -130,12 +130,6 @@ STATICFILES_DIRS = [os.path.join(OPENSLIDES_USER_DATA_PATH, 'static')] + STATICF MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_PATH, 'media', '') -# Whoosh search library -# https://whoosh.readthedocs.io/en/latest/ - -SEARCH_INDEX = os.path.join(OPENSLIDES_USER_DATA_PATH, 'search_index') - - # Customization of OpenSlides apps MOTION_IDENTIFIER_MIN_DIGITS = 1 diff --git a/requirements_production.txt b/requirements_production.txt index a74509a17..0a8f51e49 100644 --- a/requirements_production.txt +++ b/requirements_production.txt @@ -6,5 +6,4 @@ jsonfield>=1.0,<1.1 PyPDF2>=1.26,<1.27 roman>=2.0,<2.1 setuptools>=29.0,<35.0 -Whoosh>=2.7,<2.8 bleach>=1.5.0,<1.6 diff --git a/tests/old/settings.py b/tests/old/settings.py index 93b3426f5..33fe3429f 100644 --- a/tests/old/settings.py +++ b/tests/old/settings.py @@ -60,12 +60,6 @@ STATICFILES_DIRS.insert(0, os.path.join(OPENSLIDES_USER_DATA_PATH, 'static')) # MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_PATH, '') -# Whoosh search library -# https://whoosh.readthedocs.io/en/latest/ - -SEARCH_INDEX = 'ram' - - # Customization of OpenSlides apps MOTION_IDENTIFIER_MIN_DIGITS = 1 diff --git a/tests/settings.py b/tests/settings.py index 9aabb13af..4c771882d 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -60,12 +60,6 @@ STATICFILES_DIRS.insert(0, os.path.join(OPENSLIDES_USER_DATA_PATH, 'static')) # MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_PATH, '') -# Whoosh search library -# https://whoosh.readthedocs.io/en/latest/ - -SEARCH_INDEX = 'ram' - - # Customization of OpenSlides apps MOTION_IDENTIFIER_MIN_DIGITS = 1