Merge pull request #2943 from emanuelschuetze/issue2865

New full text search on client-side (Fixed #2865).
This commit is contained in:
Emanuel Schütze 2017-02-10 23:05:15 +01:00 committed by GitHub
commit da61aac89a
29 changed files with 227 additions and 416 deletions

View File

@ -47,6 +47,7 @@ Core:
- Validate HTML strings from CKEditor against XSS attacks. - Validate HTML strings from CKEditor against XSS attacks.
- Added success/error symbol to config to show if saving was successful. - Added success/error symbol to config to show if saving was successful.
- Added UTF-8 byte order mark for every CSV export. - Added UTF-8 byte order mark for every CSV export.
- Moved full-text search to client-side (removed the server-side search engine Whoosh).
Motions: Motions:
- Added adjustable line numbering mode (outside, inside, none) for each - Added adjustable line numbering mode (outside, inside, none) for each

View File

@ -179,8 +179,6 @@ OpenSlides uses the following projects or parts of them:
* `txaio <https://github.com/crossbario/txaio>`_, License: MIT * `txaio <https://github.com/crossbario/txaio>`_, License: MIT
* `Whoosh <https://bitbucket.org/mchaput/whoosh/wiki/Home>`_, License: BSD
* `zope.interface <https://github.com/zopefoundation/zope.interface>`, * `zope.interface <https://github.com/zopefoundation/zope.interface>`,
License: ZPL 2.1 License: ZPL 2.1

View File

@ -87,6 +87,23 @@ angular.module('OpenSlidesApp.agenda', ['OpenSlidesApp.users'])
getAgendaTitle: function () { getAgendaTitle: function () {
return this.title; return this.title;
}, },
// link name which is shown in search result
getSearchResultName: function () {
return this.getAgendaTitle();
},
// return true if a specific relation matches for given searchquery
// (here: speakers)
hasSearchResult: function (results) {
var item = this;
// search for speakers (check if any user.id from already found users matches)
return _.some(results, function(result) {
if (result.getResourceName() === "users/user") {
if (_.some(item.speakers, {'user_id': result.id})) {
return true;
}
}
});
},
getListViewTitle: function () { getListViewTitle: function () {
var title; var title;
try { try {

View File

@ -23,6 +23,19 @@ angular.module('OpenSlidesApp.agenda.site', [
} }
]) ])
.config([
'SearchProvider',
'gettext',
function (SearchProvider, gettext) {
SearchProvider.register({
'verboseName': gettext('Agenda'),
'collectionName': 'agenda/item',
'urlDetailState': 'agenda.item.detail',
'weight': 200,
});
}
])
.config([ .config([
'$stateProvider', '$stateProvider',
'gettext', 'gettext',

View File

@ -19,7 +19,6 @@ from openslides.poll.models import (
from openslides.utils.autoupdate import inform_changed_data from openslides.utils.autoupdate import inform_changed_data
from openslides.utils.exceptions import OpenSlidesError from openslides.utils.exceptions import OpenSlidesError
from openslides.utils.models import RESTModelMixin from openslides.utils.models import RESTModelMixin
from openslides.utils.search import user_name_helper
from .access_permissions import AssignmentAccessPermissions from .access_permissions import AssignmentAccessPermissions
@ -354,16 +353,6 @@ class Assignment(RESTModelMixin, models.Model):
""" """
return self.agenda_item.pk return self.agenda_item.pk
def get_search_index_string(self):
"""
Returns a string that can be indexed for the search.
"""
return " ".join((
self.title,
self.description,
user_name_helper(self.related_users.all()),
" ".join(tag.name for tag in self.tags.all())))
class AssignmentVote(RESTModelMixin, BaseVote): class AssignmentVote(RESTModelMixin, BaseVote):
option = models.ForeignKey( option = models.ForeignKey(

View File

@ -341,9 +341,18 @@ angular.module('OpenSlidesApp.assignments', [])
getSearchResultName: function () { getSearchResultName: function () {
return this.getAgendaTitle(); return this.getAgendaTitle();
}, },
// subtitle of search result // return true if a specific relation matches for given searchquery
getSearchResultSubtitle: function () { // (here: related_users/candidates)
return "Election"; hasSearchResult: function (results) {
var assignment = this;
// search for related users (check if any user.id from already found users matches)
return _.some(results, function(result) {
if (result.getResourceName() === "users/user") {
if (_.some(assignment.assignment_related_users, {'user_id': result.id})) {
return true;
}
}
});
}, },
// override project function of jsDataModel factory // override project function of jsDataModel factory
project: function (projectorId, pollId) { project: function (projectorId, pollId) {

View File

@ -23,6 +23,19 @@ angular.module('OpenSlidesApp.assignments.site', [
} }
]) ])
.config([
'SearchProvider',
'gettext',
function (SearchProvider, gettext) {
SearchProvider.register({
'verboseName': gettext('Elections'),
'collectionName': 'assignments/assignment',
'urlDetailState': 'assignments.assignment.detail',
'weight': 400,
});
}
])
.config([ .config([
'$stateProvider', '$stateProvider',
'gettext', 'gettext',

View File

@ -14,11 +14,9 @@ class CoreAppConfig(AppConfig):
from . import projector # noqa from . import projector # noqa
# Import all required stuff. # Import all required stuff.
from django.db.models import signals
from .config import config from .config import config
from .signals import post_permission_creation from .signals import post_permission_creation
from ..utils.rest_api import router from ..utils.rest_api import router
from ..utils.search import index_add_instance, index_del_instance
from .config_variables import get_config_variables from .config_variables import get_config_variables
from .signals import delete_django_app_permissions from .signals import delete_django_app_permissions
from .views import ( from .views import (
@ -46,17 +44,6 @@ class CoreAppConfig(AppConfig):
router.register(self.get_model('ProjectorMessage').get_collection_string(), ProjectorMessageViewSet) router.register(self.get_model('ProjectorMessage').get_collection_string(), ProjectorMessageViewSet)
router.register(self.get_model('Countdown').get_collection_string(), CountdownViewSet) router.register(self.get_model('Countdown').get_collection_string(), CountdownViewSet)
# Update the search when a model is saved or deleted
signals.post_save.connect(
index_add_instance,
dispatch_uid='index_add_instance')
signals.post_delete.connect(
index_del_instance,
dispatch_uid='index_del_instance')
signals.m2m_changed.connect(
index_add_instance,
dispatch_uid='m2m_index_add_instance')
def get_startup_elements(self): def get_startup_elements(self):
from .config import config from .config import config
from ..utils.collection import Collection from ..utils.collection import Collection

View File

@ -1406,8 +1406,14 @@ img {
.searchresults li { .searchresults li {
margin-bottom: 12px; margin-bottom: 12px;
} }
.searchresults li .subtitle { .searchresults h3 {
margin-bottom: 3px;
padding-bottom: 0;
}
.searchresults .hits {
margin-bottom: 10px;
color: #999999; color: #999999;
font-size: 85%;
} }
/* ngDialog: override ngdialog-theme-default */ /* ngDialog: override ngdialog-theme-default */

View File

@ -97,6 +97,27 @@ angular.module('OpenSlidesApp.core.site', [
} }
]) ])
// Provider to register a searchable module/app.
.provider('Search', [
function() {
var searchModules = [];
this.register = function(module) {
searchModules.push(module);
};
this.$get = [
function () {
return {
getAll: function () {
return searchModules;
}
};
}
];
}
])
.run([ .run([
'editableOptions', 'editableOptions',
'gettext', 'gettext',
@ -952,76 +973,59 @@ angular.module('OpenSlidesApp.core.site', [
// Search Controller // Search Controller
.controller('SearchCtrl', [ .controller('SearchCtrl', [
'$scope', '$scope',
'$http', '$filter',
'$stateParams', '$stateParams',
'$location', 'Search',
'$sanitize',
'DS', 'DS',
function ($scope, $http, $stateParams, $location, $sanitize, DS) { 'Motion',
$scope.fullword = false; function ($scope, $filter, $stateParams, Search, DS, Motion) {
$scope.filterAgenda = true; $scope.searchresults = [];
$scope.filterMotion = true; var searchModules = Search.getAll();
$scope.filterAssignment = true;
$scope.filterUser = true;
$scope.filterMedia = true;
// search function // search function
$scope.search = function() { $scope.search = function() {
var query = _.escape($scope.query); $scope.results = [];
if (query !== '') { var foundObjects = [];
var lastquery = query; // search in rest properties of all defined searchModule
// attach asterisks if search is not for full words only // (does not found any related objects, e.g. speakers of items)
if (!$scope.fullword) { _.forEach(searchModules, function(searchModule) {
if (query.charAt(0) != '*'){ var result = {};
query = "*" + query; result.verboseName = searchModule.verboseName;
} result.collectionName = searchModule.collectionName;
if (query.charAt(query.length - 1) != '*'){ result.urlDetailState = searchModule.urlDetailState;
query = query + "*"; result.weight = searchModule.weight;
} result.checked = true;
} result.elements = $filter('filter')(DS.getAll(searchModule.collectionName), $scope.searchquery);
$scope.query = lastquery; $scope.results.push(result);
$http.get('/core/search_api/?q=' + query).then(function(success) { _.forEach(result.elements, function(element) {
$scope.results = []; foundObjects.push(element);
var elements = success.data.elements;
angular.forEach(elements, function(element) {
DS.find(element.collection, element.id).then(function(data) {
data.urlState = element.collection.replace('/','.')+'.detail';
data.urlParam = {id: element.id};
$scope.results.push(data);
});
});
}); });
$location.url('/search/?q=' + lastquery); });
} // search additionally in specific releations of all defined searchModules
_.forEach(searchModules, function(searchModule) {
_.forEach(DS.getAll(searchModule.collectionName), function(object) {
if (_.isFunction(object.hasSearchResult)) {
if (object.hasSearchResult(foundObjects, $scope.searchquery)) {
// releation found, check if object is not yet in search results
_.forEach($scope.results, function(result) {
if ((object.getResourceName() === result.collectionName) &&
_.findIndex(result.elements, {'id': object.id}) === -1) {
result.elements.push(object);
}
});
}
} else {
return false;
}
});
});
}; };
//get search string from parameters submitted from outside the scope //get search string from parameters submitted from outside the scope
if ($stateParams.q) { if ($stateParams.q) {
$scope.query = $stateParams.q; $scope.searchquery = $stateParams.q;
$scope.search(); $scope.search();
} }
// returns element if part of the current search selection
$scope.filterresult = function() {
return function(result) {
if ($scope.filterUser && result.urlState == 'users.user.detail') {
return result;
}
if ($scope.filterMotion && result.urlState == 'motions.motion.detail') {
return result;
}
if ($scope.filterAgenda && result.urlState == 'topics.topic.detail') {
return result;
}
if ($scope.filterAssignment && result.urlState == 'assignments.assignment.detail') {
return result;
}
if ($scope.filterMedia && result.urlState== 'mediafiles.mediafile.detail') {
return result;
}
return;
};
};
} }
]) ])

View File

@ -5,52 +5,38 @@
</div> </div>
<div class="details"> <div class="details">
<form class="input-group" ng-submit="search()"> <form class="input-group" ng-submit="search()">
<input type="text" ng-model="query" class="form-control"> <input type="text" ng-change="search()" ng-model="searchquery" class="form-control">
<span class="input-group-btn"> <span class="input-group-btn">
<button type="submit" class="btn btn-default" translate>Search</button> <button type="submit" class="btn btn-default" translate>Search</button>
</span> </span>
</form> </form>
<div class="searchfilter spacer-top">
<label class="checkbox-inline"> <div class="searchfilter spacer-top">
<input type="checkbox" ng-model="filterAgenda"> <label ng-repeat="filter in results | orderBy: 'weight'" class="checkbox-inline">
<translate>Agenda items</translate> <input type="checkbox" ng-model="filter.checked">
</label> {{ filter.verboseName | translate }}
<label class="checkbox-inline"> </label>
<input type="checkbox" ng-model="filterMotion"> </div>
<translate>Motions</translate>
</label> <div class="searchresults spacer-top-lg">
<label class="checkbox-inline"> <div ng-show="results">
<input type="checkbox" ng-model="filterAssignment"> <div ng-repeat="result in results | orderBy: 'weight'" ng-if="result.checked && result.elements.length">
<translate>Elections</translate> <h3>{{ result.verboseName | translate }}</h3>
</label> <div class="hits">
<label class="checkbox-inline"> {{ result.elements.length}} <translate>results</translate>
<input type="checkbox" ng-model="filterUser"> </div>
<translate>Participants</translate> <ol class="list-unstyled">
</label> <li ng-repeat="element in result.elements">
<label class="checkbox-inline"> <a ng-if="!element.mediafileUrl" ui-sref="{{ result.urlDetailState }}({id: {{element.id}}})">
<input type="checkbox" ng-model="filterMedia"> {{ element.getSearchResultName() }}
<translate>Files</translate> </a>
</label> <a ng-if="element.mediafileUrl" href="{{ element.mediafileUrl }}" target="_blank">
<div> {{ element.getSearchResultName() }}
<label class="checkbox-inline"> </a>
<input type="checkbox" ng-model="fullword" ng-change="search()"> </ol>
<translate>Only whole words</translate>
</label>
</div> </div>
</div> </div>
<div class="searchresults spacer-top-lg">
<ol ng-show="results">
<li ng-repeat="result in results | filter:filterresult()">
<a ng-if="!result.mediafileUrl" ui-sref="{{ result.urlState }}({{ result.urlParam }})">
{{ result.getSearchResultName() }}
</a>
<a ng-if="result.mediafileUrl" href="{{ result.mediafileUrl }}" target="_blank">
{{ result.getSearchResultName() }}
</a>
<br>
<span class="grey">{{ result.getSearchResultSubtitle() | translate }}</span>
</ol>
<p ng-show="!results || results.length == 0" translate>No results.</p> <p ng-show="!results || results.length == 0" translate>No results.</p>
</div>
</div> </div>

View File

@ -11,10 +11,6 @@ urlpatterns = [
views.VersionView.as_view(), views.VersionView.as_view(),
name='core_version'), name='core_version'),
url(r'^core/search_api/$',
views.SearchView.as_view(),
name='core_search'),
url(r'^core/encode_media/$', url(r'^core/encode_media/$',
views.MediaEncoder.as_view(), views.MediaEncoder.as_view(),
name="core_mediaencoding"), name="core_mediaencoding"),

View File

@ -5,7 +5,6 @@ import uuid
from collections import OrderedDict from collections import OrderedDict
from operator import attrgetter from operator import attrgetter
from textwrap import dedent from textwrap import dedent
from urllib.parse import unquote
from django.apps import apps from django.apps import apps
from django.conf import settings from django.conf import settings
@ -34,7 +33,6 @@ from ..utils.rest_api import (
detail_route, detail_route,
list_route, list_route,
) )
from ..utils.search import search
from .access_permissions import ( from .access_permissions import (
ChatMessageAccessPermissions, ChatMessageAccessPermissions,
ConfigAccessPermissions, ConfigAccessPermissions,
@ -814,25 +812,6 @@ class VersionView(utils_views.APIView):
return result return result
class SearchView(utils_views.APIView):
"""
Accepts a search string and returns a list of objects where each object
is a dictonary with the keywords collection and id.
This view expects a get argument 'q' with a search string.
See: https://pythonhosted.org/Whoosh/querylang.html for the format of the
search string.
"""
http_method_names = ['get']
def get_context_data(self, **context):
query = self.request.GET.get('q', '')
return super().get_context_data(
elements=search(unquote(query)),
**context)
class MediaEncoder(utils_views.APIView): class MediaEncoder(utils_views.APIView):
""" """
MediaEncoder is a class based view to prepare encoded media for pdfMake MediaEncoder is a class based view to prepare encoded media for pdfMake

View File

@ -2,8 +2,6 @@ from django.conf import settings
from django.db import models from django.db import models
from django.utils.translation import ugettext as _ from django.utils.translation import ugettext as _
from openslides.utils.search import user_name_helper
from ..utils.models import RESTModelMixin from ..utils.models import RESTModelMixin
from .access_permissions import MediafileAccessPermissions from .access_permissions import MediafileAccessPermissions
@ -73,11 +71,3 @@ class Mediafile(RESTModelMixin, models.Model):
kB = size / 1024 kB = size / 1024
size_string = '%d kB' % kB size_string = '%d kB' % kB
return size_string return size_string
def get_search_index_string(self):
"""
Returns a string that can be indexed for the search.
"""
return " ".join((
self.title,
user_name_helper(self.uploader)))

View File

@ -36,9 +36,17 @@ angular.module('OpenSlidesApp.mediafiles.resources', [
getSearchResultName: function () { getSearchResultName: function () {
return this.title; return this.title;
}, },
// subtitle of search result // return true if a specific relation matches for given searchquery
getSearchResultSubtitle: function () { // (here: speakers)
return "File"; hasSearchResult: function (results) {
var mediafile = this;
// search for speakers (check if any user.id from already found users matches)
return _.some(results, function(result) {
if ((result.getResourceName() === "users/user") &&
(mediafile.uploader_id === result.id)) {
return true;
}
});
}, },
}, },
computed: { computed: {

View File

@ -22,6 +22,19 @@ angular.module('OpenSlidesApp.mediafiles.states', [
} }
]) ])
.config([
'SearchProvider',
'gettext',
function (SearchProvider, gettext) {
SearchProvider.register({
'verboseName': gettext('Files'),
'collectionName': 'mediafiles/mediafile',
'urlDetailState': 'mediafiles.mediafile.detail',
'weight': 600,
});
}
])
.config([ .config([
'gettext', 'gettext',
'$stateProvider', '$stateProvider',

View File

@ -20,7 +20,6 @@ from openslides.poll.models import (
) )
from openslides.utils.autoupdate import inform_changed_data from openslides.utils.autoupdate import inform_changed_data
from openslides.utils.models import RESTModelMixin from openslides.utils.models import RESTModelMixin
from openslides.utils.search import user_name_helper
from .access_permissions import ( from .access_permissions import (
CategoryAccessPermissions, CategoryAccessPermissions,
@ -648,19 +647,6 @@ class Motion(RESTModelMixin, models.Model):
yield amendment yield amendment
yield from amendment.get_amendments_deep() yield from amendment.get_amendments_deep()
def get_search_index_string(self):
"""
Returns a string that can be indexed for the search.
"""
return " ".join((
self.title or '',
self.text or '',
self.reason or '',
str(self.category) if self.category else '',
user_name_helper(self.submitters.all()),
user_name_helper(self.supporters.all()),
" ".join(tag.name for tag in self.tags.all())))
class MotionVersion(RESTModelMixin, models.Model): class MotionVersion(RESTModelMixin, models.Model):
""" """

View File

@ -192,6 +192,7 @@ angular.module('OpenSlidesApp.motions', [
.factory('Motion', [ .factory('Motion', [
'DS', 'DS',
'$http', '$http',
'$filter',
'MotionPoll', 'MotionPoll',
'MotionChangeRecommendation', 'MotionChangeRecommendation',
'MotionComment', 'MotionComment',
@ -204,7 +205,7 @@ angular.module('OpenSlidesApp.motions', [
'OpenSlidesSettings', 'OpenSlidesSettings',
'Projector', 'Projector',
'operator', 'operator',
function(DS, $http, MotionPoll, MotionChangeRecommendation, MotionComment, jsDataModel, gettext, gettextCatalog, function(DS, $http, $filter, MotionPoll, MotionChangeRecommendation, MotionComment, jsDataModel, gettext, gettextCatalog,
Config, lineNumberingService, diffService, OpenSlidesSettings, Projector, operator) { Config, lineNumberingService, diffService, OpenSlidesSettings, Projector, operator) {
var name = 'motions/motion'; var name = 'motions/motion';
return DS.defineResource({ return DS.defineResource({
@ -385,9 +386,25 @@ angular.module('OpenSlidesApp.motions', [
getSearchResultName: function () { getSearchResultName: function () {
return this.getTitle(); return this.getTitle();
}, },
// subtitle of search result // return true if a specific relation matches for given searchquery
getSearchResultSubtitle: function () { // e.g. submitter, supporters or category
return "Motion"; hasSearchResult: function (results, searchquery) {
var motion = this;
// search for submitters and supporters (check if any user.id from already found users matches)
var foundSomething = _.some(results, function(result) {
if (result.getResourceName() === "users/user") {
if (_.some(motion.submitters, {'id': result.id})) {
return true;
} else if (_.some(motion.supporters, { 'id': result.id })) {
return true;
}
}
});
// search for category
if (!foundSomething && motion.category && motion.category.name.match(new RegExp(searchquery, 'i'))) {
foundSomething = true;
}
return foundSomething;
}, },
getChangeRecommendations: function (versionId, order) { getChangeRecommendations: function (versionId, order) {
/* /*

View File

@ -26,6 +26,19 @@ angular.module('OpenSlidesApp.motions.site', [
} }
]) ])
.config([
'SearchProvider',
'gettext',
function (SearchProvider, gettext) {
SearchProvider.register({
'verboseName': gettext('Motions'),
'collectionName': 'motions/motion',
'urlDetailState': 'motions.motion.detail',
'weight': 300,
});
}
])
.config([ .config([
'$stateProvider', '$stateProvider',
'gettext', 'gettext',

View File

@ -63,11 +63,3 @@ class Topic(RESTModelMixin, models.Model):
def get_agenda_list_view_title(self): def get_agenda_list_view_title(self):
return self.title return self.title
def get_search_index_string(self):
"""
Returns a string that can be indexed for the search.
"""
return " ".join((
self.title,
self.text))

View File

@ -21,14 +21,6 @@ angular.module('OpenSlidesApp.topics', [])
getAgendaTitle: function () { getAgendaTitle: function () {
return this.title; return this.title;
}, },
// link name which is shown in search result
getSearchResultName: function () {
return this.getAgendaTitle();
},
// subtitle of search result
getSearchResultSubtitle: function () {
return 'Topic';
},
}, },
relations: { relations: {
belongsTo: { belongsTo: {

View File

@ -12,8 +12,6 @@ from django.contrib.auth.models import (
from django.db import models from django.db import models
from django.db.models import Prefetch, Q from django.db.models import Prefetch, Q
from openslides.utils.search import user_name_helper
from ..utils.collection import CollectionElement from ..utils.collection import CollectionElement
from ..utils.models import RESTModelMixin from ..utils.models import RESTModelMixin
from .access_permissions import GroupAccessPermissions, UserAccessPermissions from .access_permissions import GroupAccessPermissions, UserAccessPermissions
@ -207,15 +205,6 @@ class User(RESTModelMixin, PermissionsMixin, AbstractBaseUser):
CollectionElement.from_instance(self) CollectionElement.from_instance(self)
return super().save(*args, **kwargs) return super().save(*args, **kwargs)
def get_search_index_string(self):
"""
Returns a string that can be indexed for the search.
"""
return " ".join((
user_name_helper(self),
self.structure_level,
self.about_me))
def has_perm(self, perm): def has_perm(self, perm):
""" """
This method is closed. Do not use it but use openslides.utils.auth.has_perm. This method is closed. Do not use it but use openslides.utils.auth.has_perm.

View File

@ -22,6 +22,18 @@ angular.module('OpenSlidesApp.users.site', [
}); });
} }
]) ])
.config([
'SearchProvider',
'gettext',
function (SearchProvider, gettext) {
SearchProvider.register({
'verboseName': gettext('Participants'),
'collectionName': 'users/user',
'urlDetailState': 'users.user.detail',
'weight': 500,
});
}
])
.config([ .config([
'$stateProvider', '$stateProvider',

View File

@ -137,7 +137,7 @@ def get_default_settings_context(user_data_path=None):
The argument 'user_data_path' is a given path for user specific data or None. The argument 'user_data_path' is a given path for user specific data or None.
""" """
# Setup path for user specific data (SQLite3 database, media, search index, ...): # Setup path for user specific data (SQLite3 database, media, ...):
# Take it either from command line or get default path # Take it either from command line or get default path
default_context = {} default_context = {}
if user_data_path: if user_data_path:

View File

@ -1,180 +0,0 @@
import os
import shutil
from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.exceptions import ImproperlyConfigured
from django.db.models import QuerySet
from whoosh import fields
from whoosh.filedb.filestore import RamStorage
from whoosh.index import create_in, exists_in, open_dir
from whoosh.qparser import QueryParser
from whoosh.writing import AsyncWriter
def get_schema():
"""
This method creates the whoosh schema. It is only needed when the search
index is build. After this, the schema is saved and loaded with the index.
When the schema is changed, then the index has to be recreated or the index
has to be altert. See:
https://pythonhosted.org/Whoosh/schema.html#modifying-the-schema-after-indexing
"""
return fields.Schema(
id=fields.ID(stored=True),
collection=fields.ID(stored=True),
id_collection=fields.ID(unique=True),
content=fields.TEXT)
class Index:
"""
Represents the whoosh index.
"""
def get_index_path(self):
"""
Returns the index path.
Raises ImproperlyConfigured if the path is not set in the settings.
"""
try:
return settings.SEARCH_INDEX
except AttributeError:
raise ImproperlyConfigured("Set SEARCH_INDEX into your settings.")
def create_index(self):
"""
Creats the whoosh index. Delets an existing index if exists.
Returns the index.
"""
path = self.get_index_path()
if path == 'ram':
self.storage = RamStorage().create_index(get_schema())
else:
if os.path.exists(path):
shutil.rmtree(path)
os.mkdir(path)
self.storage = create_in(path, get_schema())
return self.storage
def get_or_create_index(self):
"""
Returns an index object.
Creats the index if it does not exist
"""
# Try to return a storage object that was created before.
try:
return self.storage
except AttributeError:
pass
path = self.get_index_path()
if path != 'ram' and exists_in(path):
# Quick fix to bypass errors when many clients login.
# TODO: Solve this hack.
try:
return open_dir(path)
except FileNotFoundError:
pass
return self.create_index()
index = Index()
def combine_id_and_collection(instance):
"""
Returns a string where the id and the collection string of an instance
are combined.
"""
return "{}{}".format(instance.id, instance.get_collection_string())
def user_name_helper(users):
"""
Helper to index a user or a list of users.
Returns a string which contains the names of all users seperated by a space.
users can be a list, a queryset or an user object. If it is something else
then the str(users) is returned.
"""
if isinstance(users, list) or isinstance(users, QuerySet):
user_string = " ".join(str(user) for user in users)
elif isinstance(users, get_user_model()):
user_string = str(users)
else:
user_string = str(users)
return user_string
def index_add_instance(sender, instance, **kwargs):
"""
Receiver that should be called by the post_save signal and the m2m_changed
signal.
If the instance has an method get_search_string, then it is written
into the search index. The method has to return an dictonary that can be
used as keyword arguments to writer.add_document.
This function uses whoosh.writing.AsyncWriter.
"""
try:
get_search_index_string = instance.get_search_index_string
except AttributeError:
# If the instance is not searchable, then exit this signal early.
return
created = kwargs.get('created', False)
writer_kwargs = {
'id_collection': combine_id_and_collection(instance),
'id': str(instance.pk),
'collection': instance.get_collection_string(),
'content': get_search_index_string()}
with AsyncWriter(index.get_or_create_index()) as writer:
if created:
writer.add_document(**writer_kwargs)
else:
writer.update_document(**writer_kwargs)
def index_del_instance(sender, instance, **kwargs):
"""
Like index_add_instance but deletes the instance from the index.
Should be called by the post_delete signal.
This function uses whoosh.writing.AsyncWriter.
"""
try:
# Try to get the arrribute get_search_attributes. It is not needed
# in this method (and therefore not called) but it tells us if the
# instance is searchable.
instance.get_search_index_string
except AttributeError:
# If the instance is not searchable, then exit this signal early.
return
with AsyncWriter(index.get_or_create_index()) as writer:
writer.delete_by_term('id_collection', combine_id_and_collection(instance))
def search(query):
"""
Searchs elements.
query has to be a query string. See: https://pythonhosted.org/Whoosh/querylang.html
The return value is a list of dictonaries where each dictonary has the keys
id and collection.
"""
search_index = index.get_or_create_index()
parser = QueryParser("content", search_index.schema)
query = parser.parse(query)
result = search_index.searcher().search(query, limit=None)
return [dict(element) for element in result]

View File

@ -130,12 +130,6 @@ STATICFILES_DIRS = [os.path.join(OPENSLIDES_USER_DATA_PATH, 'static')] + STATICF
MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_PATH, 'media', '') MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_PATH, 'media', '')
# Whoosh search library
# https://whoosh.readthedocs.io/en/latest/
SEARCH_INDEX = os.path.join(OPENSLIDES_USER_DATA_PATH, 'search_index')
# Customization of OpenSlides apps # Customization of OpenSlides apps
MOTION_IDENTIFIER_MIN_DIGITS = 1 MOTION_IDENTIFIER_MIN_DIGITS = 1

View File

@ -6,5 +6,4 @@ jsonfield>=1.0,<1.1
PyPDF2>=1.26,<1.27 PyPDF2>=1.26,<1.27
roman>=2.0,<2.1 roman>=2.0,<2.1
setuptools>=29.0,<35.0 setuptools>=29.0,<35.0
Whoosh>=2.7,<2.8
bleach>=1.5.0,<1.6 bleach>=1.5.0,<1.6

View File

@ -60,12 +60,6 @@ STATICFILES_DIRS.insert(0, os.path.join(OPENSLIDES_USER_DATA_PATH, 'static')) #
MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_PATH, '') MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_PATH, '')
# Whoosh search library
# https://whoosh.readthedocs.io/en/latest/
SEARCH_INDEX = 'ram'
# Customization of OpenSlides apps # Customization of OpenSlides apps
MOTION_IDENTIFIER_MIN_DIGITS = 1 MOTION_IDENTIFIER_MIN_DIGITS = 1

View File

@ -60,12 +60,6 @@ STATICFILES_DIRS.insert(0, os.path.join(OPENSLIDES_USER_DATA_PATH, 'static')) #
MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_PATH, '') MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_PATH, '')
# Whoosh search library
# https://whoosh.readthedocs.io/en/latest/
SEARCH_INDEX = 'ram'
# Customization of OpenSlides apps # Customization of OpenSlides apps
MOTION_IDENTIFIER_MIN_DIGITS = 1 MOTION_IDENTIFIER_MIN_DIGITS = 1