ag-admin/OpenSlides
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Motion import improved:

Browse Source 
- add semicolon support (use utils/csv_ext.py as participant import)
- show all warnings of each line in one warning message
- show success message with total counter (x of n motions imported).
- improved warning and error strings
This commit is contained in:
Emanuel Schuetze 2013-05-29 22:29:08 +02:00
parent 4ec33106ed
commit a97cf4a680
4 changed files with 42 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
openslides/locale/de/LC_MESSAGES/django.po
View File

@ -1970,7 +1970,7 @@ msgstr "Import auf Grund von schweren Fehlern in der Quelldatei abgebrochen."
msgid "Import file has wrong character encoding, only UTF-8 is supported!" msgid "Import file has wrong character encoding, only UTF-8 is supported!"
msgstr "" msgstr ""
"Die Quelldatei benutzt eine ungültige Zeichenkodierung, es wird nur UTF-8 " "Die Quelldatei benutzt eine ungültige Zeichenkodierung, es wird nur UTF-8 "
"wird unterstützt!" "unterstützt!"
#: participant/forms.py:29 participant/views.py:542 #: participant/forms.py:29 participant/views.py:542
#: participant/templates/participant/group_overview.html:7 #: participant/templates/participant/group_overview.html:7

39
openslides/motion/csv_import.py
View File

@ -19,7 +19,9 @@ import csv
from django.db import transaction from django.db import transaction
from django.utils.translation import ugettext as _, ugettext_noop from django.utils.translation import ugettext as _, ugettext_noop
from openslides.utils import csv_ext
from openslides.utils.person.api import Persons from openslides.utils.person.api import Persons
from openslides.utils.utils import html_strong
from .models import Motion, Category from .models import Motion, Category
@ -37,25 +39,33 @@ def import_motions(csv_file, default_submitter, override=False, importing_person
error_messages = [] error_messages = []
warning_messages = [] warning_messages = []
count_success = 0 count_success = 0
count_lines = 0
# Check encoding # Check encoding
try: try:
csv_file.read().decode('utf8') csv_file.read().decode('utf8')
except UnicodeDecodeError: except UnicodeDecodeError:
return (0, [_('Encoding error in import file. Ensure using UTF-8.')], []) return (0, [_('Import file has wrong character encoding, only UTF-8 is supported!')], [])
csv_file.seek(0) csv_file.seek(0)
with transaction.commit_on_success(): with transaction.commit_on_success():
for (line_no, line) in enumerate(csv.reader(csv_file)): dialect = csv.Sniffer().sniff(csv_file.readline())
dialect = csv_ext.patchup(dialect)
csv_file.seek(0)
for (line_no, line) in enumerate(csv.reader(csv_file, dialect=dialect)):
warnings = []
if line_no < 1: if line_no < 1:
# Do not read the header line # Do not read the header line
continue continue
count_lines += 1
# Check format # Check format
try: try:
(identifier, title, text, reason, submitter, category) = line[:6] (identifier, title, text, reason, submitter, category) = line[:6]
except ValueError: except ValueError:
error_messages.append(_('Ignoring malformed line %d in import file.') % (line_no + 1)) error_line = html_strong(_('Line %d of import file:') % (line_no + 1))
msg = _('Line is malformed. Motion not imported. Please check the required values.')
error_messages.append("%s<br>%s" % (error_line, msg))
continue continue
# Check existing motions according to the identifier # Check existing motions according to the identifier
@ -66,7 +76,9 @@ def import_motions(csv_file, default_submitter, override=False, importing_person
motion = Motion(identifier=identifier) motion = Motion(identifier=identifier)
else: else:
if not override: if not override:
error_messages.append(_('Line %d in import file: Ignoring existing motion.') % (line_no + 1)) error_line = html_strong(_('Line %d of import file:') % (line_no + 1))
msg = _('Identifier already exists. Motion not imported.')
error_messages.append("%s<br>%s" % (error_line, msg))
continue continue
else: else:
motion = Motion() motion = Motion()
@ -79,9 +91,9 @@ def import_motions(csv_file, default_submitter, override=False, importing_person
try: try:
motion.category = Category.objects.get(name=category) motion.category = Category.objects.get(name=category)
except Category.DoesNotExist: except Category.DoesNotExist:
error_messages.append(_('Line %d in import file: Category not found.') % (line_no + 1)) warnings.append(_('Category unknown. No category is used.'))
except Category.MultipleObjectsReturned: except Category.MultipleObjectsReturned:
error_messages.append(_('Line %d in import file: Multiple categories found.') % (line_no + 1)) warnings.append(_('Several suitable categories found. No category is used.'))
motion.save() motion.save()
# Add submitter # Add submitter
@ -90,15 +102,24 @@ def import_motions(csv_file, default_submitter, override=False, importing_person
for person in Persons(): for person in Persons():
if person.clean_name == submitter.decode('utf8'): if person.clean_name == submitter.decode('utf8'):
if person_found: if person_found:
error_messages.append(_('Line %d in import file: Multiple persons found.') % (line_no + 1)) warnings.append(_('Several suitable submitters found.'))
person_found = False person_found = False
break break
else: else:
new_submitter = person new_submitter = person
person_found = True person_found = True
if not person_found: if not person_found:
warning_messages.append(_('Line %d in import file: Default submitter is used.') % (line_no + 1)) warnings.append(_('Submitter unknown. Default submitter is used.'))
new_submitter = default_submitter new_submitter = default_submitter
# show summarized warning message for each import line
if warnings:
warning_line = _('Line %d of import file:') % (line_no + 1)
warning_message_string = "%s<ul>" % html_strong(warning_line)
for w in warnings:
warning_message_string += "<li>%s</li>" % w
warning_message_string += "</ul>"
warning_messages.append(warning_message_string)
motion.clear_submitters() motion.clear_submitters()
motion.add_submitter(new_submitter) motion.add_submitter(new_submitter)
@ -106,4 +127,4 @@ def import_motions(csv_file, default_submitter, override=False, importing_person
person=importing_person) person=importing_person)
count_success += 1 count_success += 1
return (count_success, error_messages, warning_messages) return (count_success, count_lines, error_messages, warning_messages)

4
openslides/motion/views.py
View File

@ -707,7 +707,7 @@ class MotionCSVImportView(FormView):
""" """
Processes the import function. Processes the import function.
""" """
count_success, error_messages, warning_messages = import_motions( count_success, count_lines, error_messages, warning_messages = import_motions(
self.request.FILES['csvfile'], self.request.FILES['csvfile'],
default_submitter=form.cleaned_data['default_submitter'], default_submitter=form.cleaned_data['default_submitter'],
override=form.cleaned_data['override'], override=form.cleaned_data['override'],
@ -719,7 +719,7 @@ class MotionCSVImportView(FormView):
if count_success: if count_success:
messages.success( messages.success(
self.request, self.request,
_('%d motions were successfully imported.') % count_success) "<strong>%s</strong><br>%s" % (_('Summary'), _('%d of %d motions successfully imported.') % (count_success, count_lines)))
return super(MotionCSVImportView, self).form_valid(form) return super(MotionCSVImportView, self).form_valid(form)
motion_csv_import = MotionCSVImportView.as_view() motion_csv_import = MotionCSVImportView.as_view()

18
tests/motion/test_csv_import.py
View File

@ -56,7 +56,7 @@ class CSVImport(TestCase):
csv_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'extras', 'csv-examples') csv_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'extras', 'csv-examples')
self.assertEqual(Motion.objects.count(), 0) self.assertEqual(Motion.objects.count(), 0)
with open(csv_dir + '/motions-demo_de.csv') as f: with open(csv_dir + '/motions-demo_de.csv') as f:
count_success, error_messages, warning_messages = import_motions(csv_file=f, default_submitter=self.normal_user.person_id) count_success, count_lines, error_messages, warning_messages = import_motions(csv_file=f, default_submitter=self.normal_user.person_id)
self.assertEqual(Motion.objects.count(), 4) self.assertEqual(Motion.objects.count(), 4)
self.assertEqual(count_success, 4) self.assertEqual(count_success, 4)
@ -68,8 +68,8 @@ class CSVImport(TestCase):
self.assertEqual(len(motion1.submitter.all()), 1) self.assertEqual(len(motion1.submitter.all()), 1)
self.assertEqual(motion1.submitter.all()[0].person, self.normal_user) self.assertEqual(motion1.submitter.all()[0].person, self.normal_user)
self.assertTrue(motion1.category is None) self.assertTrue(motion1.category is None)
self.assertTrue('Line 2 in import file: Default submitter is used.' in warning_messages) self.assertTrue(any('Submitter unknown.' in w for w in warning_messages))
self.assertTrue('Line 2 in import file: Category not found.' in error_messages) self.assertTrue(any('Category unknown.' in w for w in warning_messages))
motion2 = Motion.objects.get(pk=2) motion2 = Motion.objects.get(pk=2)
self.assertEqual(motion2.identifier, 'S 2') self.assertEqual(motion2.identifier, 'S 2')
@ -81,15 +81,15 @@ class CSVImport(TestCase):
self.assertEqual(motion2.submitter.all()[0].person, special_user) self.assertEqual(motion2.submitter.all()[0].person, special_user)
self.assertEqual(motion2.category, self.category1) self.assertEqual(motion2.category, self.category1)
self.assertTrue('Line 5 in import file: Multiple persons found.' in error_messages) self.assertTrue(any('Several suitable submitters found.' in w for w in warning_messages))
self.assertTrue('Line 5 in import file: Multiple categories found.' in error_messages) self.assertTrue(any('Several suitable categories found.' in w for w in warning_messages))
def test_malformed_file(self): def test_malformed_file(self):
csv_file = StringIO.StringIO() csv_file = StringIO.StringIO()
csv_file.write('Header\nMalformed data,\n,Title,Text,,,\n') csv_file.write('Header\nMalformed data,\n,Title,Text,,,\n')
count_success, error_messages, warning_messages = import_motions(csv_file=csv_file, default_submitter=self.normal_user.person_id) count_success, count_lines, error_messages, warning_messages = import_motions(csv_file=csv_file, default_submitter=self.normal_user.person_id)
self.assertEqual(count_success, 1) self.assertEqual(count_success, 0)
self.assertTrue('Ignoring malformed line 2 in import file.' in error_messages) self.assertTrue(any('Line is malformed.' in e for e in error_messages))
def test_wrong_encoding(self): def test_wrong_encoding(self):
csv_file = StringIO.StringIO() csv_file = StringIO.StringIO()
@ -98,4 +98,4 @@ class CSVImport(TestCase):
csv_file.seek(0) csv_file.seek(0)
count_success, error_messages, warning_messages = import_motions(csv_file=csv_file, default_submitter=self.normal_user.person_id) count_success, error_messages, warning_messages = import_motions(csv_file=csv_file, default_submitter=self.normal_user.person_id)
self.assertEqual(count_success, 0) self.assertEqual(count_success, 0)
self.assertTrue('Encoding error in import file. Ensure using UTF-8.' in error_messages) self.assertTrue('Import file has wrong character encoding, only UTF-8 is supported!' in error_messages)