From a97cf4a6805ae809b5a2252e216a15595a3c9a95 Mon Sep 17 00:00:00 2001 From: Emanuel Schuetze Date: Wed, 29 May 2013 22:29:08 +0200 Subject: [PATCH] Motion import improved: - add semicolon support (use utils/csv_ext.py as participant import) - show all warnings of each line in one warning message - show success message with total counter (x of n motions imported). - improved warning and error strings --- openslides/locale/de/LC_MESSAGES/django.po | 2 +- openslides/motion/csv_import.py | 39 +++++++++++++++++----- openslides/motion/views.py | 4 +-- tests/motion/test_csv_import.py | 18 +++++----- 4 files changed, 42 insertions(+), 21 deletions(-) diff --git a/openslides/locale/de/LC_MESSAGES/django.po b/openslides/locale/de/LC_MESSAGES/django.po index 0e38a1bbb..665cbc08d 100644 --- a/openslides/locale/de/LC_MESSAGES/django.po +++ b/openslides/locale/de/LC_MESSAGES/django.po @@ -1970,7 +1970,7 @@ msgstr "Import auf Grund von schweren Fehlern in der Quelldatei abgebrochen." msgid "Import file has wrong character encoding, only UTF-8 is supported!" msgstr "" "Die Quelldatei benutzt eine ungültige Zeichenkodierung, es wird nur UTF-8 " -"wird unterstützt!" +"unterstützt!" #: participant/forms.py:29 participant/views.py:542 #: participant/templates/participant/group_overview.html:7 diff --git a/openslides/motion/csv_import.py b/openslides/motion/csv_import.py index b6a60ab1b..7c6ab4e71 100644 --- a/openslides/motion/csv_import.py +++ b/openslides/motion/csv_import.py @@ -19,7 +19,9 @@ import csv from django.db import transaction from django.utils.translation import ugettext as _, ugettext_noop +from openslides.utils import csv_ext from openslides.utils.person.api import Persons +from openslides.utils.utils import html_strong from .models import Motion, Category @@ -37,25 +39,33 @@ def import_motions(csv_file, default_submitter, override=False, importing_person error_messages = [] warning_messages = [] count_success = 0 + count_lines = 0 # Check encoding try: csv_file.read().decode('utf8') except UnicodeDecodeError: - return (0, [_('Encoding error in import file. Ensure using UTF-8.')], []) + return (0, [_('Import file has wrong character encoding, only UTF-8 is supported!')], []) csv_file.seek(0) with transaction.commit_on_success(): - for (line_no, line) in enumerate(csv.reader(csv_file)): + dialect = csv.Sniffer().sniff(csv_file.readline()) + dialect = csv_ext.patchup(dialect) + csv_file.seek(0) + for (line_no, line) in enumerate(csv.reader(csv_file, dialect=dialect)): + warnings = [] if line_no < 1: # Do not read the header line continue + count_lines += 1 # Check format try: (identifier, title, text, reason, submitter, category) = line[:6] except ValueError: - error_messages.append(_('Ignoring malformed line %d in import file.') % (line_no + 1)) + error_line = html_strong(_('Line %d of import file:') % (line_no + 1)) + msg = _('Line is malformed. Motion not imported. Please check the required values.') + error_messages.append("%s
%s" % (error_line, msg)) continue # Check existing motions according to the identifier @@ -66,7 +76,9 @@ def import_motions(csv_file, default_submitter, override=False, importing_person motion = Motion(identifier=identifier) else: if not override: - error_messages.append(_('Line %d in import file: Ignoring existing motion.') % (line_no + 1)) + error_line = html_strong(_('Line %d of import file:') % (line_no + 1)) + msg = _('Identifier already exists. Motion not imported.') + error_messages.append("%s
%s" % (error_line, msg)) continue else: motion = Motion() @@ -79,9 +91,9 @@ def import_motions(csv_file, default_submitter, override=False, importing_person try: motion.category = Category.objects.get(name=category) except Category.DoesNotExist: - error_messages.append(_('Line %d in import file: Category not found.') % (line_no + 1)) + warnings.append(_('Category unknown. No category is used.')) except Category.MultipleObjectsReturned: - error_messages.append(_('Line %d in import file: Multiple categories found.') % (line_no + 1)) + warnings.append(_('Several suitable categories found. No category is used.')) motion.save() # Add submitter @@ -90,15 +102,24 @@ def import_motions(csv_file, default_submitter, override=False, importing_person for person in Persons(): if person.clean_name == submitter.decode('utf8'): if person_found: - error_messages.append(_('Line %d in import file: Multiple persons found.') % (line_no + 1)) + warnings.append(_('Several suitable submitters found.')) person_found = False break else: new_submitter = person person_found = True if not person_found: - warning_messages.append(_('Line %d in import file: Default submitter is used.') % (line_no + 1)) + warnings.append(_('Submitter unknown. Default submitter is used.')) new_submitter = default_submitter + + # show summarized warning message for each import line + if warnings: + warning_line = _('Line %d of import file:') % (line_no + 1) + warning_message_string = "%s" + warning_messages.append(warning_message_string) motion.clear_submitters() motion.add_submitter(new_submitter) @@ -106,4 +127,4 @@ def import_motions(csv_file, default_submitter, override=False, importing_person person=importing_person) count_success += 1 - return (count_success, error_messages, warning_messages) + return (count_success, count_lines, error_messages, warning_messages) diff --git a/openslides/motion/views.py b/openslides/motion/views.py index 213f1db9e..eef3f0e2c 100644 --- a/openslides/motion/views.py +++ b/openslides/motion/views.py @@ -707,7 +707,7 @@ class MotionCSVImportView(FormView): """ Processes the import function. """ - count_success, error_messages, warning_messages = import_motions( + count_success, count_lines, error_messages, warning_messages = import_motions( self.request.FILES['csvfile'], default_submitter=form.cleaned_data['default_submitter'], override=form.cleaned_data['override'], @@ -719,7 +719,7 @@ class MotionCSVImportView(FormView): if count_success: messages.success( self.request, - _('%d motions were successfully imported.') % count_success) + "%s
%s" % (_('Summary'), _('%d of %d motions successfully imported.') % (count_success, count_lines))) return super(MotionCSVImportView, self).form_valid(form) motion_csv_import = MotionCSVImportView.as_view() diff --git a/tests/motion/test_csv_import.py b/tests/motion/test_csv_import.py index c7705ecd0..32e1e1103 100644 --- a/tests/motion/test_csv_import.py +++ b/tests/motion/test_csv_import.py @@ -56,7 +56,7 @@ class CSVImport(TestCase): csv_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'extras', 'csv-examples') self.assertEqual(Motion.objects.count(), 0) with open(csv_dir + '/motions-demo_de.csv') as f: - count_success, error_messages, warning_messages = import_motions(csv_file=f, default_submitter=self.normal_user.person_id) + count_success, count_lines, error_messages, warning_messages = import_motions(csv_file=f, default_submitter=self.normal_user.person_id) self.assertEqual(Motion.objects.count(), 4) self.assertEqual(count_success, 4) @@ -68,8 +68,8 @@ class CSVImport(TestCase): self.assertEqual(len(motion1.submitter.all()), 1) self.assertEqual(motion1.submitter.all()[0].person, self.normal_user) self.assertTrue(motion1.category is None) - self.assertTrue('Line 2 in import file: Default submitter is used.' in warning_messages) - self.assertTrue('Line 2 in import file: Category not found.' in error_messages) + self.assertTrue(any('Submitter unknown.' in w for w in warning_messages)) + self.assertTrue(any('Category unknown.' in w for w in warning_messages)) motion2 = Motion.objects.get(pk=2) self.assertEqual(motion2.identifier, 'S 2') @@ -81,15 +81,15 @@ class CSVImport(TestCase): self.assertEqual(motion2.submitter.all()[0].person, special_user) self.assertEqual(motion2.category, self.category1) - self.assertTrue('Line 5 in import file: Multiple persons found.' in error_messages) - self.assertTrue('Line 5 in import file: Multiple categories found.' in error_messages) + self.assertTrue(any('Several suitable submitters found.' in w for w in warning_messages)) + self.assertTrue(any('Several suitable categories found.' in w for w in warning_messages)) def test_malformed_file(self): csv_file = StringIO.StringIO() csv_file.write('Header\nMalformed data,\n,Title,Text,,,\n') - count_success, error_messages, warning_messages = import_motions(csv_file=csv_file, default_submitter=self.normal_user.person_id) - self.assertEqual(count_success, 1) - self.assertTrue('Ignoring malformed line 2 in import file.' in error_messages) + count_success, count_lines, error_messages, warning_messages = import_motions(csv_file=csv_file, default_submitter=self.normal_user.person_id) + self.assertEqual(count_success, 0) + self.assertTrue(any('Line is malformed.' in e for e in error_messages)) def test_wrong_encoding(self): csv_file = StringIO.StringIO() @@ -98,4 +98,4 @@ class CSVImport(TestCase): csv_file.seek(0) count_success, error_messages, warning_messages = import_motions(csv_file=csv_file, default_submitter=self.normal_user.person_id) self.assertEqual(count_success, 0) - self.assertTrue('Encoding error in import file. Ensure using UTF-8.' in error_messages) + self.assertTrue('Import file has wrong character encoding, only UTF-8 is supported!' in error_messages)