Merge pull request #3290 from FinnStutzenstein/Issue1533-neu

Massimport for users

CHANGELOG

@@ -37,6 +37,7 @@ Users:
 - Added support for password validation using Django or custom validators
   e. g. for minimum password length [#3200].
 - Fixed compare of duplicated users while csv user import [#3201].
+- Added fast mass import for users [#3290].
 
 Core:
 - No reload on logoff. OpenSlides is now a full single page

openslides/users/serializers.py

@@ -72,16 +72,22 @@ class UserFullSerializer(ModelSerializer):
                 data.get('last_name', ''))
         return data
 
-    def create(self, validated_data):
+    def prepare_password(self, validated_data):
         """
-        Creates the user. Sets the default password.
+        Sets the default password.
         """
         # Prepare setup password.
         if not validated_data.get('default_password'):
             validated_data['default_password'] = User.objects.generate_password()
         validated_data['password'] = make_password(validated_data['default_password'], '', 'md5')
+        return validated_data
+
+    def create(self, validated_data):
+        """
+        Creates the user.
+        """
         # Perform creation in the database and return new user.
-        user = super().create(validated_data)
+        user = super().create(self.prepare_password(validated_data))
         # TODO: This autoupdate call is redundant (required by issue #2727). See #2736.
         inform_changed_data(user)
         return user

openslides/users/static/js/users/site.js

@@ -755,13 +755,12 @@ angular.module('OpenSlidesApp.users.site', [
 .controller('UserUpdateCtrl', [
     '$scope',
     '$state',
-    '$http',
     'User',
     'UserForm',
     'Group',
     'userId',
     'ErrorMessage',
-    function($scope, $state, $http, User, UserForm, Group, userId, ErrorMessage) {
+    function($scope, $state, User, UserForm, Group, userId, ErrorMessage) {
         Group.bindAll({where: {id: {'>': 2}}}, $scope, 'groups');
         $scope.alert = {};
         // set initial values for form model by create deep copy of user object
@@ -898,34 +897,45 @@ angular.module('OpenSlidesApp.users.site', [
 
 .controller('UserImportCtrl', [
     '$scope',
+    '$http',
     '$q',
     'gettext',
     'gettextCatalog',
     'User',
     'Group',
     'UserCsvExport',
-    function($scope, $q, gettext, gettextCatalog, User, Group, UserCsvExport) {
+    'ErrorMessage',
+    function($scope, $http, $q, gettext, gettextCatalog, User, Group, UserCsvExport, ErrorMessage) {
         // import from textarea
         $scope.importByLine = function () {
-            $scope.usernames = $scope.userlist[0].split("\n");
+            var usernames = $scope.userlist[0].split("\n");
-            $scope.importcounter = 0;
+            // Ignore empty lines.
-            $scope.usernames.forEach(function(name) {
+            /*usernames = _.filter(usernames, function (name) {
+                return name !== '';
+            });*/
+            var users = _.map(usernames, function (name) {
                 // Split each full name in first and last name.
                 // The last word is set as last name, rest is the first name(s).
                 // (e.g.: "Max Martin Mustermann" -> last_name = "Mustermann")
                 var names = name.split(" ");
                 var last_name = names.slice(-1)[0];
                 var first_name = names.slice(0, -1).join(" ");
-                var user = {
+                return {
                     first_name: first_name,
                     last_name: last_name,
-                    groups_id: []
+                    groups_id: [],
                 };
-                User.create(user).then(
+            });
-                    function(success) {
+            $http.post('/rest/users/user/mass_import/', {
-                        $scope.importcounter++;
+                users: users
-                    }
+            }).then(function (success) {
-                );
+                $scope.alert = {
+                    show: true,
+                    type: 'success',
+                    msg: success.data.detail,
+                };
+            }, function (error) {
+                $scope.alert = ErrorMessage.forAlert(error);
             });
         };
 
@@ -955,6 +965,7 @@ angular.module('OpenSlidesApp.users.site', [
         'groups', 'comment', 'is_active', 'is_present', 'is_committee', 'default_password'];
         $scope.users = [];
         $scope.onCsvChange = function (csv) {
+            $scope.csvImporting = false;
             // All user objects are already loaded via the resolve statement from ui-router.
             var users = User.getAll();
             $scope.users = [];
@@ -967,7 +978,8 @@ angular.module('OpenSlidesApp.users.site', [
                 }
             });
             $scope.duplicates = 0;
-            _.forEach(csvUsers, function (user) {
+            _.forEach(csvUsers, function (user, index) {
+                user.importTrackId = index;
                 user.selected = true;
                 if (!user.first_name && !user.last_name) {
                     user.importerror = true;
@@ -1110,6 +1122,11 @@ angular.module('OpenSlidesApp.users.site', [
                 var allGroups = Group.getAll();
                 var existingUsers = User.getAll();
 
+                // For option 'delete existing user' on duplicates
+                var deletePromises = [];
+                // Array of users for mass import
+                var usersToBeImported = [];
+
                 _.forEach($scope.users, function (user) {
                     if (user.selected && !user.importerror) {
                         // Assign all groups
@@ -1126,7 +1143,6 @@ angular.module('OpenSlidesApp.users.site', [
                         // Do nothing on duplicateAction==duplicateActions[0] (keep original)
                         if (user.duplicate && (user.duplicateAction == $scope.duplicateActions[1])) {
                             // delete existing user
-                            var deletePromises = [];
                             existingUsers.forEach(function(user_) {
                                 user_.fullname = [
                                     user_.title,
@@ -1142,30 +1158,45 @@ angular.module('OpenSlidesApp.users.site', [
                                     deletePromises.push(User.destroy(user_.id));
                                 }
                             });
-                            $q.all(deletePromises).then(function() {
+                            usersToBeImported.push(user);
-                                User.create(user).then(
-                                    function(success) {
-                                        user.imported = true;
-                                    }
-                                );
-                            });
                         } else if (!user.duplicate ||
                                    (user.duplicateAction == $scope.duplicateActions[2])) {
                             // create user
-                            User.create(user).then(
+                            usersToBeImported.push(user);
-                                function(success) {
-                                    user.imported = true;
-                                }
-                            );
                         }
                     }
                 });
+                $q.all(deletePromises).then(function () {
+                    $http.post('/rest/users/user/mass_import/', {
+                        users: usersToBeImported
+                    }).then(function (success) {
+                        _.forEach(success.data.importedTrackIds, function (trackId) {
+                            _.find($scope.users, function (user) {
+                                return user.importTrackId === trackId;
+                            }).imported = true;
+                        });
                         $scope.csvimported = true;
+                    }, function (error) {
+                        $scope.alert = ErrorMessage.forAlert(error);
+                    });
+                });
             });
         };
         $scope.clear = function () {
             $scope.users = null;
         };
+        $scope.excludeImportedUsers = function () {
+            $scope.users = _.filter($scope.users, function (user) {
+                return !user.imported;
+            });
+            $scope.csvImporting = false;
+            $scope.calcStats();
+        };
+        $scope.someImportedUsers = function () {
+            return _.some($scope.users, function (user) {
+                return user.imported;
+            });
+        };
         // download CSV example file
         $scope.downloadCSVExample = function () {
             UserCsvExport.downloadExample();

openslides/users/static/templates/users/user-import.html

@@ -11,6 +11,10 @@
 </div>
 
 <div class="details">
+  <div uib-alert ng-show="alert.show" ng-class="'alert-' + (alert.type || 'warning')" close="alert={}">
+    {{ alert.msg }}
+  </div>
+
   <h2 translate>Import by copy/paste</h2>
   <p translate>Copy and paste your participant names in this textbox.
   Keep each person in a single line.</p>
@@ -23,11 +27,6 @@
 
   <div class="clearfix">
     <button ng-click="importByLine()" class="btn btn-primary btn-sm pull-left" translate>Import</button>
-    <div class="col-xs-5" ng-if="usernames">
-      <progressbar animate="false" type="success" max="usernames.length" value="importcounter">
-        <i>{{ importcounter }} / {{ usernames.length }} {{ "imported" | translate }}</i>
-      </progressbar>
-    </div>
   </div>
   <div class="spacer">
     <a ng-if="importcounter > 0 && importcounter == usernames.length" ui-sref="users.user.list"
@@ -204,6 +203,9 @@
         <i class="fa fa-check-circle-o fa-lg"></i>
         {{ usersWillBeImported }}
         <translate>participants will be imported.</translate>
+        <span ng-if="csvImporting && !csvimported">
+          <i class="fa fa-spinner fa-pulse fa-lg"></i>
+        </span>
       </div>
       <div ng-repeat="user in usersImported = (users | filter:{imported:true})"></div>
       <div ng-if="usersImported.length > 0" class="text-success">
@@ -219,6 +221,9 @@
       <button ng-click="clear()" class="btn btn-default btn-sm" translate>
         Clear preview
       </button>
+      <button ng-if="someImportedUsers()" ng-click="excludeImportedUsers()" class="btn btn-default btn-sm" translate>
+        Exclude already imported users
+      </button>
       <button ng-if="!csvImporting && usersWillBeImported > 0" ng-click="import()" class="btn btn-primary btn-sm" translate>
         Import {{ usersWillBeImported }} participants
       </button>

openslides/users/views.py

@@ -4,13 +4,17 @@ from django.contrib.auth import update_session_auth_hash
 from django.contrib.auth.forms import AuthenticationForm
 from django.contrib.auth.password_validation import validate_password
 from django.core.exceptions import ValidationError as DjangoValidationError
+from django.db import transaction
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext as _
 
 from ..core.config import config
 from ..core.signals import permission_change
 from ..utils.auth import anonymous_is_enabled, has_perm
-from ..utils.autoupdate import inform_data_collection_element_list
+from ..utils.autoupdate import (
+    inform_changed_data,
+    inform_data_collection_element_list,
+)
 from ..utils.collection import CollectionElement, CollectionElementList
 from ..utils.rest_api import (
     ModelViewSet,
@@ -18,6 +22,7 @@ from ..utils.rest_api import (
     SimpleMetadata,
     ValidationError,
     detail_route,
+    list_route,
     status,
 )
 from ..utils.views import APIView
@@ -48,7 +53,7 @@ class UserViewSet(ModelViewSet):
             result = has_perm(self.request.user, 'users.can_see_name')
         elif self.action in ('update', 'partial_update'):
             result = self.request.user.is_authenticated()
-        elif self.action in ('create', 'destroy', 'reset_password'):
+        elif self.action in ('create', 'destroy', 'reset_password', 'mass_import'):
             result = (has_perm(self.request.user, 'users.can_see_name') and
                       has_perm(self.request.user, 'users.can_see_extra_data') and
                       has_perm(self.request.user, 'users.can_manage'))
@@ -114,6 +119,46 @@ class UserViewSet(ModelViewSet):
         else:
             raise ValidationError({'detail': 'Password has to be a string.'})
 
+    @list_route(methods=['post'])
+    @transaction.atomic
+    def mass_import(self, request):
+        """
+        API endpoint to create multiple users at once.
+
+        Example: {"users": [{"first_name": "Max"}, {"first_name": "Maxi"}]}
+        """
+        users = request.data.get('users')
+        if not isinstance(users, list):
+            raise ValidationError({'detail': 'Users has to be a list.'})
+
+        created_users = []
+        # List of all track ids of all imported users. The track ids are just used in the client.
+        imported_track_ids = []
+
+        for user in users:
+            serializer = self.get_serializer(data=user)
+            try:
+                serializer.is_valid(raise_exception=True)
+            except ValidationError:
+                # Skip invalid users.
+                continue
+            data = serializer.prepare_password(serializer.data)
+            groups = data['groups_id']
+            del data['groups_id']
+
+            db_user = User(**data)
+            db_user.save(skip_autoupdate=True)
+            db_user.groups.add(*groups)
+            created_users.append(db_user)
+            if 'importTrackId' in user:
+                imported_track_ids.append(user['importTrackId'])
+
+        # Now infom all clients and send a response
+        inform_changed_data(created_users)
+        return Response({
+            'detail': _('{number} users successfully imported.').format(number=len(created_users)),
+            'importedTrackIds': imported_track_ids})
+
 
 class GroupViewSetMetadata(SimpleMetadata):
     """

tests/integration/users/test_viewset.py

@@ -306,6 +306,33 @@ class UserResetPassword(TestCase):
         self.assertTrue(User.objects.get(pk=user.pk).check_password(default_password))
 
 
+class UserMassImport(TestCase):
+    """
+    Tests mass import of users.
+    """
+    def setUp(self):
+        self.client = APIClient()
+        self.client.login(username='admin', password='admin')
+
+    def test_mass_import(self):
+        user_1 = {
+            'first_name': 'first_name_kafaith3woh3thie7Ciy',
+            'last_name': 'last_name_phah0jaeph9ThoongaeL',
+            'groups_id': []
+        }
+        user_2 = {
+            'first_name': 'first_name_kohdao7Eibouwee8ma2O',
+            'last_name': 'last_name_kafaith3woh3thie7Ciy',
+            'groups_id': []
+        }
+        response = self.client.post(
+            reverse('user-mass-import'),
+            {'users': [user_1, user_2]},
+            format='json')
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(User.objects.count(), 3)
+
+
 class GroupMetadata(TestCase):
     def test_options_request_as_anonymous_user_activated(self):
         config['general_system_enable_anonymous'] = True