Merge pull request 'Update dependency flask-migrate to v4' (!111) from renovate/flask-migrate-4.x into main

Reviewed-on: #111

.dockerignore

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: WTF Kooperative eG <https://wtf-eg.de/>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+*
+!Pipfile
+!Pipfile.lock
+!data/
+!ki/
+!LICENSES/
+!migrations/
+!app.py
+!run_prod.py

.drone.yml

@@ -15,15 +15,28 @@ trigger:
 
 steps:
   - name: install-lint-test
-    image: git.wtf-eg.de/kompetenzinventar/builder:1.0.2
+    image: python:3.8.20-alpine@sha256:3d93b1f77efce339aa77db726656872517b0d67837989aa7c4b35bd5ae7e81ba
+    env:
+      PYROOT: '/pyroot'
+      PYTHONUSERBASE: '/pyroot'
     commands:
+      - apk add --no-cache gcc g++ musl-dev python3-dev
+      - pip3 install pipenv
+      - pipenv verify
       - pipenv install --dev
       - pipenv run flake8
       - pipenv run reuse lint
       - pipenv run python -m unittest discover ki
-
-image_pull_secrets:
-  - dockerconfig
+  - name: docker-dry-run
+    image: plugins/docker:20.18.4@sha256:a8d3d86853c721492213264815f1d00d3ed13f42f5c1855a02f47fa4d5f1e042
+    settings:
+      registry: git.wtf-eg.de
+      repo: git.wtf-eg.de/kompetenzinventar/backend
+      target: ki-backend
+      dry_run: true
+    when:
+      event:
+        - pull_request
 
 ---
 kind: pipeline
@@ -41,7 +54,7 @@ depends_on:
 
 steps:
   - name: docker-publish
-    image: plugins/docker
+    image: plugins/docker:20.18.4@sha256:a8d3d86853c721492213264815f1d00d3ed13f42f5c1855a02f47fa4d5f1e042
     settings:
       registry: git.wtf-eg.de
       repo: git.wtf-eg.de/kompetenzinventar/backend
@@ -68,7 +81,7 @@ depends_on:
 
 steps:
   - name: deploy-dev
-    image: appleboy/drone-ssh
+    image: appleboy/drone-ssh:1.7.5@sha256:995677e073454912f26d4c0fdd2f9df2e1f5a30d6603d3f2ece667311b6babb3
     settings:
       host:
         - dev01.wtf-eg.net
@@ -91,14 +104,19 @@ trigger:
 
 steps:
   - name: install-lint-test
-    image: git.wtf-eg.de/kompetenzinventar/builder:1.0.2
+    image: python:3.8.20-alpine@sha256:3d93b1f77efce339aa77db726656872517b0d67837989aa7c4b35bd5ae7e81ba
+    env:
+      PYROOT: '/pyroot'
+      PYTHONUSERBASE: '/pyroot'
     commands:
+      - apk add --no-cache gcc g++ musl-dev python3-dev
+      - pip3 install pipenv
       - pipenv install --dev
       - pipenv run flake8
       - pipenv run reuse lint
       - pipenv run python -m unittest discover ki
   - name: docker-publish
-    image: plugins/docker
+    image: plugins/docker:20.18.4@sha256:a8d3d86853c721492213264815f1d00d3ed13f42f5c1855a02f47fa4d5f1e042
     settings:
       registry: git.wtf-eg.de
       repo: git.wtf-eg.de/kompetenzinventar/backend

.python-version

@@ -0,0 +1 @@
+3.8.20

.reuse/dep5

@@ -1,12 +0,0 @@
-Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: Kompetenzinventar
-Upstream-Contact: Michael Weimann <mail@michael-weimann.eu>
-Source: https://git.wtf-eg.de/kompetenzinventar/ki-backend
-
-Files: data/imgs/flags/*
-Copyright: 2013 Panayiotis Lipiridis <https://flagicons.lipis.dev/>
-License: MIT
-
-Files: Pipfile.lock migrations/*
-Copyright: WTF Kooperative eG <https://wtf-eg.de/>
-License: AGPL-3.0-or-later

.yapfignore

@@ -2,5 +2,4 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
-*
-!.gitignore
+migrations/*.py

Dockerfile

@@ -2,7 +2,17 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
-FROM git.wtf-eg.de/kompetenzinventar/builder:1.0.2 as builder
+FROM python:3.8.20-alpine@sha256:3d93b1f77efce339aa77db726656872517b0d67837989aa7c4b35bd5ae7e81ba AS builder
+
+ENV PYROOT=/pyroot
+ENV PYTHONUSERBASE=$PYROOT
+
+RUN apk add --no-cache \
+    gcc \
+    g++ \
+    musl-dev \
+    python3-dev && \
+    pip3 install pipenv
 
 COPY Pipfile* ./
 
@@ -10,7 +20,10 @@ RUN PIP_USER=1 PIP_IGNORE_INSTALLED=1 pipenv install --system --deploy --ignore-
 RUN pip3 uninstall --yes pipenv
 
 
-FROM git.wtf-eg.de/kompetenzinventar/base:1.0.2 as ki-backend
+FROM python:3.8.20-alpine@sha256:3d93b1f77efce339aa77db726656872517b0d67837989aa7c4b35bd5ae7e81ba AS ki-backend
+
+ENV PYROOT=/pyroot
+ENV PYTHONUSERBASE=$PYROOT
 
 # Install six explicitly. Otherwise Python complains about it missing.
 RUN pip3 install six

Pipfile

@@ -8,26 +8,26 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-flask = "~=3.0.0"
-python-dotenv = "~=0.17.1"
-flask-migrate = "~=4.0.5"
-flask-sqlalchemy  = "~=3.1.1"
-sqlalchemy = "~=2.0.25"
-waitress = "~=2.1.2"
-pyyaml = "~=6.0.1"
-flask-cors = "~=4.0.0"
-ldap3 = "~=2.9.1"
-pymysql = "~=1.1.0"
-werkzeug = "~=3.0.1"
+flask = "==2.3.3"
+python-dotenv = "==1.0.1"
+flask-migrate = "==4.0.7"
+flask-sqlalchemy  = "==2.5.1"
+sqlalchemy = "==1.4.54"
+waitress = "==2.1.2"
+pyyaml = "==6.0.2"
+flask-cors = "==5.0.0"
+ldap3 = "==2.9.1"
+pymysql = "==1.1.1"
+werkzeug = "==2.3.8"
 
 [dev-packages]
-flake8 = "~=3.9.2"
-yapf = "~=0.40.2"
-pre-commit = "~=2.13.0"
-reuse = "~=0.13.0"
+flake8 = "==7.1.1"
+yapf = "==0.40.2"
+pre-commit = "==2.21.0"
+reuse = "==4.0.3"
 
 [requires]
-python_version = "3.11"
+python_version = "3.8"
 
 [scripts]
-clean = "rm storage/ki.sqlite"
+clean = "rm data/ki.sqlite"

REUSE.toml

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: NONE
+# SPDX-License-Identifier: CC0-1.0
+
+version = 1
+SPDX-PackageName = "Kompetenzinventar Backend"
+SPDX-PackageDownloadLocation = "https://git.wtf-eg.de/kompetenzinventar/ki-backend"
+
+[[annotations]]
+path = "data/imgs/flags/**"
+precedence = "aggregate"
+SPDX-FileCopyrightText = "2013 Panayiotis Lipiridis <https://flagicons.lipis.dev/>"
+SPDX-License-Identifier = "MIT"
+
+[[annotations]]
+path = ["Pipfile.lock", "migrations/**"]
+precedence = "aggregate"
+SPDX-FileCopyrightText = "WTF Kooperative eG <https://wtf-eg.de/>"
+SPDX-License-Identifier = "AGPL-3.0-or-later"
+
+[[annotations]]
+path = ["renovate.json", ".python-version"]
+precedence = "aggregate"
+SPDX-FileCopyrightText = "WTF Kooperative eG <https://wtf-eg.de/>"
+SPDX-License-Identifier = "AGPL-3.0-or-later"

app.py

@@ -38,7 +38,7 @@ app.config["KI_LDAP_AUTH_PASSWORD"] = os.getenv("KI_LDAP_AUTH_PASSWORD")
 app.config["KI_LDAP_BASE_DN"] = os.getenv("KI_LDAP_BASE_DN")
 
 CORS(app)
-db = SQLAlchemy(app, session_options={"future": True})
+db = SQLAlchemy(app)
 migrate = Migrate(app, db, compare_type=True)
 
 app.logger.info("Hello from KI")

env.dev

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
-SQLALCHEMY_DATABASE_URI=sqlite:///../storage/ki.sqlite
+SQLALCHEMY_DATABASE_URI=sqlite:///storage/ki.sqlite
 
 CORS_ORIGINS=*
 

ki/actions/seed.py

@@ -1,4 +1,5 @@
 # SPDX-FileCopyrightText: WTF Kooperative eG <https://wtf-eg.de/>
+#
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 import csv
@@ -18,7 +19,7 @@ def seed_contacttypes():
 
         for contacttype in csv_reader:
             id = int(contacttype["id"])
-            db_contacttype = db.session.get(ContactType, id)
+            db_contacttype = ContactType.query.get(id)
 
             if db_contacttype is None:
                 db.session.add(ContactType(id=int(contacttype["id"]), name=contacttype["name"]))
@@ -98,7 +99,7 @@ def seed(dev: bool):
 
         for skill in skills_csv_reader:
             id = int(skill["id"])
-            db_skill = db.session.get(Skill, id)
+            db_skill = Skill.query.get(id)
 
             if db_skill is None:
                 db.session.add(Skill(id=int(skill["id"]), name=skill["name"]))
@@ -112,7 +113,7 @@ def seed(dev: bool):
 
         for iso in iso_csv_reader:
             id = iso["639-1"]
-            db_language = db.session.get(Language, id)
+            db_language = Language.query.get(id)
 
             if db_language is None:
                 db.session.add(Language(id=iso["639-1"], name=iso["Sprache"]))
@@ -146,13 +147,12 @@ def seed(dev: bool):
                   freetext="1001010010111!!!",
                   skills=[(1, 5)],
                   address=("Friedrich Witzig", "", "", "", "", "", ""))
-        # all_skills = [(skill.id, 3) for skill in Skill.query.all()] # query causes problems
-        # seed_user("jutta", languages=[("fr", 5)], skills=all_skills)
+
+        all_skills = [(skill.id, 3) for skill in Skill.query.all()]
+        seed_user("jutta", languages=[("fr", 5)], skills=all_skills)
 
         seed_user("giesela", skills=[(9, 3), (10, 5)])
         seed_user("bertha", visible=False, skills=[(11, 3), (10, 5)])
         seed_user("monique", languages=[("fr", 4)])
-    print("seeding done")
-    with app.app_context():
-        db.session.commit()  # also problematic
-    print("commit done")
+
+    db.session.commit()

ki/handlers/update_profile.py

@@ -33,7 +33,7 @@ def update_languages(profile, languages_data):
         if "id" not in language_data["language"]:
             continue
 
-        language = db.session.get(Language, language_data["language"]["id"])
+        language = Language.query.get(language_data["language"]["id"])
         profile_language = ProfileLanguage.query.filter(ProfileLanguage.profile == profile,
                                                         ProfileLanguage.language == language).first()
 
@@ -110,7 +110,7 @@ def update_contacts(profile, contacts_data):
         if "id" in contact_data:
             contact_id = int(contact_data["id"])
             contact_ids_to_be_deleted.remove(contact_id)
-            contact = db.session.get(Contact, contact_id)
+            contact = Contact.query.get(contact_id)
         else:
             contact = Contact(profile=profile, contacttype=contacttype)
             db.session.add(contact)
@@ -122,7 +122,7 @@ def update_contacts(profile, contacts_data):
 
 
 def update_profile(user_id: int):
-    user = db.session.get(User, user_id)
+    user = User.query.get(user_id)
 
     if user is None:
         return make_response({}, 404)

ki/routes.py

@@ -10,7 +10,7 @@ from ki.auth import auth
 from ki.handlers import find_profiles as find_profiles_handler
 from ki.handlers import update_profile as update_profile_handler
 from ki.models import ContactType, Language, Skill, Token, User
-from app import app, db
+from app import app
 
 content_type_svg = "image/svg+xml"
 content_type_png = "image/png"
@@ -66,7 +66,7 @@ def handle_completion_request(model, key):
 
 
 def handle_icon_request(model, id, path):
-    object = db.session.get(model, id)
+    object = model.query.get(id)
 
     if object is None:
         return make_response({}, 404)

ki/test/ApiTest.py

@@ -1,5 +1,6 @@
-#  SPDX-FileCopyrightText: WTF Kooperative eG <https://wtf-eg.de/>
-#  SPDX-License-Identifier: AGPL-3.0-or-later
+# SPDX-FileCopyrightText: WTF Kooperative eG <https://wtf-eg.de/>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
 
 from alembic import command
 import json
@@ -8,14 +9,12 @@ import unittest
 from app import app, db, migrate
 from ki.actions import seed
 from ki.models import Skill
-from sqlalchemy import select
 
 
 class ApiTest(unittest.TestCase):
     maxDiff = None
 
     def setUp(self):
-        print("Running setup")
         app.debug = True
         app.config["KI_AUTH"] = "file"
         app.config["TESTING"] = True
@@ -23,38 +22,19 @@ class ApiTest(unittest.TestCase):
 
         self.client = app.test_client()
 
-        config = migrate.get_config()
         with app.app_context():
+            config = migrate.get_config()
             command.upgrade(config, "head")
-            seed(True)
 
-            # statement = select(Skill).order_by(Skill.id.desc())
-            # print(statement)
-            # skill_obj = db.session.scalars(statement).all()
-            # print(skill_obj)
-            # statement = select(Skill.id)
-            # print(statement)
-            # max_skill = db.session.Skill().order_by(Skill.id.desc()).first()
-            # max_skill = Skill.query.order_by(Skill.id.desc()).first() # TODO: problematic
-            with db.session.no_autoflush:  # only works on first test run
-                max_skill = db.session.query(Skill).order_by(Skill.id.desc()).first()  # TODO: also problematic,
-            # skills = db.session.execute(db.select(Skill)).scalars()
-            # print(max_skill)
-            # max_skill = db.session.execute(db.select(Skill)
-            #                   .order_by(Skill.id.desc())
-            #                   ).scalar_one()
-            print(max_skill)
-            print("max_skill done")
+        seed(True)
+        max_skill = Skill.query.order_by(Skill.id.desc()).first()
         self.max_skill_id = max_skill.id
 
     def tearDown(self):
-        print("Running teardown")
-        with app.app_context():
-            db.drop_all()
-            db.engine.dispose()
+        db.drop_all()
+        db.engine.dispose()
 
     def login(self, username, password):
-        # with app.app_context():
         login_data = {"username": username, "password": password}
         login_response = self.client.post("/users/login", data=json.dumps(login_data), content_type="application/json")
 

ki/test/test_contacttypes_endpoint.py

@@ -1,4 +1,5 @@
 # SPDX-FileCopyrightText: WTF Kooperative eG <https://wtf-eg.de/>
+#
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 import unittest
@@ -7,39 +8,37 @@ from ki.test.ApiTest import ApiTest
 
 
 class TestContactTypesEndpoint(ApiTest):
+
     def test_skills_options(self):
-        print("test_skills_options")
-        # with app.app_context():
         response = self.client.options("/contacttypes")
         self.assertEqual(response.status_code, 200)
         self.assertIn("Access-Control-Allow-Origin", response.headers)
         self.assertEqual(response.headers["Access-Control-Allow-Origin"], "*")
 
-    # def test_get_contacttypes_unauthorised(self):
-    #     print("test_get_contacttypes_unauthorised")
-    #     response = self.client.get("/contacttypes?search=m")
-    #     self.assertEqual(response.status_code, 401)
+    def test_get_contacttypes_unauthorised(self):
+        response = self.client.get("/contacttypes?search=m")
+        self.assertEqual(response.status_code, 401)
 
-    # def test_get_contacttypes(self):
-    #     token = self.login("peter", "geheim")["token"]
+    def test_get_contacttypes(self):
+        token = self.login("peter", "geheim")["token"]
 
-    #     response = self.client.get("/contacttypes?search=m", headers={"Authorization": "Bearer " + token})
-    #     self.assertEqual(response.status_code, 200)
-    #     self.assertEqual(
-    #         {
-    #             "contacttypes": [{
-    #                 "id": 5,
-    #                 "name": "Mastodon"
-    #             }, {
-    #                 "id": 4,
-    #                 "name": "Matrix"
-    #             }, {
-    #                 "id": 2,
-    #                 "name": "Mobiltelefon"
-    #             }]
-    #         }, response.json)
-    #     self.assertIn("Access-Control-Allow-Origin", response.headers)
-    #     self.assertEqual(response.headers["Access-Control-Allow-Origin"], "*")
+        response = self.client.get("/contacttypes?search=m", headers={"Authorization": "Bearer " + token})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(
+            {
+                "contacttypes": [{
+                    "id": 5,
+                    "name": "Mastodon"
+                }, {
+                    "id": 4,
+                    "name": "Matrix"
+                }, {
+                    "id": 2,
+                    "name": "Mobiltelefon"
+                }]
+            }, response.json)
+        self.assertIn("Access-Control-Allow-Origin", response.headers)
+        self.assertEqual(response.headers["Access-Control-Allow-Origin"], "*")
 
 
 if __name__ == "main":

ki/test/test_find_profiles_endpoint.py

@@ -8,6 +8,7 @@ from ki.test.ApiTest import ApiTest
 
 
 class TestFindProfilesEndpoint(ApiTest):
+
     def test_find_profiles_options(self):
         response = self.client.options("/users/profiles")
         self.assertEqual(response.status_code, 200)

ki/test/test_languages_endpoint.py

@@ -8,6 +8,7 @@ from ki.test.ApiTest import ApiTest
 
 
 class TestLanguagesEndpoint(ApiTest):
+
     def test_skills_options(self):
         response = self.client.options("/languages")
         self.assertEqual(response.status_code, 200)

ki/test/test_login_endpoint.py

@@ -10,6 +10,7 @@ from ki.test.ApiTest import ApiTest
 
 
 class TestLoginEndpoint(ApiTest):
+
     def test_login(self):
         response1_data = self.login("peter", "geheim")
         response2_data = self.login("peter", "geheim")

ki/test/test_skills_endpoint.py

@@ -8,6 +8,7 @@ from ki.test.ApiTest import ApiTest
 
 
 class TestSkillsEndpoint(ApiTest):
+
     def test_skills_options(self):
         response = self.client.options("/skills")
         self.assertEqual(response.status_code, 200)

migrations/env.py

@@ -19,7 +19,7 @@ logger = logging.getLogger('alembic.env')
 # target_metadata = mymodel.Base.metadata
 config.set_main_option(
     'sqlalchemy.url',
-    str(current_app.extensions['migrate'].db.engine.url).replace(
+    str(current_app.extensions['migrate'].db.get_engine().url).replace(
         '%', '%%'))
 target_metadata = current_app.extensions['migrate'].db.metadata
 
@@ -68,7 +68,7 @@ def run_migrations_online():
                 directives[:] = []
                 logger.info('No changes in schema detected.')
 
-    connectable = current_app.extensions['migrate'].db.engine
+    connectable = current_app.extensions['migrate'].db.get_engine()
 
     with connectable.connect() as connection:
         context.configure(

renovate.json

@@ -0,0 +1,18 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:best-practices",
+    ":disableDependencyDashboard",
+    ":maintainLockFilesMonthly",
+    ":pinVersions",
+    ":separateMultipleMajorReleases"
+  ],
+  "packageRules": [
+    {
+      "matchDepNames": ["python"],
+      "groupName": "Python",
+      "separateMinorPatch": true,
+      "separateMultipleMinor": true
+    }
+  ]
+}