From 04dd02b88a5c2c3a1284de98b5fc80168069a142 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Tue, 3 Nov 2020 20:52:55 +0100 Subject: [PATCH] Tests should not depend on Matrix-nio, that uses lxml that needs a lot of dependencies and a lot of time to build --- .gitlab-ci.yml | 4 +- Dockerfile | 2 +- .../commands/fix_matrix_channels.py | 31 ++++--- apps/participation/models.py | 3 +- apps/participation/tests.py | 2 - apps/registration/tests.py | 2 +- corres2math/matrix.py | 91 ++++++++++--------- corres2math/middlewares.py | 1 - corres2math/settings.py | 10 +- requirements.txt | 34 +++---- tox.ini | 15 ++- 11 files changed, 110 insertions(+), 85 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49bdcaf..863abd5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,7 @@ py38: stage: test image: python:3.8-alpine before_script: - - apk add --no-cache gcc libc-dev libffi-dev libmagic libxml2-dev libxslt-dev libxml2-dev libxslt-dev + - apk add --no-cache libmagic - pip install tox --no-cache-dir script: tox -e py38 @@ -14,7 +14,7 @@ py39: stage: test image: python:3.9-alpine before_script: - - apk add --no-cache gcc libc-dev libffi-dev libmagic libxml2-dev libxslt-dev libxml2-dev libxslt-dev + - apk add --no-cache gcc libmagic - pip install tox --no-cache-dir script: tox -e py39 diff --git a/Dockerfile b/Dockerfile index 9be9d7f..cce2279 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN apk add --no-cache bash RUN mkdir /code WORKDIR /code COPY requirements.txt /code/requirements.txt -RUN pip install -r requirements.txt psycopg2-binary sympasoap --no-cache-dir +RUN pip install -r requirements.txt --no-cache-dir COPY . /code/ diff --git a/apps/participation/management/commands/fix_matrix_channels.py b/apps/participation/management/commands/fix_matrix_channels.py index bff9181..764b41e 100644 --- a/apps/participation/management/commands/fix_matrix_channels.py +++ b/apps/participation/management/commands/fix_matrix_channels.py @@ -1,9 +1,8 @@ import os from asgiref.sync import async_to_sync -from corres2math.matrix import Matrix, RoomVisibility, UploadError +from corres2math.matrix import Matrix, RoomPreset, RoomVisibility from django.core.management import BaseCommand -from nio import RoomPreset from registration.models import AdminRegistration, Registration @@ -11,19 +10,23 @@ class Command(BaseCommand): def handle(self, *args, **options): Matrix.set_display_name("Bot des Correspondances") - if not os.path.isfile(".matrix_avatar"): # pragma: no cover - stat_file = os.stat("corres2math/static/logo.png") - with open("corres2math/static/logo.png", "rb") as f: - resp, _ = Matrix.upload(f, filename="logo.png", content_type="image/png", filesize=stat_file.st_size) - if isinstance(resp, UploadError): - raise Exception(resp) - avatar_uri = resp.content_uri - with open(".matrix_avatar", "w") as f: - f.write(avatar_uri) - Matrix.set_avatar(avatar_uri) + if not os.getenv("SYNAPSE_PASSWORD"): + avatar_uri = "plop" + else: # pragma: no cover + if not os.path.isfile(".matrix_avatar"): + stat_file = os.stat("corres2math/static/logo.png") + with open("corres2math/static/logo.png", "rb") as f: + resp, _ = Matrix.upload(f, filename="logo.png", content_type="image/png", + filesize=stat_file.st_size) + if not hasattr(resp, "content_uri"): + raise Exception(resp) + avatar_uri = resp.content_uri + with open(".matrix_avatar", "w") as f: + f.write(avatar_uri) + Matrix.set_avatar(avatar_uri) - with open(".matrix_avatar", "r") as f: - avatar_uri = f.read().rstrip(" \t\r\n") + with open(".matrix_avatar", "r") as f: + avatar_uri = f.read().rstrip(" \t\r\n") if not async_to_sync(Matrix.resolve_room_alias)("#faq:correspondances-maths.fr"): Matrix.create_room( diff --git a/apps/participation/models.py b/apps/participation/models.py index 746550f..8b6af3e 100644 --- a/apps/participation/models.py +++ b/apps/participation/models.py @@ -2,7 +2,7 @@ import os import re from corres2math.lists import get_sympa_client -from corres2math.matrix import Matrix +from corres2math.matrix import Matrix, RoomPreset, RoomVisibility from django.core.exceptions import ObjectDoesNotExist from django.core.validators import RegexValidator from django.db import models @@ -13,7 +13,6 @@ from django.utils import timezone from django.utils.crypto import get_random_string from django.utils.text import format_lazy from django.utils.translation import gettext_lazy as _ -from nio import RoomPreset, RoomVisibility class Team(models.Model): diff --git a/apps/participation/tests.py b/apps/participation/tests.py index c8b7d4a..4a6d119 100644 --- a/apps/participation/tests.py +++ b/apps/participation/tests.py @@ -1,4 +1,3 @@ -import os from datetime import timedelta from django.contrib.auth.models import User @@ -705,7 +704,6 @@ class TestStudentParticipation(TestCase): call_command('fix_matrix_channels') call_command('setup_third_phase') - os.remove(".matrix_avatar") class TestAdmin(TestCase): diff --git a/apps/registration/tests.py b/apps/registration/tests.py index 02170e9..2db3763 100644 --- a/apps/registration/tests.py +++ b/apps/registration/tests.py @@ -12,7 +12,7 @@ from django.utils.encoding import force_bytes from django.utils.http import urlsafe_base64_encode from .auth import CustomAuthUser -from .models import AdminRegistration, CoachRegistration, Registration, StudentRegistration +from .models import AdminRegistration, CoachRegistration, StudentRegistration class TestIndexPage(TestCase): diff --git a/corres2math/matrix.py b/corres2math/matrix.py index 00ae20c..d01e44f 100644 --- a/corres2math/matrix.py +++ b/corres2math/matrix.py @@ -1,12 +1,7 @@ +from enum import Enum import os -from typing import Any, Dict, Optional, Tuple, Union from asgiref.sync import async_to_sync -from nio import AsyncClient, DataProvider, ProfileSetAvatarError, ProfileSetAvatarResponse, \ - ProfileSetDisplayNameError, ProfileSetDisplayNameResponse, RoomCreateError, RoomCreateResponse, \ - RoomInviteError, RoomInviteResponse, RoomKickError, RoomKickResponse, RoomPreset, \ - RoomPutStateError, RoomPutStateResponse, RoomResolveAliasResponse, RoomVisibility, TransferMonitor, \ - UploadError, UploadResponse class Matrix: @@ -18,11 +13,11 @@ class Matrix: Tasks are normally asynchronous, but for compatibility we make them synchronous. """ - _token: str = None - _device_id: str = None + _token = None + _device_id = None @classmethod - async def _get_client(cls) -> Union[AsyncClient, "FakeMatrixClient"]: # pragma: no cover + async def _get_client(cls): # pragma: no cover """ Retrieve the bot account. If not logged, log in and store access token. @@ -30,6 +25,7 @@ class Matrix: if not os.getenv("SYNAPSE_PASSWORD"): return FakeMatrixClient() + from nio import AsyncClient client = AsyncClient("https://correspondances-maths.fr", "@corres2mathbot:correspondances-maths.fr") client.user_id = "@corres2mathbot:correspondances-maths.fr" @@ -53,7 +49,7 @@ class Matrix: @classmethod @async_to_sync - async def set_display_name(cls, name: str) -> Union[ProfileSetDisplayNameResponse, ProfileSetDisplayNameError]: + async def set_display_name(cls, name: str): """ Set the display name of the bot account. """ @@ -62,7 +58,7 @@ class Matrix: @classmethod @async_to_sync - async def set_avatar(cls, avatar_url: str) -> Union[ProfileSetAvatarResponse, ProfileSetAvatarError]: + async def set_avatar(cls, avatar_url: str): # pragma: no cover """ Set the display avatar of the bot account. """ @@ -73,13 +69,13 @@ class Matrix: @async_to_sync async def upload( cls, - data_provider: DataProvider, + data_provider, content_type: str = "application/octet-stream", - filename: Optional[str] = None, + filename: str = None, encrypt: bool = False, - monitor: Optional[TransferMonitor] = None, - filesize: Optional[int] = None, - ) -> Tuple[Union[UploadResponse, UploadError], Optional[Dict[str, Any]]]: + monitor=None, + filesize: int = None, + ): # pragma: no cover """ Upload a file to the content repository. @@ -134,24 +130,24 @@ class Matrix: """ client = await cls._get_client() return await client.upload(data_provider, content_type, filename, encrypt, monitor, filesize) \ - if isinstance(client, AsyncClient) else UploadResponse("debug mode"), None + if not isinstance(client, FakeMatrixClient) else None, None @classmethod @async_to_sync async def create_room( cls, - visibility: RoomVisibility = RoomVisibility.private, - alias: Optional[str] = None, - name: Optional[str] = None, - topic: Optional[str] = None, - room_version: Optional[str] = None, - federate: bool = True, - is_direct: bool = False, - preset: Optional[RoomPreset] = None, + visibility=None, + alias=None, + name=None, + topic=None, + room_version=None, + federate=True, + is_direct=False, + preset=None, invite=(), initial_state=(), - power_level_override: Optional[Dict[str, Any]] = None, - ) -> Union[RoomCreateResponse, RoomCreateError]: + power_level_override=None, + ): """ Create a new room. @@ -213,18 +209,18 @@ class Matrix: power_level_override) @classmethod - async def resolve_room_alias(cls, room_alias: str) -> Optional[str]: + async def resolve_room_alias(cls, room_alias: str): """ Resolve a room alias to a room ID. Return None if the alias does not exist. """ client = await cls._get_client() - resp: RoomResolveAliasResponse = await client.room_resolve_alias(room_alias) - return resp.room_id if isinstance(resp, RoomResolveAliasResponse) else None + resp = await client.room_resolve_alias(room_alias) + return resp.room_id if resp else None @classmethod @async_to_sync - async def invite(cls, room_id: str, user_id: str) -> Union[RoomInviteResponse, RoomInviteError]: + async def invite(cls, room_id: str, user_id: str): """ Invite a user to a room. @@ -266,13 +262,13 @@ class Matrix: @classmethod @async_to_sync - async def kick(cls, room_id: str, user_id: str, reason: str = None) -> Union[RoomKickResponse, RoomKickError]: + async def kick(cls, room_id: str, user_id: str, reason: str = None): """ Kick a user from a room, or withdraw their invitation. Kicking a user adjusts their membership to "leave" with an optional reason. - +² Returns either a `RoomKickResponse` if the request was successful or a `RoomKickError` if there was an error with the request. @@ -289,8 +285,7 @@ class Matrix: @classmethod @async_to_sync - async def set_room_power_level(cls, room_id: str, user_id: str, power_level: int)\ - -> Union[RoomPutStateResponse, RoomPutStateError]: # pragma: no cover + async def set_room_power_level(cls, room_id: str, user_id: str, power_level: int): # pragma: no cover """ Put a given power level to a user in a certain room. @@ -306,7 +301,7 @@ class Matrix: """ client = await cls._get_client() if isinstance(client, FakeMatrixClient): - return RoomPutStateError("debug mode") + return None if room_id.startswith("#"): room_id = await cls.resolve_room_alias(room_id) @@ -317,8 +312,7 @@ class Matrix: @classmethod @async_to_sync - async def set_room_power_level_event(cls, room_id: str, event: str, power_level: int)\ - -> Union[RoomPutStateResponse, RoomPutStateError]: # pragma: no cover + async def set_room_power_level_event(cls, room_id: str, event: str, power_level: int): # pragma: no cover """ Define the minimal power level to have to send a certain event type in a given room. @@ -335,7 +329,7 @@ class Matrix: """ client = await cls._get_client() if isinstance(client, FakeMatrixClient): - return RoomPutStateError("debug mode") + return None if room_id.startswith("#"): room_id = await cls.resolve_room_alias(room_id) @@ -349,8 +343,7 @@ class Matrix: @classmethod @async_to_sync - async def set_room_avatar(cls, room_id: str, avatar_uri: str)\ - -> Union[RoomPutStateResponse, RoomPutStateError]: + async def set_room_avatar(cls, room_id: str, avatar_uri: str): """ Define the avatar of a room. @@ -370,6 +363,22 @@ class Matrix: }, state_key="") +if os.getenv("SYNAPSE_PASSWORD"): # pragma: no cover + from nio import RoomVisibility, RoomPreset + RoomVisibility = RoomVisibility + RoomPreset = RoomPreset +else: + # When running tests, faking matrix-nio classes to don't include the module + class RoomVisibility(Enum): + private = 'private' + public = 'public' + + class RoomPreset(Enum): + private_chat = "private_chat" + trusted_private_chat = "trusted_private_chat" + public_chat = "public_chat" + + class FakeMatrixClient: """ Simulate a Matrix client to run tests, if no Matrix homeserver is connected. diff --git a/corres2math/middlewares.py b/corres2math/middlewares.py index a8ccef6..40cc04d 100644 --- a/corres2math/middlewares.py +++ b/corres2math/middlewares.py @@ -2,7 +2,6 @@ from threading import local from django.conf import settings from django.contrib.auth.models import AnonymousUser, User -from django.contrib.sessions.backends.db import SessionStore USER_ATTR_NAME = getattr(settings, 'LOCAL_USER_ATTR_NAME', '_current_user') SESSION_ATTR_NAME = getattr(settings, 'LOCAL_SESSION_ATTR_NAME', '_current_session') diff --git a/corres2math/settings.py b/corres2math/settings.py index bae7440..25d1362 100644 --- a/corres2math/settings.py +++ b/corres2math/settings.py @@ -51,16 +51,14 @@ INSTALLED_APPS = [ 'django.forms', 'bootstrap_datepicker_plus', + 'cas_server', 'crispy_forms', - 'django_extensions', 'django_tables2', 'haystack', 'logs', - 'mailer', 'polymorphic', 'rest_framework', 'rest_framework.authtoken', - 'cas_server', 'api', 'eastereggs', @@ -68,6 +66,12 @@ INSTALLED_APPS = [ 'participation', ] +if "test" not in sys.argv: # pragma: no cover + INSTALLED_APPS += [ + 'django_extensions', + 'mailer', + ] + MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', diff --git a/requirements.txt b/requirements.txt index fe4e57a..49db7a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,19 @@ -Django~=3.0 -django-bootstrap-datepicker-plus -django-cas-server -django-crispy-forms -django-extensions -django-filter~=2.3.0 +Django~=3.1 +django-bootstrap-datepicker-plus~=3.0 +django-cas-server~=1.2 +django-crispy-forms~=1.9 +django-extensions~=3.0 +django-filter~=2.3 django-haystack~=3.0 -django-mailer -django-polymorphic -django-tables2 -djangorestframework~=3.11.1 -django-rest-polymorphic -matrix-nio -ptpython -python-magic~=0.4.18 -gunicorn -whoosh \ No newline at end of file +django-mailer~=2.0 +django-polymorphic~=3.0 +django-tables2~=2.3 +djangorestframework~=3.12 +django-rest-polymorphic~=0.1 +gunicorn~=20.0 +matrix-nio~=0.15 +psycopg2-binary~=2.8 +ptpython~=3.0 +python-magic~=0.4 +sympasoap~=1.0 +whoosh~=2.7 \ No newline at end of file diff --git a/tox.ini b/tox.ini index 021529b..9a86cd3 100644 --- a/tox.ini +++ b/tox.ini @@ -7,10 +7,21 @@ envlist = skipsdist = True [testenv] -sitepackages = True +sitepackages = False deps = - -r{toxinidir}/requirements.txt coverage + Django~=3.1 + django-bootstrap-datepicker-plus~=3.0 + django-cas-server~=1.2 + django-crispy-forms~=1.9 + django-filter~=2.3 + django-haystack~=3.0 + django-polymorphic~=3.0 + django-tables2~=2.3 + djangorestframework~=3.12 + django-rest-polymorphic~=0.1 + python-magic~=0.4 + whoosh~=2.7 commands = coverage run --source=apps,corres2math ./manage.py test apps/ corres2math/ coverage report -m