diff --git a/nixos/tests/matrix/mjolnir.nix b/nixos/tests/matrix/mjolnir.nix
index 8a888b17a3d7..265b13821ebd 100644
--- a/nixos/tests/matrix/mjolnir.nix
+++ b/nixos/tests/matrix/mjolnir.nix
@@ -109,8 +109,8 @@ import ../make-test-python.nix (
         environment.systemPackages = [
           (pkgs.writers.writePython3Bin "create_management_room_and_invite_mjolnir"
             { libraries = with pkgs.python3Packages; [
-                matrix-nio
-              ] ++ matrix-nio.optional-dependencies.e2e;
+                (matrix-nio.override { withOlm = true; })
+              ];
             } ''
             import asyncio
 
diff --git a/pkgs/applications/networking/instant-messengers/matrix-commander/default.nix b/pkgs/applications/networking/instant-messengers/matrix-commander/default.nix
index 6d39ab785cfa..856aa55f132d 100644
--- a/pkgs/applications/networking/instant-messengers/matrix-commander/default.nix
+++ b/pkgs/applications/networking/instant-messengers/matrix-commander/default.nix
@@ -41,7 +41,7 @@ buildPythonApplication rec {
   propagatedBuildInputs = [
     cacert
     setuptools
-    matrix-nio
+    (matrix-nio.override { withOlm = true; })
     python-magic
     markdown
     pillow
@@ -51,7 +51,7 @@ buildPythonApplication rec {
     pyxdg
     python-olm
     emoji
-  ] ++ matrix-nio.optional-dependencies.e2e;
+  ];
 
   meta = with lib; {
     description = "Simple but convenient CLI-based Matrix client app for sending and receiving";
diff --git a/pkgs/applications/networking/instant-messengers/pantalaimon/default.nix b/pkgs/applications/networking/instant-messengers/pantalaimon/default.nix
index f5118bb96863..2f22a6e23a47 100644
--- a/pkgs/applications/networking/instant-messengers/pantalaimon/default.nix
+++ b/pkgs/applications/networking/instant-messengers/pantalaimon/default.nix
@@ -39,11 +39,10 @@ python3Packages.buildPythonApplication rec {
     janus
     keyring
     logbook
-    matrix-nio
+    (matrix-nio.override { withOlm = true; })
     peewee
     prompt-toolkit
   ]
-  ++ matrix-nio.optional-dependencies.e2e
   ++ lib.optionals enableDbusUi optional-dependencies.ui;
 
   optional-dependencies.ui = with python3Packages; [
diff --git a/pkgs/applications/networking/irc/weechat/scripts/weechat-matrix/default.nix b/pkgs/applications/networking/irc/weechat/scripts/weechat-matrix/default.nix
index a67903c6fe98..265065a445e3 100644
--- a/pkgs/applications/networking/irc/weechat/scripts/weechat-matrix/default.nix
+++ b/pkgs/applications/networking/irc/weechat/scripts/weechat-matrix/default.nix
@@ -56,10 +56,10 @@ in buildPythonPackage {
     attrs
     logbook
     pygments
-    matrix-nio
+    (matrix-nio.override { withOlm = true; })
     aiohttp
     requests
-  ] ++ matrix-nio.optional-dependencies.e2e;
+  ];
 
   passthru.scripts = [ "matrix.py" ];
 
diff --git a/pkgs/applications/networking/opsdroid/default.nix b/pkgs/applications/networking/opsdroid/default.nix
index 7169850327be..3ef92e059e05 100644
--- a/pkgs/applications/networking/opsdroid/default.nix
+++ b/pkgs/applications/networking/opsdroid/default.nix
@@ -18,48 +18,45 @@ python3Packages.buildPythonPackage rec {
 
   build-system = with python3Packages; [ setuptools ];
 
-  dependencies =
-    with python3Packages;
-    [
-      aiohttp
-      aiohttp-middlewares
-      aioredis
-      aiosqlite
-      appdirs
-      arrow
-      babel
-      bitstring
-      bleach
-      # botbuilder-core, connector for teams
-      certifi
-      click
-      # dialogflow, connector for Dialogflow
-      dnspython
-      emoji
-      get-video-properties
-      ibm-watson
-      matrix-nio
-      mattermostdriver
-      motor
-      multidict
-      nbconvert
-      nbformat
-      opsdroid-get-image-size
-      parse
-      puremagic
-      pycron
-      python-olm
-      pyyaml
-      regex
-      rich
-      slack-sdk
-      tailer
-      voluptuous
-      watchgod
-      webexteamssdk
-      wrapt
-    ]
-    ++ matrix-nio.optional-dependencies.e2e;
+  dependencies = with python3Packages; [
+    aiohttp
+    aiohttp-middlewares
+    aioredis
+    aiosqlite
+    appdirs
+    arrow
+    babel
+    bitstring
+    bleach
+    # botbuilder-core, connector for teams
+    certifi
+    click
+    # dialogflow, connector for Dialogflow
+    dnspython
+    emoji
+    get-video-properties
+    ibm-watson
+    (matrix-nio.override { withOlm = true; })
+    mattermostdriver
+    motor
+    multidict
+    nbconvert
+    nbformat
+    opsdroid-get-image-size
+    parse
+    puremagic
+    pycron
+    python-olm
+    pyyaml
+    regex
+    rich
+    slack-sdk
+    tailer
+    voluptuous
+    watchgod
+    webexteamssdk
+    wrapt
+  ];
 
   passthru.python = python3Packages.python;
 
diff --git a/pkgs/development/python-modules/matrix-nio/allow-tests-without-olm.patch b/pkgs/development/python-modules/matrix-nio/allow-tests-without-olm.patch
new file mode 100644
index 000000000000..f8ae4015a3f9
--- /dev/null
+++ b/pkgs/development/python-modules/matrix-nio/allow-tests-without-olm.patch
@@ -0,0 +1,59 @@
+diff --git a/tests/async_client_test.py b/tests/async_client_test.py
+index 846c854a32..3a66af2baa 100644
+--- a/tests/async_client_test.py
++++ b/tests/async_client_test.py
+@@ -129,7 +129,10 @@
+ )
+ from nio.api import EventFormat, ResizingMethod, RoomPreset, RoomVisibility
+ from nio.client.async_client import connect_wrapper, on_request_chunk_sent
+-from nio.crypto import OlmDevice, Session, decrypt_attachment
++try:
++    from nio.crypto import OlmDevice, Session, decrypt_attachment
++except ImportError:
++    pass
+ 
+ TEST_ROOM_ID = "!testroom:example.org"
+ 
+diff --git a/tests/conftest.py b/tests/conftest.py
+index ae37ca1169..e5f791a31e 100644
+--- a/tests/conftest.py
++++ b/tests/conftest.py
+@@ -9,11 +9,17 @@
+     async_client_pair,
+     async_client_pair_same_user,
+ )
+-from olm import Account
++try:
++    from olm import Account
++except ImportError:
++    pass
+ 
+ from nio import Client, ClientConfig, HttpClient
+-from nio.crypto import Olm, OlmDevice
+-from nio.store import SqliteMemoryStore
++try:
++    from nio.crypto import Olm, OlmDevice
++    from nio.store import SqliteMemoryStore
++except ImportError:
++    pass
+ 
+ ALICE_ID = "@alice:example.org"
+ ALICE_DEVICE_ID = "JLAFKJWSCS"
+diff --git a/tests/helpers.py b/tests/helpers.py
+index 63445b605a..05096d1dc3 100644
+--- a/tests/helpers.py
++++ b/tests/helpers.py
+@@ -26,8 +26,11 @@
+     WindowUpdateFrame,
+ )
+ 
+-from nio.crypto import OlmAccount, OlmDevice
+-from nio.store import Ed25519Key
++try:
++    from nio.crypto import OlmAccount, OlmDevice
++    from nio.store import Ed25519Key
++except ImportError:
++    pass
+ 
+ SAMPLE_SETTINGS = {
+     SettingsFrame.HEADER_TABLE_SIZE: 4096,
diff --git a/pkgs/development/python-modules/matrix-nio/default.nix b/pkgs/development/python-modules/matrix-nio/default.nix
index 6e28fe1cffef..2ddbc6bb58a3 100644
--- a/pkgs/development/python-modules/matrix-nio/default.nix
+++ b/pkgs/development/python-modules/matrix-nio/default.nix
@@ -38,6 +38,8 @@
   pantalaimon,
   weechatScripts,
   zulip,
+
+  withOlm ? false,
 }:
 
 buildPythonPackage rec {
@@ -52,9 +54,14 @@ buildPythonPackage rec {
     hash = "sha256-XlswVHLvKOi1qr+I7Mbm4IBjn1DG7glgDsNY48NA5Ew=";
   };
 
-  nativeBuildInputs = [ poetry-core ];
+  patches = [
+    # Ignore olm import failures when testing
+    ./allow-tests-without-olm.patch
+  ];
 
-  propagatedBuildInputs = [
+  build-system = [ poetry-core ];
+
+  dependencies = [
     aiofiles
     aiohttp
     aiohttp-socks
@@ -63,13 +70,9 @@ buildPythonPackage rec {
     jsonschema
     pycryptodome
     unpaddedbase64
-  ];
+  ] ++ lib.optionals withOlm optional-dependencies.e2e;
 
-  pythonRelaxDeps = [
-    "aiohttp-socks" # Pending matrix-nio/matrix-nio#516
-  ];
-
-  passthru.optional-dependencies = {
+  optional-dependencies = {
     e2e = [
       atomicwrites
       cachetools
@@ -78,6 +81,10 @@ buildPythonPackage rec {
     ];
   };
 
+  pythonRelaxDeps = [
+    "aiohttp-socks" # Pending matrix-nio/matrix-nio#516
+  ];
+
   nativeCheckInputs = [
     aioresponses
     faker
@@ -87,17 +94,60 @@ buildPythonPackage rec {
     pytest-aiohttp
     pytest-benchmark
     pytestCheckHook
-  ] ++ passthru.optional-dependencies.e2e;
+  ];
 
   pytestFlagsArray = [ "--benchmark-disable" ];
 
-  disabledTests = [
-    # touches network
-    "test_connect_wrapper"
-    # time dependent and flaky
-    "test_transfer_monitor_callbacks"
+  disabledTestPaths = lib.optionals (!withOlm) [
+    "tests/encryption_test.py"
+    "tests/key_export_test.py"
+    "tests/memory_store_test.py"
+    "tests/sas_test.py"
+    "tests/sessions_test.py"
+    "tests/store_test.py"
   ];
 
+  disabledTests =
+    [
+      # touches network
+      "test_connect_wrapper"
+      # time dependent and flaky
+      "test_transfer_monitor_callbacks"
+    ]
+    ++ lib.optionals (!withOlm) [
+      "test_client_account_sharing"
+      "test_client_key_query"
+      "test_client_login"
+      "test_client_protocol_error"
+      "test_client_restore_login"
+      "test_client_room_creation"
+      "test_device_store"
+      "test_e2e_sending"
+      "test_early_store_loading"
+      "test_encrypted_data_generator"
+      "test_http_client_keys_query"
+      "test_key_claiming"
+      "test_key_exports"
+      "test_key_invalidation"
+      "test_key_sharing"
+      "test_key_sharing_callbacks"
+      "test_key_sharing_cancellation"
+      "test_keys_query"
+      "test_keys_upload"
+      "test_marking_sessions_as_shared"
+      "test_message_sending"
+      "test_query_rule"
+      "test_room_devices"
+      "test_sas_verification"
+      "test_sas_verification_cancel"
+      "test_session_sharing"
+      "test_session_sharing_2"
+      "test_session_unwedging"
+      "test_storing_room_encryption_state"
+      "test_sync_forever"
+      "test_sync_token_restoring"
+    ];
+
   passthru.tests = {
     inherit (nixosTests)
       dendrite
diff --git a/pkgs/development/python-modules/mautrix/default.nix b/pkgs/development/python-modules/mautrix/default.nix
index 9895f6417184..1746fa01dbe9 100644
--- a/pkgs/development/python-modules/mautrix/default.nix
+++ b/pkgs/development/python-modules/mautrix/default.nix
@@ -4,6 +4,7 @@
   fetchFromGitHub,
   pythonOlder,
   # deps
+  setuptools,
   aiohttp,
   attrs,
   yarl,
@@ -18,12 +19,14 @@
   aiosqlite,
   asyncpg,
   ruamel-yaml,
+
+  withOlm ? false,
 }:
 
 buildPythonPackage rec {
   pname = "mautrix";
   version = "0.20.6";
-  format = "setuptools";
+  pyproject = true;
 
   disabled = pythonOlder "3.10";
 
@@ -34,13 +37,15 @@ buildPythonPackage rec {
     hash = "sha256-g6y2u3ipSp5HoakHqd/ryPlyA+kR7zO6uY4AqfqbwiE=";
   };
 
-  propagatedBuildInputs = [
+  build-system = [ setuptools ];
+
+  dependencies = [
     aiohttp
     attrs
     yarl
-  ];
+  ] ++ lib.optionals withOlm optional-dependencies.encryption;
 
-  passthru.optional-dependencies = {
+  optional-dependencies = {
     detect_mimetype = [ python-magic ];
     encryption = [
       python-olm
@@ -55,7 +60,9 @@ buildPythonPackage rec {
     aiosqlite
     asyncpg
     ruamel-yaml
-  ] ++ passthru.optional-dependencies.encryption;
+  ];
+
+  disabledTestPaths = lib.optionals (!withOlm) [ "mautrix/crypto/" ];
 
   pythonImportsCheck = [ "mautrix" ];
 
diff --git a/pkgs/servers/mautrix-facebook/default.nix b/pkgs/servers/mautrix-facebook/default.nix
index cb7897853b52..3857e5abd45d 100644
--- a/pkgs/servers/mautrix-facebook/default.nix
+++ b/pkgs/servers/mautrix-facebook/default.nix
@@ -21,7 +21,7 @@ python3.pkgs.buildPythonPackage rec {
     aiohttp
     asyncpg
     commonmark
-    mautrix
+    (mautrix.override { withOlm = true; })
     paho-mqtt
     pillow
     prometheus-client
diff --git a/pkgs/servers/mautrix-googlechat/default.nix b/pkgs/servers/mautrix-googlechat/default.nix
index e0f5bc80b8f1..8f8bef86ab2b 100644
--- a/pkgs/servers/mautrix-googlechat/default.nix
+++ b/pkgs/servers/mautrix-googlechat/default.nix
@@ -55,7 +55,7 @@
     commonmark
     python-magic
     protobuf
-    mautrix
+    (mautrix.override { withOlm = enableE2be; })
   ] ++ lib.optionals enableE2be passthru.optional-dependencies.e2be
   ++ lib.optionals enableMetrics passthru.optional-dependencies.metrics
   ++ lib.optionals enableSqlite passthru.optional-dependencies.sqlite;
diff --git a/pkgs/servers/mautrix-telegram/default.nix b/pkgs/servers/mautrix-telegram/default.nix
index 4e9d00a9534b..53bdb18c22ae 100644
--- a/pkgs/servers/mautrix-telegram/default.nix
+++ b/pkgs/servers/mautrix-telegram/default.nix
@@ -43,7 +43,7 @@ python.pkgs.buildPythonPackage rec {
     commonmark
     aiohttp
     yarl
-    mautrix
+    (mautrix.override { withOlm = withE2BE; })
     tulir-telethon
     asyncpg
     mako
diff --git a/pkgs/tools/networking/maubot/default.nix b/pkgs/tools/networking/maubot/default.nix
index b66940d92390..dd932804fd1a 100644
--- a/pkgs/tools/networking/maubot/default.nix
+++ b/pkgs/tools/networking/maubot/default.nix
@@ -62,7 +62,7 @@ let
     propagatedBuildInputs = with python.pkgs; [
       setuptools
       # requirements.txt
-      mautrix
+      (mautrix.override { withOlm = encryptionSupport; })
       aiohttp
       yarl
       sqlalchemy