From 26f38fab473deda7c309db54442124bfc910e1b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Goran=20Meki=C4=87?= Date: Fri, 19 Jun 2026 03:18:22 +0200 Subject: [PATCH] Use LDAP as mailing list storage --- freenit/api/mailinglist.py | 143 +++++---- freenit/base_config.py | 17 ++ freenit/mailinglist/store.py | 55 ++++ freenit/mailinglist/store_ldap.py | 447 +++++++++++++++++++++++++++++ freenit/mailinglist/store_sql.py | 163 +++++++++++ freenit/mailinglist/worker.py | 14 +- freenit/models/ldap/base.py | 20 ++ freenit/models/ldap/mailinglist.py | 154 ++++++++++ freenit/models/mailinglist.py | 10 + freenit/models/sql/base.py | 64 ----- freenit/models/sql/mailinglist.py | 76 +++++ freenit/stalwart.py | 6 + ldap/README.md | 79 ++++- ldap/mailinglist.ldif | 190 ++++++++++++ ldap/mailinglist.schema | 217 ++++++++++++++ tests/test_mailinglist.py | 22 +- 16 files changed, 1522 insertions(+), 155 deletions(-) create mode 100644 freenit/mailinglist/store.py create mode 100644 freenit/mailinglist/store_ldap.py create mode 100644 freenit/mailinglist/store_sql.py create mode 100644 freenit/models/ldap/mailinglist.py create mode 100644 freenit/models/mailinglist.py create mode 100644 freenit/models/sql/mailinglist.py create mode 100644 ldap/mailinglist.ldif create mode 100644 ldap/mailinglist.schema diff --git a/freenit/api/mailinglist.py b/freenit/api/mailinglist.py index 86cfcd5..55d633b 100644 --- a/freenit/api/mailinglist.py +++ b/freenit/api/mailinglist.py @@ -3,13 +3,13 @@ from typing import Any, List from uuid import uuid4 -import oxyde import pydantic from fastapi import Depends, Header, HTTPException, Request from freenit.api.router import route from freenit.config import getConfig from freenit.decorators import description +from freenit.mailinglist import store from freenit.mailinglist.mail import ( subscribe_confirmation, unsubscribe_confirmation, @@ -20,8 +20,8 @@ reject_message, ) from freenit.mail import sendmail -from freenit.models.pagination import Page, paginate -from freenit.models.sql.base import MailingList, ModerationMessage, PendingSubscriber +from freenit.models.mailinglist import MailingList, ModerationMessage, PendingSubscriber +from freenit.models.pagination import Page from freenit.models.user import User from freenit.permissions import mailinglist_perms from freenit.stalwart import ( @@ -34,6 +34,7 @@ fetch_email_summaries, fetch_mailbox_messages, fetch_principal, + list_domains, remove_external_member, ) @@ -59,7 +60,7 @@ def _parse_address(address: pydantic.EmailStr) -> tuple[str, str]: class MailingListCreate(pydantic.BaseModel): name: str - address: pydantic.EmailStr + domain: str description: str | None = None public: bool = True archive_enabled: bool = True @@ -132,28 +133,15 @@ class SubscriberResponse(pydantic.BaseModel): async def _get_list(id: int) -> MailingList: - try: - return await MailingList.objects.get(id=id) - except oxyde.NotFoundError: - raise HTTPException(status_code=404, detail="No such mailing list") + return await store.get_mailing_list(id) async def _get_pending(id: int, token: str, action: str) -> PendingSubscriber: - try: - return await PendingSubscriber.objects.filter( - mailing_list_id=id, token=token, action=action - ).get() - except oxyde.NotFoundError: - raise HTTPException(status_code=404, detail="Invalid or expired token") + return await store.get_pending_subscriber(id, token, action) async def _get_moderation(id: int, msg_id: int) -> ModerationMessage: - try: - return await ModerationMessage.objects.filter( - id=msg_id, mailing_list_id=id - ).get() - except oxyde.NotFoundError: - raise HTTPException(status_code=404, detail="No such moderation message") + return await store.get_moderation_message(id, msg_id) @route("/mailinglists", tags=tags) @@ -166,7 +154,7 @@ async def get( cur_user: User = Depends(mailinglist_perms), ) -> Page[MailingListResponse]: _require_admin(cur_user) - return await paginate(MailingList.objects, page, perpage) + return await store.list_mailing_lists(page, perpage) @staticmethod @description("Create mailing list") @@ -175,18 +163,21 @@ async def post( cur_user: User = Depends(mailinglist_perms), ) -> MailingListResponse: _require_admin(cur_user) - local, domain = _parse_address(data.address) + if "@" in data.name or "/" in data.name: + raise HTTPException(status_code=400, detail="Invalid mailing list name") + address = f"{data.name}@{data.domain}" + local, domain = _parse_address(address) distribution_address = f"{local}-members@{domain}" archive_address = f"{local}-archive@{domain}" - existing_count = await MailingList.objects.filter( - address__in=[data.address, distribution_address, archive_address] - ).count() + existing_count = await store.count_mailing_lists_by_addresses( + [address, distribution_address, archive_address] + ) if existing_count > 0: raise HTTPException(status_code=409, detail="Mailing list address already in use") try: - inbox_id = await create_inbox_account(data.name, data.address) + inbox_id = await create_inbox_account(data.name, address) list_id = await create_list_principal(data.name, distribution_address) archive_id = await create_archive_account(data.name, archive_address) await add_external_member(list_id, archive_address) @@ -194,25 +185,22 @@ async def post( log.error("Failed to create Stalwart principals: %s", e) raise HTTPException(status_code=502, detail=f"Stalwart error: {e}") - try: - now = datetime.utcnow() - mailing_list = await MailingList.objects.create( - name=data.name, - address=data.address, - distribution_address=distribution_address, - archive_address=archive_address, - description=data.description, - public=data.public, - archive_enabled=data.archive_enabled, - moderation_enabled=data.moderation_enabled, - principal_id=list_id, - inbox_principal_id=inbox_id, - archive_principal_id=archive_id, - created_at=now, - updated_at=now, - ) - except oxyde.IntegrityError as e: - raise HTTPException(status_code=409, detail=f"Mailing list already exists: {e}") + now = datetime.utcnow() + mailing_list = await store.create_mailing_list( + name=data.name, + address=address, + distribution_address=distribution_address, + archive_address=archive_address, + description=data.description, + public=data.public, + archive_enabled=data.archive_enabled, + moderation_enabled=data.moderation_enabled, + principal_id=list_id, + inbox_principal_id=inbox_id, + archive_principal_id=archive_id, + created_at=now, + updated_at=now, + ) return MailingListResponse.model_validate(mailing_list) @@ -225,11 +213,22 @@ async def get( page: int = Header(default=1), perpage: int = Header(default=10), ) -> Page[PublicMailingListResponse]: - return await paginate( - MailingList.objects.filter(public=True), - page, - perpage, - ) + return await store.list_mailing_lists(page, perpage, public=True) + + +@route("/mailinglists/domains", tags=tags) +class MailingListDomainsAPI: + @staticmethod + @description("Get available mail domains from Stalwart") + async def get( + cur_user: User = Depends(mailinglist_perms), + ) -> List[str]: + _require_admin(cur_user) + try: + return await list_domains() + except Exception as e: + log.error("Failed to list Stalwart domains: %s", e) + raise HTTPException(status_code=502, detail=f"Stalwart error: {e}") @route("/mailinglists/{id}", tags=tags) @@ -248,7 +247,7 @@ async def patch( ) -> MailingListResponse: _require_admin(cur_user) mailing_list = await _get_list(id) - await mailing_list.patch(data) + await store.update_mailing_list(mailing_list, data) return MailingListResponse.model_validate(mailing_list) @staticmethod @@ -265,7 +264,7 @@ async def delete(id: int, cur_user: User = Depends(mailinglist_perms)) -> Mailin await delete_principal(principal_id) except Exception as e: log.error("Failed to delete Stalwart principals for list %s: %s", id, e) - await mailing_list.delete() + await store.delete_mailing_list(mailing_list) return MailingListResponse.model_validate(mailing_list) @@ -402,17 +401,17 @@ async def post( raise HTTPException(status_code=403, detail="Subscribing is not allowed") try: - pending = await PendingSubscriber.objects.filter( - mailing_list_id=id, email=data.email, action="subscribe" - ).get() - except oxyde.NotFoundError: - pending = await PendingSubscriber.objects.create( + pending = await store.get_pending_subscriber_by_email( + id, data.email, "subscribe" + ) + except HTTPException as exc: + if exc.status_code != 404: + raise + pending = await store.create_pending_subscriber( mailing_list=mailing_list, - mailing_list_id=mailing_list.id, email=data.email, action="subscribe", token=str(uuid4()), - created_at=datetime.utcnow(), ) confirm_url = f"{request.base_url}api/v1/mailinglists/{id}/confirm/{pending.token}" msg = subscribe_confirmation(mailing_list.name, mailing_list.address, confirm_url) @@ -439,7 +438,7 @@ async def get(id: int, token: str) -> dict[str, str]: except Exception as e: log.error("Failed to add member %s to list %s: %s", pending.email, id, e) raise HTTPException(status_code=502, detail=f"Stalwart error: {e}") - await pending.delete() + await store.delete_pending_subscriber(pending) return {"detail": "Subscription confirmed"} @@ -457,17 +456,17 @@ async def post( raise HTTPException(status_code=403, detail="Unsubscribing is not allowed") try: - pending = await PendingSubscriber.objects.filter( - mailing_list_id=id, email=data.email, action="unsubscribe" - ).get() - except oxyde.NotFoundError: - pending = await PendingSubscriber.objects.create( + pending = await store.get_pending_subscriber_by_email( + id, data.email, "unsubscribe" + ) + except HTTPException as exc: + if exc.status_code != 404: + raise + pending = await store.create_pending_subscriber( mailing_list=mailing_list, - mailing_list_id=mailing_list.id, email=data.email, action="unsubscribe", token=str(uuid4()), - created_at=datetime.utcnow(), ) confirm_url = f"{request.base_url}api/v1/mailinglists/{id}/unsubscribe/{pending.token}" msg = unsubscribe_confirmation(mailing_list.name, mailing_list.address, confirm_url) @@ -494,7 +493,7 @@ async def get(id: int, token: str) -> dict[str, str]: except Exception as e: log.error("Failed to remove member %s from list %s: %s", pending.email, id, e) raise HTTPException(status_code=502, detail=f"Stalwart error: {e}") - await pending.delete() + await store.delete_pending_subscriber(pending) return {"detail": "Unsubscription confirmed"} @@ -510,12 +509,8 @@ async def get( ) -> Page[ModerationMessageResponse]: _require_admin(cur_user) mailing_list = await _get_list(id) - return await paginate( - ModerationMessage.objects.filter( - mailing_list_id=mailing_list.id, status="pending" - ).order_by("-created_at"), - page, - perpage, + return await store.list_moderation_messages( + mailing_list.id, "pending", page, perpage ) diff --git a/freenit/base_config.py b/freenit/base_config.py index 201ffb3..9bfe8dc 100644 --- a/freenit/base_config.py +++ b/freenit/base_config.py @@ -84,6 +84,14 @@ def __init__( gidNextField="gidNumber", domainDN="ou={}", domainClasses=["organizationalUnit", "pmiDelegationPath"], + mailinglistBase="ou=mailinglists,dc=ldap", + mailinglistDN="cn={}", + mailinglistClasses=["freenitMailingList"], + pendingSubscriberClasses=["freenitPendingSubscriber"], + moderationMessageClasses=["freenitModerationMessage"], + mlidNextClass="freenitMailingListIdNext", + mlidNextDN="cn=mlidnext,dc=ldap", + mlidNextField="mlidNumber", ): self.host = host self.tls = tls @@ -108,6 +116,14 @@ def __init__( self.gidNextField = gidNextField self.domainDN = domainDN self.domainClasses = domainClasses + self.mailinglistBase = mailinglistBase + self.mailinglistDN = f"{mailinglistDN},{mailinglistBase}" + self.mailinglistClasses = mailinglistClasses + self.pendingSubscriberClasses = pendingSubscriberClasses + self.moderationMessageClasses = moderationMessageClasses + self.mlidNextClass = mlidNextClass + self.mlidNextDN = mlidNextDN + self.mlidNextField = mlidNextField class BaseConfig: @@ -123,6 +139,7 @@ class BaseConfig: user = "freenit.models.sql.user" role = "freenit.models.sql.role" theme = "freenit.models.sql.theme" + mailinglist = "freenit.models.sql.mailinglist" theme_name = "Freenit" meta = None auth = Auth() diff --git a/freenit/mailinglist/store.py b/freenit/mailinglist/store.py new file mode 100644 index 0000000..d0c4596 --- /dev/null +++ b/freenit/mailinglist/store.py @@ -0,0 +1,55 @@ +from freenit.models.mailinglist import MailingList + +if MailingList.dbtype() == "sql": + from .store_sql import ( + count_mailing_lists_by_addresses, + create_mailing_list, + create_moderation_message, + create_pending_subscriber, + delete_mailing_list, + delete_pending_subscriber, + get_mailing_list, + get_moderation_message, + get_pending_subscriber, + get_pending_subscriber_by_email, + list_mailing_lists, + list_moderation_messages, + save_moderation_message, + update_mailing_list, + ) +elif MailingList.dbtype() == "ldap": + from .store_ldap import ( + count_mailing_lists_by_addresses, + create_mailing_list, + create_moderation_message, + create_pending_subscriber, + delete_mailing_list, + delete_pending_subscriber, + get_mailing_list, + get_moderation_message, + get_pending_subscriber, + get_pending_subscriber_by_email, + list_mailing_lists, + list_moderation_messages, + save_moderation_message, + update_mailing_list, + ) +else: + raise RuntimeError(f"Unsupported mailing list backend: {MailingList.dbtype()}") + +__all__ = [ + "count_mailing_lists_by_addresses", + "create_mailing_list", + "create_moderation_message", + "create_pending_subscriber", + "delete_mailing_list", + "delete_pending_subscriber", + "get_mailing_list", + "get_moderation_message", + "get_pending_subscriber", + "get_pending_subscriber_by_email", + "list_mailing_lists", + "list_moderation_messages", + "save_moderation_message", + "update_mailing_list", +] diff --git a/freenit/mailinglist/store_ldap.py b/freenit/mailinglist/store_ldap.py new file mode 100644 index 0000000..20a1877 --- /dev/null +++ b/freenit/mailinglist/store_ldap.py @@ -0,0 +1,447 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from math import ceil + +from bonsai import LDAPEntry, LDAPSearchScope, errors +from fastapi import HTTPException + +from freenit.config import getConfig +from freenit.models.ldap.base import class2filter, get_client, next_mlid, save_data +from freenit.models.mailinglist import MailingList, ModerationMessage, PendingSubscriber +from freenit.models.pagination import Page + +config = getConfig() + + +def _ldap_dt(dt: datetime | None) -> str | None: + if dt is None: + return None + if dt.tzinfo is not None: + dt = dt.astimezone(timezone.utc).replace(tzinfo=None) + return dt.strftime("%Y%m%d%H%M%SZ") + + +def _ldap_bool(value: bool) -> str: + return "TRUE" if value else "FALSE" + + +def _list_dn(name: str) -> str: + return config.ldap.mailinglistDN.format(name) + + +def _pending_dn(list_name: str, token: str) -> str: + return f"cn={token},{_list_dn(list_name)}" + + +def _moderation_dn(list_name: str, msg_id: int) -> str: + return f"cn={msg_id},{_list_dn(list_name)}" + + +def _ml_filter(): + return class2filter(config.ldap.mailinglistClasses) + + +def _pending_filter(): + return class2filter(config.ldap.pendingSubscriberClasses) + + +def _moderation_filter(): + return class2filter(config.ldap.moderationMessageClasses) + + +async def _search_list_by_id(list_id: int): + classes = _ml_filter() + client = get_client() + async with client.connect(is_async=True) as conn: + res = await conn.search( + config.ldap.mailinglistBase, + LDAPSearchScope.SUB, + f"(&{classes}(mlidNumber={list_id}))", + ) + if len(res) < 1: + raise HTTPException(status_code=404, detail="No such mailing list") + if len(res) > 1: + raise HTTPException(status_code=409, detail="Multiple mailing lists found") + return res[0] + + +async def _search_list_by_name(name: str): + classes = _ml_filter() + client = get_client() + async with client.connect(is_async=True) as conn: + res = await conn.search( + config.ldap.mailinglistBase, + LDAPSearchScope.SUB, + f"(&{classes}(cn={name}))", + ) + return res[0] if res else None + + +async def _count_by_addresses(addresses: list[str]) -> int: + classes = _ml_filter() + address_filters = "" + for addr in addresses: + address_filters += ( + f"(mailinglistAddress={addr})" + f"(mailinglistDistributionAddress={addr})" + f"(mailinglistArchiveAddress={addr})" + ) + client = get_client() + async with client.connect(is_async=True) as conn: + res = await conn.search( + config.ldap.mailinglistBase, + LDAPSearchScope.SUB, + f"(&{classes}(|{address_filters}))", + ) + return len(res) + + +async def get_mailing_list(id: int) -> MailingList: + entry = await _search_list_by_id(id) + return MailingList.from_entry(entry) + + +async def list_mailing_lists( + page: int, + perpage: int, + public: bool | None = None, +) -> Page[MailingList]: + classes = _ml_filter() + filter_exp = classes + if public is not None: + filter_exp = f"(&{classes}(mailinglistPublic={_ldap_bool(public)}))" + client = get_client() + async with client.connect(is_async=True) as conn: + res = await conn.search( + config.ldap.mailinglistBase, + LDAPSearchScope.SUB, + filter_exp, + ) + data = [MailingList.from_entry(entry) for entry in res] + data.sort(key=lambda ml: ml.id, reverse=True) + total = len(data) + pages = ceil(total / perpage) if perpage else 1 + if total > 0 and page > pages: + raise HTTPException(status_code=404, detail="No such page") + offset = max(page - 1, 0) * perpage + page_data = data[offset : offset + perpage] + return Page(total=total, page=page, pages=pages, perpage=perpage, data=page_data) + + +async def count_mailing_lists_by_addresses(addresses: list[str]) -> int: + return await _count_by_addresses(addresses) + + +async def create_mailing_list(**kwargs) -> MailingList: + name = kwargs["name"] + address = kwargs["address"] + distribution_address = kwargs["distribution_address"] + archive_address = kwargs["archive_address"] + + existing = await _search_list_by_name(name) + if existing is not None: + raise HTTPException(status_code=409, detail="Mailing list already exists") + + if await _count_by_addresses([address, distribution_address, archive_address]) > 0: + raise HTTPException( + status_code=409, detail="Mailing list address already in use" + ) + + list_id = await next_mlid() + now = kwargs.get("created_at") or datetime.utcnow() + dn = _list_dn(name) + entry = LDAPEntry(dn) + entry["objectClass"] = config.ldap.mailinglistClasses + entry["cn"] = name + entry["mlidNumber"] = list_id + entry["mailinglistAddress"] = address + entry["mailinglistDistributionAddress"] = distribution_address + entry["mailinglistArchiveAddress"] = archive_address + description = kwargs.get("description") + if description: + entry["description"] = description + entry["mailinglistPublic"] = _ldap_bool(kwargs.get("public", True)) + entry["mailinglistArchiveEnabled"] = _ldap_bool(kwargs.get("archive_enabled", True)) + entry["mailinglistModerationEnabled"] = _ldap_bool( + kwargs.get("moderation_enabled", False) + ) + for key, attr in [ + ("principal_id", "principalId"), + ("inbox_principal_id", "inboxPrincipalId"), + ("archive_principal_id", "archivePrincipalId"), + ]: + value = kwargs.get(key) + if value is not None: + entry[attr] = int(value) + entry["createdAt"] = _ldap_dt(now) + entry["updatedAt"] = _ldap_dt(now) + + try: + await save_data(entry) + except errors.AlreadyExists: + raise HTTPException(status_code=409, detail="Mailing list already exists") + + return MailingList( + dn=dn, + id=list_id, + name=name, + address=address, + distribution_address=distribution_address, + archive_address=archive_address, + description=description, + public=kwargs.get("public", True), + archive_enabled=kwargs.get("archive_enabled", True), + moderation_enabled=kwargs.get("moderation_enabled", False), + principal_id=kwargs.get("principal_id"), + inbox_principal_id=kwargs.get("inbox_principal_id"), + archive_principal_id=kwargs.get("archive_principal_id"), + created_at=now, + updated_at=now, + ) + + +async def update_mailing_list(mailing_list: MailingList, data) -> MailingList: + fields = data.model_dump(exclude_none=True) + if not fields: + return mailing_list + + client = get_client() + async with client.connect(is_async=True) as conn: + res = await conn.search(mailing_list.dn, LDAPSearchScope.BASE) + if len(res) < 1: + raise HTTPException(status_code=404, detail="No such mailing list") + entry = res[0] + + if "description" in fields: + entry["description"] = fields["description"] or [] + if "public" in fields: + entry["mailinglistPublic"] = _ldap_bool(fields["public"]) + if "archive_enabled" in fields: + entry["mailinglistArchiveEnabled"] = _ldap_bool(fields["archive_enabled"]) + if "moderation_enabled" in fields: + entry["mailinglistModerationEnabled"] = _ldap_bool( + fields["moderation_enabled"] + ) + entry["updatedAt"] = _ldap_dt(datetime.utcnow()) + await entry.modify() + + for key, value in fields.items(): + setattr(mailing_list, key, value) + mailing_list.updated_at = datetime.utcnow() + return mailing_list + + +async def delete_mailing_list(mailing_list: MailingList) -> None: + client = get_client() + async with client.connect(is_async=True) as conn: + children = await conn.search( + mailing_list.dn, + LDAPSearchScope.SUB, + "(objectClass=*)", + attrlist=["objectClass"], + ) + for child in children: + child_dn = str(child["dn"]) + if child_dn != mailing_list.dn: + await child.delete() + res = await conn.search(mailing_list.dn, LDAPSearchScope.BASE) + if res: + await res[0].delete() + + +async def get_pending_subscriber( + mailing_list_id: int, + token: str, + action: str, +) -> PendingSubscriber: + list_entry = await _search_list_by_id(mailing_list_id) + list_name = str(list_entry["cn"][0]) + classes = _pending_filter() + client = get_client() + async with client.connect(is_async=True) as conn: + res = await conn.search( + _list_dn(list_name), + LDAPSearchScope.ONELEVEL, + f"(&{classes}(mailinglistToken={token})(mailinglistAction={action}))", + ) + if len(res) < 1: + raise HTTPException(status_code=404, detail="Invalid or expired token") + return PendingSubscriber.from_entry(res[0], mailing_list_id) + + +async def get_pending_subscriber_by_email( + mailing_list_id: int, + email: str, + action: str, +) -> PendingSubscriber: + list_entry = await _search_list_by_id(mailing_list_id) + list_name = str(list_entry["cn"][0]) + classes = _pending_filter() + client = get_client() + async with client.connect(is_async=True) as conn: + res = await conn.search( + _list_dn(list_name), + LDAPSearchScope.ONELEVEL, + f"(&{classes}(mail={email})(mailinglistAction={action}))", + ) + if len(res) < 1: + raise HTTPException(status_code=404, detail="Pending subscriber not found") + return PendingSubscriber.from_entry(res[0], mailing_list_id) + + +async def create_pending_subscriber( + mailing_list: MailingList, + email: str, + action: str, + token: str, +) -> PendingSubscriber: + pending_id = await next_mlid() + created_at = datetime.utcnow() + dn = _pending_dn(mailing_list.name, token) + entry = LDAPEntry(dn) + entry["objectClass"] = config.ldap.pendingSubscriberClasses + entry["cn"] = token + entry["mlidNumber"] = pending_id + entry["mail"] = email + entry["mailinglistToken"] = token + entry["mailinglistAction"] = action + entry["createdAt"] = _ldap_dt(created_at) + await save_data(entry) + return PendingSubscriber( + dn=dn, + id=pending_id, + mailing_list_id=mailing_list.id, + email=email, + token=token, + action=action, + created_at=created_at, + ) + + +async def delete_pending_subscriber(pending: PendingSubscriber) -> None: + client = get_client() + async with client.connect(is_async=True) as conn: + res = await conn.search(pending.dn, LDAPSearchScope.BASE) + if res: + await res[0].delete() + + +async def get_moderation_message( + mailing_list_id: int, + msg_id: int, +) -> ModerationMessage: + list_entry = await _search_list_by_id(mailing_list_id) + list_name = str(list_entry["cn"][0]) + classes = _moderation_filter() + client = get_client() + async with client.connect(is_async=True) as conn: + res = await conn.search( + _list_dn(list_name), + LDAPSearchScope.ONELEVEL, + f"(&{classes}(mlidNumber={msg_id}))", + ) + if len(res) < 1: + raise HTTPException(status_code=404, detail="No such moderation message") + return ModerationMessage.from_entry(res[0], mailing_list_id) + + +async def list_moderation_messages( + mailing_list_id: int, + status: str, + page: int, + perpage: int, +) -> Page[ModerationMessage]: + list_entry = await _search_list_by_id(mailing_list_id) + list_name = str(list_entry["cn"][0]) + classes = _moderation_filter() + client = get_client() + async with client.connect(is_async=True) as conn: + res = await conn.search( + _list_dn(list_name), + LDAPSearchScope.ONELEVEL, + f"(&{classes}(mailinglistStatus={status}))", + ) + data = [ModerationMessage.from_entry(entry, mailing_list_id) for entry in res] + data.sort(key=lambda msg: msg.created_at or datetime.min, reverse=True) + total = len(data) + pages = ceil(total / perpage) if perpage else 1 + if total > 0 and page > pages: + raise HTTPException(status_code=404, detail="No such page") + offset = max(page - 1, 0) * perpage + page_data = data[offset : offset + perpage] + return Page(total=total, page=page, pages=pages, perpage=perpage, data=page_data) + + +async def create_moderation_message( + mailing_list: MailingList, + message_id: str | None, + subject: str | None, + sender: str | None, + sent_at: datetime, + text_body: str | None, + html_body: str | None, + status: str, + created_at: datetime, +) -> ModerationMessage: + msg_id = await next_mlid() + dn = _moderation_dn(mailing_list.name, msg_id) + entry = LDAPEntry(dn) + entry["objectClass"] = config.ldap.moderationMessageClasses + entry["cn"] = str(msg_id) + entry["mlidNumber"] = msg_id + if message_id: + entry["mailinglistMessageId"] = message_id + if subject: + entry["mailinglistSubject"] = subject + if sender: + entry["mailinglistSender"] = sender + entry["sentAt"] = _ldap_dt(sent_at) + if text_body: + entry["textBody"] = text_body + if html_body: + entry["htmlBody"] = html_body + entry["mailinglistStatus"] = status + entry["createdAt"] = _ldap_dt(created_at) + await save_data(entry) + return ModerationMessage( + dn=dn, + id=msg_id, + mailing_list_id=mailing_list.id, + message_id=message_id, + subject=subject, + sender=sender, + sent_at=sent_at, + text_body=text_body, + html_body=html_body, + status=status, + created_at=created_at, + decided_at=None, + ) + + +async def save_moderation_message( + moderation_message: ModerationMessage, + update_fields: list[str], +) -> None: + client = get_client() + async with client.connect(is_async=True) as conn: + res = await conn.search(moderation_message.dn, LDAPSearchScope.BASE) + if len(res) < 1: + raise HTTPException(status_code=404, detail="No such moderation message") + entry = res[0] + mapping = { + "status": "mailinglistStatus", + "decided_at": "decidedAt", + } + for field in update_fields: + attr = mapping.get(field, field) + value = getattr(moderation_message, field) + if field == "decided_at": + value = _ldap_dt(value) + if value is None: + if attr in entry: + del entry[attr] + else: + entry[attr] = value + await entry.modify() diff --git a/freenit/mailinglist/store_sql.py b/freenit/mailinglist/store_sql.py new file mode 100644 index 0000000..0522499 --- /dev/null +++ b/freenit/mailinglist/store_sql.py @@ -0,0 +1,163 @@ +from datetime import datetime + +from fastapi import HTTPException + +from freenit.models.mailinglist import ( + IntegrityError, + MailingList, + ModerationMessage, + NotFoundError, + PendingSubscriber, +) +from freenit.models.pagination import Page, paginate + + +async def get_mailing_list(id: int) -> MailingList: + try: + return await MailingList.objects.get(id=id) + except NotFoundError: + raise HTTPException(status_code=404, detail="No such mailing list") + + +async def list_mailing_lists( + page: int, + perpage: int, + public: bool | None = None, +) -> Page[MailingList]: + query = MailingList.objects + if public is not None: + query = query.filter(public=public) + return await paginate(query, page, perpage) + + +async def count_mailing_lists_by_addresses(addresses: list[str]) -> int: + return await MailingList.objects.filter(address__in=addresses).count() + + +async def create_mailing_list(**kwargs) -> MailingList: + try: + return await MailingList.objects.create(**kwargs) + except IntegrityError as e: + raise HTTPException( + status_code=409, detail=f"Mailing list already exists: {e}" + ) + + +async def update_mailing_list(mailing_list: MailingList, data) -> MailingList: + await mailing_list.patch(data) + return mailing_list + + +async def delete_mailing_list(mailing_list: MailingList) -> None: + await mailing_list.delete() + + +async def get_pending_subscriber( + mailing_list_id: int, + token: str, + action: str, +) -> PendingSubscriber: + try: + return await PendingSubscriber.objects.filter( + mailing_list_id=mailing_list_id, + token=token, + action=action, + ).get() + except NotFoundError: + raise HTTPException(status_code=404, detail="Invalid or expired token") + + +async def get_pending_subscriber_by_email( + mailing_list_id: int, + email: str, + action: str, +) -> PendingSubscriber: + try: + return await PendingSubscriber.objects.filter( + mailing_list_id=mailing_list_id, + email=email, + action=action, + ).get() + except NotFoundError: + raise HTTPException(status_code=404, detail="Pending subscriber not found") + + +async def create_pending_subscriber( + mailing_list: MailingList, + email: str, + action: str, + token: str, +) -> PendingSubscriber: + return await PendingSubscriber.objects.create( + mailing_list=mailing_list, + mailing_list_id=mailing_list.id, + email=email, + action=action, + token=token, + created_at=datetime.utcnow(), + ) + + +async def delete_pending_subscriber(pending: PendingSubscriber) -> None: + await pending.delete() + + +async def get_moderation_message( + mailing_list_id: int, + msg_id: int, +) -> ModerationMessage: + try: + return await ModerationMessage.objects.filter( + id=msg_id, + mailing_list_id=mailing_list_id, + ).get() + except NotFoundError: + raise HTTPException(status_code=404, detail="No such moderation message") + + +async def list_moderation_messages( + mailing_list_id: int, + status: str, + page: int, + perpage: int, +) -> Page[ModerationMessage]: + return await paginate( + ModerationMessage.objects.filter( + mailing_list_id=mailing_list_id, + status=status, + ).order_by("-created_at"), + page, + perpage, + ) + + +async def create_moderation_message( + mailing_list: MailingList, + message_id: str | None, + subject: str | None, + sender: str | None, + sent_at: datetime, + text_body: str | None, + html_body: str | None, + status: str, + created_at: datetime, +) -> ModerationMessage: + return await ModerationMessage.objects.create( + mailing_list=mailing_list, + mailing_list_id=mailing_list.id, + message_id=message_id, + subject=subject, + sender=sender, + sent_at=sent_at, + text_body=text_body, + html_body=html_body, + status=status, + created_at=created_at, + ) + + +async def save_moderation_message( + moderation_message: ModerationMessage, + update_fields: list[str], +) -> None: + await moderation_message.save(update_fields=update_fields) diff --git a/freenit/mailinglist/worker.py b/freenit/mailinglist/worker.py index 058bc78..ce3767c 100644 --- a/freenit/mailinglist/worker.py +++ b/freenit/mailinglist/worker.py @@ -5,7 +5,8 @@ from freenit.config import getConfig from freenit.mail import sendmail -from freenit.models.sql.base import MailingList, ModerationMessage +from freenit.mailinglist import store +from freenit.models.mailinglist import MailingList, ModerationMessage from freenit.stalwart import ( destroy_emails, fetch_email_bodies, @@ -111,9 +112,8 @@ async def _store_moderation(mailing_list: MailingList, email_data: dict[str, Any message_id = message_ids[0] if isinstance(message_ids, list) and message_ids else None received_at = email_data.get("receivedAt") sent_at = datetime.fromisoformat(received_at.replace("Z", "+00:00")) if received_at else datetime.utcnow() - return await ModerationMessage.objects.create( + return await store.create_moderation_message( mailing_list=mailing_list, - mailing_list_id=mailing_list.id, message_id=message_id, subject=email_data.get("subject"), sender=_email_address(email_data.get("from")), @@ -191,10 +191,14 @@ async def approve_message(mailing_list: MailingList, moderation_message: Moderat await _distribute(mailing_list, email_data, moderation_message.text_body, moderation_message.html_body) moderation_message.status = "approved" moderation_message.decided_at = datetime.utcnow() - await moderation_message.save(update_fields=["status", "decided_at"]) + await store.save_moderation_message( + moderation_message, update_fields=["status", "decided_at"] + ) async def reject_message(moderation_message: ModerationMessage) -> None: moderation_message.status = "rejected" moderation_message.decided_at = datetime.utcnow() - await moderation_message.save(update_fields=["status", "decided_at"]) + await store.save_moderation_message( + moderation_message, update_fields=["status", "decided_at"] + ) diff --git a/freenit/models/ldap/base.py b/freenit/models/ldap/base.py index d464f5d..2fbbdd0 100644 --- a/freenit/models/ldap/base.py +++ b/freenit/models/ldap/base.py @@ -71,6 +71,26 @@ async def next_gid(increment=True): raise HTTPException(status_code=403, detail="Failed to login") +async def next_mlid(increment=True) -> int: + client = get_client() + try: + async with client.connect(is_async=True) as conn: + res = await conn.search( + config.ldap.mlidNextDN, + LDAPSearchScope.BASE, + f"objectClass={config.ldap.mlidNextClass}", + ) + if len(res) < 1: + raise HTTPException(status_code=404, detail="Can not find next MLID") + mlidNext = int(res[0][config.ldap.mlidNextField][0]) + if increment: + res[0][config.ldap.mlidNextField] = mlidNext + 1 + await res[0].modify() + return mlidNext + except errors.AuthenticationError: + raise HTTPException(status_code=403, detail="Failed to login") + + def class2filter(classes): return "".join([f"(objectClass={group})" for group in classes]) diff --git a/freenit/models/ldap/mailinglist.py b/freenit/models/ldap/mailinglist.py new file mode 100644 index 0000000..718ce8b --- /dev/null +++ b/freenit/models/ldap/mailinglist.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, EmailStr, Field + +from freenit.models.ldap.base import LDAPBaseModel + + +class NotFoundError(Exception): + pass + + +class IntegrityError(Exception): + pass + + +def _first(entry, attr, default=None): + value = entry.get(attr) + if not value: + return default + if isinstance(value, (list, tuple)): + value = value[0] + return value + + +def _int(entry, attr, default=None): + value = _first(entry, attr) + if value is None: + return default + return int(value) + + +def _bool(entry, attr, default=False): + value = _first(entry, attr) + if value is None: + return default + return str(value).upper() == "TRUE" + + +def _datetime(entry, attr, default=None): + value = _first(entry, attr) + if value is None: + return default + value = str(value) + if value.endswith("Z"): + value = value[:-1] + "+00:00" + return datetime.fromisoformat(value) + + +class MailingList(LDAPBaseModel): + id: int = Field(0, description="Numeric mailing list ID") + name: str = Field("", description="Mailing list name") + address: EmailStr = Field("", description="Primary list address") + distribution_address: EmailStr = Field("", description="Distribution list address") + archive_address: EmailStr = Field("", description="Archive account address") + description: str | None = Field(None, description="Optional description") + public: bool = Field(True, description="Publicly visible") + archive_enabled: bool = Field(True, description="Archive enabled") + moderation_enabled: bool = Field(False, description="Moderation enabled") + principal_id: int | None = Field(None, description="Stalwart list principal ID") + inbox_principal_id: int | None = Field(None, description="Stalwart inbox principal ID") + archive_principal_id: int | None = Field(None, description="Stalwart archive principal ID") + created_at: datetime | None = Field(None, description="Creation timestamp") + updated_at: datetime | None = Field(None, description="Last update timestamp") + + @classmethod + def from_entry(cls, entry): + return cls( + dn=str(entry["dn"]), + id=_int(entry, "mlidNumber", 0), + name=_first(entry, "cn", ""), + address=_first(entry, "mailinglistAddress", ""), + distribution_address=_first(entry, "mailinglistDistributionAddress", ""), + archive_address=_first(entry, "mailinglistArchiveAddress", ""), + description=_first(entry, "description"), + public=_bool(entry, "mailinglistPublic", True), + archive_enabled=_bool(entry, "mailinglistArchiveEnabled", True), + moderation_enabled=_bool(entry, "mailinglistModerationEnabled", False), + principal_id=_int(entry, "principalId"), + inbox_principal_id=_int(entry, "inboxPrincipalId"), + archive_principal_id=_int(entry, "archivePrincipalId"), + created_at=_datetime(entry, "createdAt"), + updated_at=_datetime(entry, "updatedAt"), + ) + + +class PendingSubscriber(LDAPBaseModel): + id: int = Field(0, description="Numeric pending subscriber ID") + mailing_list_id: int = Field(0, description="Parent mailing list ID") + email: EmailStr = Field("", description="Subscriber email") + token: str = Field("", description="Confirmation token") + action: str = Field("subscribe", description="subscribe or unsubscribe") + created_at: datetime | None = Field(None, description="Creation timestamp") + + @classmethod + def from_entry(cls, entry, mailing_list_id: int = 0): + return cls( + dn=str(entry["dn"]), + id=_int(entry, "mlidNumber", 0), + mailing_list_id=mailing_list_id, + email=_first(entry, "mail", ""), + token=_first(entry, "mailinglistToken", ""), + action=_first(entry, "mailinglistAction", "subscribe"), + created_at=_datetime(entry, "createdAt"), + ) + + +class ModerationMessage(LDAPBaseModel): + id: int = Field(0, description="Numeric moderation message ID") + mailing_list_id: int = Field(0, description="Parent mailing list ID") + message_id: str | None = Field(None, description="Original Message-ID header") + subject: str | None = Field(None, description="Message subject") + sender: EmailStr | None = Field(None, description="Message sender") + sent_at: datetime | None = Field(None, description="Original sent timestamp") + text_body: str | None = Field(None, description="Plain text body") + html_body: str | None = Field(None, description="HTML body") + status: str = Field("pending", description="pending/approved/rejected") + created_at: datetime | None = Field(None, description="Creation timestamp") + decided_at: datetime | None = Field(None, description="Decision timestamp") + + @classmethod + def from_entry(cls, entry, mailing_list_id: int = 0): + return cls( + dn=str(entry["dn"]), + id=_int(entry, "mlidNumber", 0), + mailing_list_id=mailing_list_id, + message_id=_first(entry, "mailinglistMessageId"), + subject=_first(entry, "mailinglistSubject"), + sender=_first(entry, "mailinglistSender"), + sent_at=_datetime(entry, "sentAt"), + text_body=_first(entry, "textBody"), + html_body=_first(entry, "htmlBody"), + status=_first(entry, "mailinglistStatus", "pending"), + created_at=_datetime(entry, "createdAt"), + decided_at=_datetime(entry, "decidedAt"), + ) + + +class MailingListCreate(BaseModel): + name: str + domain: str + description: str | None = None + public: bool = True + archive_enabled: bool = True + moderation_enabled: bool = False + + +class MailingListUpdate(BaseModel): + name: str | None = None + description: str | None = None + public: bool | None = None + archive_enabled: bool | None = None + moderation_enabled: bool | None = None diff --git a/freenit/models/mailinglist.py b/freenit/models/mailinglist.py new file mode 100644 index 0000000..7bd1cd7 --- /dev/null +++ b/freenit/models/mailinglist.py @@ -0,0 +1,10 @@ +from freenit.config import getConfig + +config = getConfig() +ml = config.get_model("mailinglist") + +MailingList = ml.MailingList +PendingSubscriber = ml.PendingSubscriber +ModerationMessage = ml.ModerationMessage +NotFoundError = ml.NotFoundError +IntegrityError = ml.IntegrityError diff --git a/freenit/models/sql/base.py b/freenit/models/sql/base.py index f2f352a..71c304b 100644 --- a/freenit/models/sql/base.py +++ b/freenit/models/sql/base.py @@ -1,8 +1,6 @@ from __future__ import annotations -from datetime import datetime from typing import ClassVar -from uuid import uuid4 import oxyde import pydantic @@ -168,68 +166,6 @@ class Meta: table_name = "theme" -class MailingList(OxydeBaseModel): - id: int | None = oxyde.Field(default=None, db_pk=True) - name: str = oxyde.Field(db_unique=True) - address: pydantic.EmailStr = oxyde.Field(db_unique=True) - distribution_address: pydantic.EmailStr = oxyde.Field(db_unique=True) - archive_address: pydantic.EmailStr = oxyde.Field(db_unique=True) - description: str | None = oxyde.Field(default=None) - public: bool = oxyde.Field(default=True) - archive_enabled: bool = oxyde.Field(default=True) - moderation_enabled: bool = oxyde.Field(default=False) - principal_id: int | None = oxyde.Field(default=None) - inbox_principal_id: int | None = oxyde.Field(default=None) - archive_principal_id: int | None = oxyde.Field(default=None) - created_at: datetime | None = oxyde.Field(default=None) - updated_at: datetime | None = oxyde.Field(default=None) - - class Meta: - is_table = True - table_name = "mailing_list" - - -class PendingSubscriber(OxydeBaseModel): - """Subscriptions/unsubscriptions awaiting email confirmation.""" - - id: int | None = oxyde.Field(default=None, db_pk=True) - mailing_list: MailingList | None = oxyde.Field( - default=None, db_fk="id", db_on_delete="CASCADE" - ) - email: pydantic.EmailStr = oxyde.Field() - token: str = oxyde.Field(default_factory=lambda: str(uuid4())) - action: str = oxyde.Field(default="subscribe") - created_at: datetime | None = oxyde.Field(default=None) - - class Meta: - is_table = True - table_name = "pending_subscriber" - unique_together = [("mailing_list_id", "email", "action")] - - -class ModerationMessage(OxydeBaseModel): - id: int | None = oxyde.Field(default=None, db_pk=True) - mailing_list: MailingList | None = oxyde.Field( - default=None, db_fk="id", db_on_delete="CASCADE" - ) - message_id: str | None = oxyde.Field(default=None) - subject: str | None = oxyde.Field(default=None) - sender: pydantic.EmailStr | None = oxyde.Field(default=None) - sent_at: datetime | None = oxyde.Field(default=None) - text_body: str | None = oxyde.Field(default=None) - html_body: str | None = oxyde.Field(default=None) - status: str = oxyde.Field(default="pending") - created_at: datetime | None = oxyde.Field(default=None) - decided_at: datetime | None = oxyde.Field(default=None) - - class Meta: - is_table = True - table_name = "moderation_message" - - User.model_rebuild() BaseRole.model_rebuild() UserRole.model_rebuild() -MailingList.model_rebuild() -PendingSubscriber.model_rebuild() -ModerationMessage.model_rebuild() diff --git a/freenit/models/sql/mailinglist.py b/freenit/models/sql/mailinglist.py new file mode 100644 index 0000000..506bacb --- /dev/null +++ b/freenit/models/sql/mailinglist.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import uuid4 + +import oxyde +import pydantic + +from freenit.models.sql.base import OxydeBaseModel + +NotFoundError = oxyde.NotFoundError +IntegrityError = oxyde.IntegrityError + + +class MailingList(OxydeBaseModel): + id: int | None = oxyde.Field(default=None, db_pk=True) + name: str = oxyde.Field(db_unique=True) + address: pydantic.EmailStr = oxyde.Field(db_unique=True) + distribution_address: pydantic.EmailStr = oxyde.Field(db_unique=True) + archive_address: pydantic.EmailStr = oxyde.Field(db_unique=True) + description: str | None = oxyde.Field(default=None) + public: bool = oxyde.Field(default=True) + archive_enabled: bool = oxyde.Field(default=True) + moderation_enabled: bool = oxyde.Field(default=False) + principal_id: int | None = oxyde.Field(default=None) + inbox_principal_id: int | None = oxyde.Field(default=None) + archive_principal_id: int | None = oxyde.Field(default=None) + created_at: datetime | None = oxyde.Field(default=None) + updated_at: datetime | None = oxyde.Field(default=None) + + class Meta: + is_table = True + table_name = "mailing_list" + + +class PendingSubscriber(OxydeBaseModel): + """Subscriptions/unsubscriptions awaiting email confirmation.""" + + id: int | None = oxyde.Field(default=None, db_pk=True) + mailing_list: MailingList | None = oxyde.Field( + default=None, db_fk="id", db_on_delete="CASCADE" + ) + email: pydantic.EmailStr = oxyde.Field() + token: str = oxyde.Field(default_factory=lambda: str(uuid4())) + action: str = oxyde.Field(default="subscribe") + created_at: datetime | None = oxyde.Field(default=None) + + class Meta: + is_table = True + table_name = "pending_subscriber" + unique_together = [("mailing_list_id", "email", "action")] + + +class ModerationMessage(OxydeBaseModel): + id: int | None = oxyde.Field(default=None, db_pk=True) + mailing_list: MailingList | None = oxyde.Field( + default=None, db_fk="id", db_on_delete="CASCADE" + ) + message_id: str | None = oxyde.Field(default=None) + subject: str | None = oxyde.Field(default=None) + sender: pydantic.EmailStr | None = oxyde.Field(default=None) + sent_at: datetime | None = oxyde.Field(default=None) + text_body: str | None = oxyde.Field(default=None) + html_body: str | None = oxyde.Field(default=None) + status: str = oxyde.Field(default="pending") + created_at: datetime | None = oxyde.Field(default=None) + decided_at: datetime | None = oxyde.Field(default=None) + + class Meta: + is_table = True + table_name = "moderation_message" + + +MailingList.model_rebuild() +PendingSubscriber.model_rebuild() +ModerationMessage.model_rebuild() diff --git a/freenit/stalwart.py b/freenit/stalwart.py index 376b4ed..7d0a7b0 100644 --- a/freenit/stalwart.py +++ b/freenit/stalwart.py @@ -264,3 +264,9 @@ async def list_principals(types: str = "list", page: int = 1, limit: int = 100) log.error("Failed to list principals: %s %s", resp.status_code, resp.text[:500]) raise RuntimeError(f"Stalwart list principals failed: {resp.text}") return resp.json().get("data", {}).get("items", []) + + +async def list_domains() -> list[str]: + items = await list_principals(types="domain", limit=1000) + domains = [item["name"] for item in items if item.get("name")] + return sorted(domains) diff --git a/ldap/README.md b/ldap/README.md index 5969ad6..2601180 100644 --- a/ldap/README.md +++ b/ldap/README.md @@ -1,32 +1,38 @@ -# OMEMO LDAP Schema +# LDAP Schemas -OpenLDAP schema extension for storing encrypted OMEMO device bundles on LDAP user entries. +This directory contains optional OpenLDAP schema extensions used by Freenit. ## Files -- `omemo.schema` — traditional schema file for `slapd.conf` setups -- `omemo.ldif` — OLC/LDIF format for modern `cn=config` setups +- `omemo.schema` / `omemo.ldif` — OMEMO device bundle storage on user entries. +- `mailinglist.schema` / `mailinglist.ldif` — mailing list metadata storage in LDAP. -## Loading the schema +## Loading the schemas -### Traditional slapd.conf - -Add to `slapd.conf`: +### Traditional `slapd.conf` ``` include /path/to/omemo.schema +include /path/to/mailinglist.schema ``` Then restart `slapd`. -### Modern cn=config (OLC) +### Modern `cn=config` (OLC) ```bash ldapadd -Y EXTERNAL -H ldapi:/// -f omemo.ldif +ldapadd -Y EXTERNAL -H ldapi:/// -f mailinglist.ldif ``` No restart required. +--- + +# OMEMO LDAP Schema + +OpenLDAP schema extension for storing encrypted OMEMO device bundles on LDAP user entries. + ## Adding omemoPerson to existing accounts `omemoPerson` is an **auxiliary** object class, so it can be added to existing entries without recreating them. @@ -83,3 +89,58 @@ ldap = LDAP( userOmemoAttr="omemoBundle", ) ``` + +--- + +# Mailing List LDAP Schema + +OpenLDAP schema extension for storing Freenit mailing list metadata in LDAP. + +This includes: + +- `freenitMailingList` — mailing list entries +- `freenitPendingSubscriber` — pending subscribe/unsubscribe tokens +- `freenitModerationMessage` — messages awaiting moderation +- `freenitMailingListIdNext` — global numeric ID counter + +## Preparing the directory + +Create the mailing list container and the ID counter before using the LDAP backend: + +```bash +ldapadd -x -D "cn=admin,dc=ldap" -W <