Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,58 @@ and this project adheres to
- The DSFR theme jar `dsfr-2.3.4.jar`. A realm that selected the `DSFR` theme
must select `lst-dsfr` once after the upgrade

### Fixed

- An inbound or imported message whose date has an offset beyond ±15:59, or
is outside the years 1-9999 once in UTC, is delivered instead of being
retried until abandoned #855
- A message date in the future is stored as the delivery time, so it no
longer pins a thread at the top of the list or keeps later messages marked
as read. An import uses the date of its topmost `Received:` header instead,
then the import time. Reply quotes never show a date other than the stored
one
- The `read_at` and `starred_at` of the flag API are refused when out of range
and clamped to now (`read_at`: to the thread's last message if later). A
year-10000 value made the thread fail to load for every mailbox sharing it
- "Mark as read" no longer leaves the latest mail unread when the device clock
is behind the server's
- A calendar invite with an invalid date no longer breaks the message view;
an invite that still fails to render is shown as a plain attachment.
All-day invites show their own days west of UTC, not the day before
- The calendar conflicts check skips a stored event whose date cannot be
read instead of failing, and an invite added to the calendar drops dates
that cannot be read, repeated DTSTART and huge sub-hourly recurrences
- A webhook's `mark_read` now marks the new message read in a thread read
before
- A delivery retry keeps the message's first `sent_at`: the manual retry age
limit no longer restarts at each attempt
- Splitting a thread whose messages share a date (imports) cuts in display
order and never moves the first message
- Autoreply weekly intervals refuse `24:00`, which read as `00:00` and made
the interval cover the whole week; a stored one ends with the day
- `compose_email` writes a sub-minute offset in UTC instead of an empty
`Date:`, and refuses a year before 100, which read back as a two-digit year
- `backfill_postmark` and `backfill_thread_snippets` read a bare `--before`
date as midnight UTC and report an impossible date as an error
- Startup check `core.E006` refuses duration settings over 100 years (e.g.
`TRASHBIN_CUTOFF_DAYS=999999`), which made the task using them fail
- Silent login and the "empty trash" progress no longer stay stuck when the
device clock was set back
- The PST import keeps a message whose `Date:` cannot be parsed, dated by the
MAPI timestamps, which also replace a `Date:` in the future
- Autoreply date ranges are compared as instants, which was wrong in the hour
repeated when DST ends, and a naive bound no longer fails validation. A
stored invalid `timezone` falls back to UTC
- The calendar conflicts API answers `400` to an out-of-range or non-string
`start` / `end`
- An API key whose `expires_at` cannot be read (anything but a date, absent,
`null` or `""`) is refused as expired instead of never expiring, and
`expires_at` is validated on write
- Migration `0038` clamps the dates already stored to the range above

_⚠️ Run `search_reindex --all --from-date <upgrade date>` after the
upgrade: the threads it changes are marked as updated._

### Security

- Keycloak 26.8.2 fixes more than 30 CVEs, among them CVE-2026-117691 (SAML
Expand Down
11 changes: 7 additions & 4 deletions src/backend/core/api/authentication.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@
from django.core.exceptions import ValidationError as DjangoValidationError
from django.db.models import Q
from django.utils import timezone
from django.utils.dateparse import parse_datetime

from rest_framework.authentication import BaseAuthentication
from rest_framework.exceptions import AuthenticationFailed

from core import models
from core.enums import ChannelScopeLevel, ChannelTypes
from core.utils import parse_utc_datetime


class ChannelApiKeyAuthentication(BaseAuthentication):
Expand Down Expand Up @@ -82,9 +82,12 @@ def authenticate(self, request):
raise AuthenticationFailed("Invalid channel or API key.")

expires_at_raw = (channel.settings or {}).get("expires_at")
if expires_at_raw:
expires_at = parse_datetime(expires_at_raw)
if expires_at is not None and expires_at < timezone.now():
# Absent, null or "": no expiry. Anything else unreadable (a typo,
# 0, false) counts as expired: failing open would make it a key that
# never expires.
if expires_at_raw not in (None, ""):
expires_at = parse_utc_datetime(expires_at_raw)
if expires_at is None or expires_at < timezone.now():
raise AuthenticationFailed("API key has expired.")

# Throttled update of last_used_at for monitoring (5 min window).
Expand Down
9 changes: 9 additions & 0 deletions src/backend/core/api/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from core.services.importer.channel import merged_state
from core.services.ssrf import SSRFValidationError, validate_hostname
from core.services.trashbin import TRASHBIN_SCOPES
from core.utils import parse_utc_datetime

logger = getLogger(__name__)

Expand Down Expand Up @@ -2720,6 +2721,14 @@ def validate_settings(self, value):
if not value:
return value

# Read by ChannelApiKeyAuthentication: absent, null or "" for no
# expiry, anything else unreadable counts as expired.
expires_at = value.get("expires_at")
if expires_at not in (None, "") and parse_utc_datetime(expires_at) is None:
raise serializers.ValidationError(
{"expires_at": "Must be a valid ISO 8601 datetime."}
)

tags = value.get("tags", [])
if not tags:
return value
Expand Down
17 changes: 13 additions & 4 deletions src/backend/core/api/viewsets/calendar.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import logging
from datetime import datetime
from datetime import timezone as dt_timezone

from django.conf import settings
from django.shortcuts import get_object_or_404
Expand Down Expand Up @@ -323,10 +324,8 @@ def post(self, request, mailbox_id): # pylint: disable=unused-argument
)

try:
if isinstance(start, str):
start = datetime.fromisoformat(start)
if isinstance(end, str):
end = datetime.fromisoformat(end)
start = datetime.fromisoformat(start)
end = datetime.fromisoformat(end)
except (ValueError, TypeError):
return Response(
{"detail": "start and end must be valid ISO 8601 datetimes."},
Expand All @@ -342,6 +341,16 @@ def post(self, request, mailbox_id): # pylint: disable=unused-argument
status=status.HTTP_400_BAD_REQUEST,
)

# Out of datetime's range once in UTC (e.g. 9999-12-31T23:59:59-05:00).
try:
start = start.astimezone(dt_timezone.utc)
end = end.astimezone(dt_timezone.utc)
except (OverflowError, ValueError):
return Response(
{"detail": "start and end must be valid ISO 8601 datetimes."},
status=status.HTTP_400_BAD_REQUEST,
)

if end <= start:
return Response(
{"detail": "end must be after start."},
Expand Down
50 changes: 42 additions & 8 deletions src/backend/core/api/viewsets/flag.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@
import logging

from django.db import transaction
from django.db.models import DateTimeField, OuterRef, Subquery, Value
from django.db.models.functions import Greatest, Least
from django.utils import timezone
from django.utils.dateparse import parse_datetime

import rest_framework as drf
from drf_spectacular.utils import (
Expand All @@ -18,12 +19,41 @@
from core import models
from core.services.search.tasks import update_threads_mailbox_flags_task
from core.services.trashbin import trashbin_timestamp
from core.utils import parse_utc_datetime

from .. import permissions

logger = logging.getLogger(__name__)


def _parse_starred_at(value):
"""Parse a client ``starred_at``, clamped to now, or None if unusable."""
parsed = parse_utc_datetime(value)
if parsed is None:
return None
return min(parsed, timezone.now())


def _clamped_read_at(read_at):
"""``read_at`` as stored, no later than now or the thread's last message.

A client clock running ahead (or a crafted value) would otherwise keep
the mail that arrives later shown as read, for every user of the
mailbox. The thread's ``messaged_at`` bounds it too: it may be a little
ahead of this host's clock when the worker that stored the message runs
ahead, and reading up to it must still mark that message read.
"""
last_message = models.Thread.objects.filter(pk=OuterRef("thread_id")).values(
"messaged_at"
)[:1]
# GREATEST ignores NULL in PostgreSQL: a thread without messages gives now.
return Least(
Value(read_at),
Greatest(Value(timezone.now()), Subquery(last_message)),
output_field=DateTimeField(),
)


def _flag_update_data(flag, value, current_time):
"""Return the message fields a trashed/archived/spam toggle writes.

Expand Down Expand Up @@ -377,11 +407,13 @@ def _handle_unread_flag(
read_at = request.data.get("read_at")

# read_at must be None or a valid datetime string
if read_at is not None and parse_datetime(str(read_at)) is None:
return drf.response.Response(
{"detail": "read_at must be a valid ISO 8601 datetime."},
status=drf.status.HTTP_400_BAD_REQUEST,
)
if read_at is not None:
read_at = parse_utc_datetime(read_at)
if read_at is None:
return drf.response.Response(
{"detail": "read_at must be a valid ISO 8601 datetime."},
status=drf.status.HTTP_400_BAD_REQUEST,
)

thread_qs = models.Thread.objects.filter(
id__in=thread_ids,
Expand All @@ -396,7 +428,9 @@ def _handle_unread_flag(
thread_ids_to_sync = [
str(tid) for tid in accesses.values_list("thread_id", flat=True)
]
updated_count = accesses.update(read_at=read_at)
updated_count = accesses.update(
read_at=None if read_at is None else _clamped_read_at(read_at)
)

if thread_ids_to_sync:
transaction.on_commit(
Expand Down Expand Up @@ -428,7 +462,7 @@ def _handle_starred_flag(
if value:
# Use provided timestamp or default to now
if starred_at is not None:
parsed = parse_datetime(str(starred_at))
parsed = _parse_starred_at(starred_at)
if parsed is None:
return drf.response.Response(
{"detail": "starred_at must be a valid ISO 8601 datetime."},
Expand Down
18 changes: 13 additions & 5 deletions src/backend/core/api/viewsets/thread.py
Original file line number Diff line number Diff line change
Expand Up @@ -969,8 +969,10 @@ def split(self, request, pk=None): # pylint: disable=unused-argument
status=status.HTTP_400_BAD_REQUEST,
)

# Get all messages in the thread ordered chronologically
all_messages = old_thread.messages.order_by("created_at")
# Get all messages in the thread ordered chronologically. ``id`` breaks
# ties (imports copy ``Date:``, to the second), as in
# ``Thread.update_stats``, so the cut below matches this order.
all_messages = old_thread.messages.order_by("created_at", "id")
total_count = all_messages.count()

if total_count <= 1:
Expand All @@ -987,7 +989,10 @@ def split(self, request, pk=None): # pylint: disable=unused-argument
)

# Determine messages to move (split_message and all later)
messages_to_move = all_messages.filter(created_at__gte=split_message.created_at)
messages_to_move = all_messages.filter(
Q(created_at__gt=split_message.created_at)
| Q(created_at=split_message.created_at, id__gte=split_message.id)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
)

with transaction.atomic():
new_subject = split_message.subject or old_thread.subject
Expand Down Expand Up @@ -1019,9 +1024,12 @@ def split(self, request, pk=None): # pylint: disable=unused-argument
# Move messages
messages_to_move.update(thread=new_thread)

# Fix cross-thread parent references
# Fix cross-thread parent references, both ways: with dates that
# tie (imports) or out of order, a kept message can have its parent
# among the moved ones.
models.Message.objects.filter(
thread=new_thread, parent__thread=old_thread
Q(thread=new_thread, parent__thread=old_thread)
| Q(thread=old_thread, parent__thread=new_thread)
).update(parent=None)

# Move ThreadEvents that belong chronologically to the new thread.
Expand Down
57 changes: 57 additions & 0 deletions src/backend/core/checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,3 +183,60 @@ def check_blob_lifecycle_redis(app_configs, **kwargs):
id="core.W004",
)
]


# Durations subtracted from now (``now - timedelta(...)``): past a few
# thousand centuries that raises OverflowError, in every run of the task
# using it. 100 years is beyond any meaningful value.
_MAX_DURATION_DAYS = 36500
_SECONDS_PER_DAY = 86400


@register()
def check_duration_settings(app_configs, **kwargs):
"""Refuse duration settings too large to subtract from now.

A value meant as "never" (``TRASHBIN_CUTOFF_DAYS=999999``) would make the
task using it fail on every run instead.
"""
durations = [
# (name, value, units per day)
(
"MESSAGES_MANUAL_RETRY_MAX_AGE",
settings.MESSAGES_MANUAL_RETRY_MAX_AGE,
_SECONDS_PER_DAY,
),
(
"MESSAGES_INBOUND_DEFERRAL_MAX_AGE",
settings.MESSAGES_INBOUND_DEFERRAL_MAX_AGE,
_SECONDS_PER_DAY,
),
(
"MESSAGES_IMPORT_STALL_TIMEOUT",
settings.MESSAGES_IMPORT_STALL_TIMEOUT,
_SECONDS_PER_DAY,
),
(
"MESSAGES_IMPORT_IMAP_POLL_INTERVAL",
settings.MESSAGES_IMPORT_IMAP_POLL_INTERVAL,
_SECONDS_PER_DAY,
),
(
"MESSAGES_BLOBS_OFFLOAD_DELAY",
settings.MESSAGES_BLOBS_OFFLOAD_DELAY,
_SECONDS_PER_DAY,
),
("TRASHBIN_CUTOFF_DAYS", settings.TRASHBIN_CUTOFF_DAYS, 1),
("PUSH_PREVIEW_MAX_IDLE_DAYS", settings.PUSH_PREVIEW_MAX_IDLE_DAYS, 1),
]
errors = []
for name, value, units_per_day in durations:
if value > _MAX_DURATION_DAYS * units_per_day:
errors.append(
Error(
f"{name}={value} is more than {_MAX_DURATION_DAYS} days.",
hint=f"Set {name} to at most {_MAX_DURATION_DAYS * units_per_day}.",
id="core.E006",
)
)
return errors
20 changes: 7 additions & 13 deletions src/backend/core/management/commands/backfill_postmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,13 @@
python manage.py backfill_postmark --before 2026-07-01 --dry-run
"""

import datetime
import logging

from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q
from django.utils.dateparse import parse_datetime

from core import models
from core.utils import parse_utc_datetime

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -82,18 +81,13 @@ def handle(self, *args, **options):

base_qs = models.Message.objects.filter(postmark__isnull=True)
if options["before"]:
# ISO date or datetime string → aware datetime.
cutoff = parse_datetime(options["before"])
# ISO date or datetime string → aware datetime; a bare date or no
# offset is read as UTC.
cutoff = parse_utc_datetime(options["before"])
if cutoff is None:
try:
# Bare date → midnight UTC.
cutoff = datetime.datetime.fromisoformat(options["before"]).replace(
tzinfo=datetime.UTC
)
except ValueError as exc:
raise CommandError(
f"--before {options['before']} is not a valid ISO date/datetime"
) from exc
raise CommandError(
f"--before {options['before']} is not a valid ISO date/datetime"
)
base_qs = base_qs.filter(created_at__lt=cutoff)

scanned = 0
Expand Down
Loading
Loading