feat: add board group models and update related interfaces

backend/app/services/agent_provisioning.py

@@ -479,6 +479,78 @@ async def _patch_gateway_agent_list(
     await openclaw_call("config.patch", params, config=config)
 
 
+async def patch_gateway_agent_heartbeats(
+    gateway: Gateway,
+    *,
+    entries: list[tuple[str, str, dict[str, Any]]],
+) -> None:
+    """Patch multiple agent heartbeat configs in a single gateway config.patch call.
+
+    Each entry is (agent_id, workspace_path, heartbeat_dict).
+    """
+    if not gateway.url:
+        raise OpenClawGatewayError("Gateway url is required")
+    config = GatewayClientConfig(url=gateway.url, token=gateway.token)
+    cfg = await openclaw_call("config.get", config=config)
+    if not isinstance(cfg, dict):
+        raise OpenClawGatewayError("config.get returned invalid payload")
+    base_hash = cfg.get("hash")
+    data = cfg.get("config") or cfg.get("parsed") or {}
+    if not isinstance(data, dict):
+        raise OpenClawGatewayError("config.get returned invalid config")
+    agents_section = data.get("agents") or {}
+    lst = agents_section.get("list") or []
+    if not isinstance(lst, list):
+        raise OpenClawGatewayError("config agents.list is not a list")
+
+    entry_by_id: dict[str, tuple[str, dict[str, Any]]] = {
+        agent_id: (workspace_path, heartbeat) for agent_id, workspace_path, heartbeat in entries
+    }
+
+    updated_ids: set[str] = set()
+    new_list: list[dict[str, Any]] = []
+    for raw_entry in lst:
+        if not isinstance(raw_entry, dict):
+            new_list.append(raw_entry)
+            continue
+        agent_id = raw_entry.get("id")
+        if not isinstance(agent_id, str) or agent_id not in entry_by_id:
+            new_list.append(raw_entry)
+            continue
+        workspace_path, heartbeat = entry_by_id[agent_id]
+        new_entry = dict(raw_entry)
+        new_entry["workspace"] = workspace_path
+        new_entry["heartbeat"] = heartbeat
+        new_list.append(new_entry)
+        updated_ids.add(agent_id)
+
+    for agent_id, (workspace_path, heartbeat) in entry_by_id.items():
+        if agent_id in updated_ids:
+            continue
+        new_list.append({"id": agent_id, "workspace": workspace_path, "heartbeat": heartbeat})
+
+    patch = {"agents": {"list": new_list}}
+    params = {"raw": json.dumps(patch)}
+    if base_hash:
+        params["baseHash"] = base_hash
+    await openclaw_call("config.patch", params, config=config)
+
+
+async def sync_gateway_agent_heartbeats(gateway: Gateway, agents: list[Agent]) -> None:
+    """Sync current Agent.heartbeat_config values to the gateway config."""
+    if not gateway.workspace_root:
+        raise OpenClawGatewayError("gateway workspace_root is required")
+    entries: list[tuple[str, str, dict[str, Any]]] = []
+    for agent in agents:
+        agent_id = _agent_key(agent)
+        workspace_path = _workspace_path(agent, gateway.workspace_root)
+        heartbeat = _heartbeat_config(agent)
+        entries.append((agent_id, workspace_path, heartbeat))
+    if not entries:
+        return
+    await patch_gateway_agent_heartbeats(gateway, entries=entries)
+
+
 async def _remove_gateway_agent_list(
     agent_id: str,
     config: GatewayClientConfig,

backend/app/services/board_group_snapshot.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Any
+from uuid import UUID
+
+from sqlalchemy import case, func
+from sqlmodel import col, select
+from sqlmodel.ext.asyncio.session import AsyncSession
+
+from app.models.agents import Agent
+from app.models.board_groups import BoardGroup
+from app.models.boards import Board
+from app.models.tasks import Task
+from app.schemas.board_groups import BoardGroupRead
+from app.schemas.boards import BoardRead
+from app.schemas.view_models import (
+    BoardGroupBoardSnapshot,
+    BoardGroupSnapshot,
+    BoardGroupTaskSummary,
+)
+
+_STATUS_ORDER = {"in_progress": 0, "review": 1, "inbox": 2, "done": 3}
+_PRIORITY_ORDER = {"high": 0, "medium": 1, "low": 2}
+
+
+def _status_weight_expr() -> Any:
+    whens = [(col(Task.status) == key, weight) for key, weight in _STATUS_ORDER.items()]
+    return case(*whens, else_=99)
+
+
+def _priority_weight_expr() -> Any:
+    whens = [(col(Task.priority) == key, weight) for key, weight in _PRIORITY_ORDER.items()]
+    return case(*whens, else_=99)
+
+
+async def build_group_snapshot(
+    session: AsyncSession,
+    *,
+    group: BoardGroup,
+    exclude_board_id: UUID | None = None,
+    include_done: bool = False,
+    per_board_task_limit: int = 5,
+) -> BoardGroupSnapshot:
+    statement = select(Board).where(col(Board.board_group_id) == group.id)
+    if exclude_board_id is not None:
+        statement = statement.where(col(Board.id) != exclude_board_id)
+    boards = list(await session.exec(statement.order_by(func.lower(col(Board.name)).asc())))
+    if not boards:
+        return BoardGroupSnapshot(group=BoardGroupRead.model_validate(group, from_attributes=True))
+
+    boards_by_id = {board.id: board for board in boards}
+    board_ids = list(boards_by_id.keys())
+
+    task_counts: dict[UUID, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+    for board_id, status_value, total in list(
+        await session.exec(
+            select(col(Task.board_id), col(Task.status), func.count(col(Task.id)))
+            .where(col(Task.board_id).in_(board_ids))
+            .group_by(col(Task.board_id), col(Task.status))
+        )
+    ):
+        if board_id is None:
+            continue
+        task_counts[board_id][str(status_value)] = int(total or 0)
+
+    task_statement = select(Task).where(col(Task.board_id).in_(board_ids))
+    if not include_done:
+        task_statement = task_statement.where(col(Task.status) != "done")
+    task_statement = task_statement.order_by(
+        col(Task.board_id).asc(),
+        _status_weight_expr().asc(),
+        _priority_weight_expr().asc(),
+        col(Task.updated_at).desc(),
+        col(Task.created_at).desc(),
+    )
+    tasks = list(await session.exec(task_statement))
+
+    assigned_ids = {task.assigned_agent_id for task in tasks if task.assigned_agent_id is not None}
+    agent_name_by_id: dict[UUID, str] = {}
+    if assigned_ids:
+        for agent_id, name in list(
+            await session.exec(
+                select(col(Agent.id), col(Agent.name)).where(col(Agent.id).in_(assigned_ids))
+            )
+        ):
+            agent_name_by_id[agent_id] = name
+
+    tasks_by_board: dict[UUID, list[BoardGroupTaskSummary]] = defaultdict(list)
+    if per_board_task_limit > 0:
+        for task in tasks:
+            if task.board_id is None:
+                continue
+            current = tasks_by_board[task.board_id]
+            if len(current) >= per_board_task_limit:
+                continue
+            board = boards_by_id.get(task.board_id)
+            if board is None:
+                continue
+            current.append(
+                BoardGroupTaskSummary(
+                    id=task.id,
+                    board_id=task.board_id,
+                    board_name=board.name,
+                    title=task.title,
+                    status=task.status,
+                    priority=task.priority,
+                    assigned_agent_id=task.assigned_agent_id,
+                    assignee=(
+                        agent_name_by_id.get(task.assigned_agent_id)
+                        if task.assigned_agent_id is not None
+                        else None
+                    ),
+                    due_at=task.due_at,
+                    in_progress_at=task.in_progress_at,
+                    created_at=task.created_at,
+                    updated_at=task.updated_at,
+                )
+            )
+
+    snapshots: list[BoardGroupBoardSnapshot] = []
+    for board in boards:
+        board_read = BoardRead.model_validate(board, from_attributes=True)
+        counts = dict(task_counts.get(board.id, {}))
+        snapshots.append(
+            BoardGroupBoardSnapshot(
+                board=board_read,
+                task_counts=counts,
+                tasks=tasks_by_board.get(board.id, []),
+            )
+        )
+
+    return BoardGroupSnapshot(
+        group=BoardGroupRead.model_validate(group, from_attributes=True),
+        boards=snapshots,
+    )
+
+
+async def build_board_group_snapshot(
+    session: AsyncSession,
+    *,
+    board: Board,
+    include_self: bool = False,
+    include_done: bool = False,
+    per_board_task_limit: int = 5,
+) -> BoardGroupSnapshot:
+    if not board.board_group_id:
+        return BoardGroupSnapshot(group=None, boards=[])
+    group = await session.get(BoardGroup, board.board_group_id)
+    if group is None:
+        return BoardGroupSnapshot(group=None, boards=[])
+    return await build_group_snapshot(
+        session,
+        group=group,
+        exclude_board_id=None if include_self else board.id,
+        include_done=include_done,
+        per_board_task_limit=per_board_task_limit,
+    )

backend/app/services/template_sync.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
 
 import asyncio
+import random
 import re
 from collections.abc import Awaitable, Callable
 from typing import TypeVar
 from uuid import UUID, uuid4
 
+from sqlalchemy import func
 from sqlmodel import col, select
 from sqlmodel.ext.asyncio.session import AsyncSession
 
@@ -14,6 +16,7 @@ from app.core.time import utcnow
 from app.integrations.openclaw_gateway import GatewayConfig as GatewayClientConfig
 from app.integrations.openclaw_gateway import OpenClawGatewayError, openclaw_call
 from app.models.agents import Agent
+from app.models.board_memory import BoardMemory
 from app.models.boards import Board
 from app.models.gateways import Gateway
 from app.models.users import User
@@ -38,10 +41,22 @@ def _is_transient_gateway_error(exc: Exception) -> bool:
         return False
     if "unsupported file" in message:
         return False
+    if "connect call failed" in message or "connection refused" in message:
+        return True
+    if "errno 111" in message or "econnrefused" in message:
+        return True
+    if "did not receive a valid http response" in message:
+        return True
+    if "no route to host" in message or "network is unreachable" in message:
+        return True
+    if "host is down" in message or "name or service not known" in message:
+        return True
     if "received 1012" in message or "service restart" in message:
         return True
     if "http 503" in message or ("503" in message and "websocket" in message):
         return True
+    if "http 502" in message or "http 504" in message:
+        return True
     if "temporar" in message:
         return True
     if "timeout" in message or "timed out" in message:
@@ -51,20 +66,58 @@ def _is_transient_gateway_error(exc: Exception) -> bool:
     return False
 
 
+class _GatewayBackoff:
+    def __init__(
+        self,
+        *,
+        timeout_s: float = 10 * 60,
+        base_delay_s: float = 0.75,
+        max_delay_s: float = 30.0,
+        jitter: float = 0.2,
+    ) -> None:
+        self._timeout_s = timeout_s
+        self._base_delay_s = base_delay_s
+        self._max_delay_s = max_delay_s
+        self._jitter = jitter
+        self._delay_s = base_delay_s
+
+    def reset(self) -> None:
+        self._delay_s = self._base_delay_s
+
+    async def run(self, fn: Callable[[], Awaitable[T]]) -> T:
+        # Use per-call deadlines so long-running syncs can still tolerate a later
+        # gateway restart without having an already-expired retry window.
+        deadline_s = asyncio.get_running_loop().time() + self._timeout_s
+        while True:
+            try:
+                value = await fn()
+                self.reset()
+                return value
+            except Exception as exc:
+                if not _is_transient_gateway_error(exc):
+                    raise
+                now = asyncio.get_running_loop().time()
+                remaining = deadline_s - now
+                if remaining <= 0:
+                    raise TimeoutError(
+                        "Gateway unreachable after 10 minutes (template sync timeout). "
+                        f"Last error: {exc}"
+                    ) from exc
+
+                sleep_s = min(self._delay_s, remaining)
+                if self._jitter:
+                    sleep_s *= 1.0 + random.uniform(-self._jitter, self._jitter)
+                sleep_s = max(0.0, min(sleep_s, remaining))
+                await asyncio.sleep(sleep_s)
+                self._delay_s = min(self._delay_s * 2.0, self._max_delay_s)
+
+
 async def _with_gateway_retry(
     fn: Callable[[], Awaitable[T]],
     *,
-    attempts: int = 3,
-    base_delay_s: float = 0.75,
+    backoff: _GatewayBackoff,
 ) -> T:
-    for attempt in range(attempts):
-        try:
-            return await fn()
-        except Exception as exc:
-            if attempt >= attempts - 1 or not _is_transient_gateway_error(exc):
-                raise
-            await asyncio.sleep(base_delay_s * (2**attempt))
-    raise AssertionError("unreachable")
+    return await backoff.run(fn)
 
 
 def _agent_id_from_session_key(session_key: str | None) -> str | None:
@@ -137,13 +190,18 @@ async def _get_agent_file(
     agent_gateway_id: str,
     name: str,
     config: GatewayClientConfig,
+    backoff: _GatewayBackoff | None = None,
 ) -> str | None:
     try:
-        payload = await openclaw_call(
-            "agents.files.get",
-            {"agentId": agent_gateway_id, "name": name},
-            config=config,
-        )
+
+        async def _do_get() -> object:
+            return await openclaw_call(
+                "agents.files.get",
+                {"agentId": agent_gateway_id, "name": name},
+                config=config,
+            )
+
+        payload = await (backoff.run(_do_get) if backoff else _do_get())
     except OpenClawGatewayError:
         return None
     if isinstance(payload, str):
@@ -167,8 +225,14 @@ async def _get_existing_auth_token(
     *,
     agent_gateway_id: str,
     config: GatewayClientConfig,
+    backoff: _GatewayBackoff | None = None,
 ) -> str | None:
-    tools = await _get_agent_file(agent_gateway_id=agent_gateway_id, name="TOOLS.md", config=config)
+    tools = await _get_agent_file(
+        agent_gateway_id=agent_gateway_id,
+        name="TOOLS.md",
+        config=config,
+        backoff=backoff,
+    )
     if not tools:
         return None
     values = _parse_tools_md(tools)
@@ -183,32 +247,45 @@ async def _gateway_default_agent_id(
     config: GatewayClientConfig,
     *,
     fallback_session_key: str | None = None,
+    backoff: _GatewayBackoff | None = None,
 ) -> str | None:
-    last_error: OpenClawGatewayError | None = None
-    # Gateways may reject WS connects transiently under load (HTTP 503).
-    for attempt in range(3):
-        try:
-            payload = await openclaw_call("agents.list", config=config)
-            agent_id = _extract_agent_id(payload)
-            if agent_id:
-                return agent_id
-            break
-        except OpenClawGatewayError as exc:
-            last_error = exc
-            message = str(exc).lower()
-            if (
-                "503" not in message
-                and "temporar" not in message
-                and "rejected" not in message
-                and "timeout" not in message
-            ):
-                break
-            await asyncio.sleep(0.5 * (2**attempt))
+    try:
 
-    _ = last_error
+        async def _do_list() -> object:
+            return await openclaw_call("agents.list", config=config)
+
+        payload = await (backoff.run(_do_list) if backoff else _do_list())
+        agent_id = _extract_agent_id(payload)
+        if agent_id:
+            return agent_id
+    except OpenClawGatewayError:
+        pass
     return _agent_id_from_session_key(fallback_session_key)
 
 
+async def _paused_board_ids(session: AsyncSession, board_ids: list[UUID]) -> set[UUID]:
+    if not board_ids:
+        return set()
+
+    commands = {"/pause", "/resume"}
+    statement = (
+        select(BoardMemory.board_id, BoardMemory.content)
+        .where(col(BoardMemory.board_id).in_(board_ids))
+        .where(col(BoardMemory.is_chat).is_(True))
+        .where(func.lower(func.trim(col(BoardMemory.content))).in_(commands))
+        .order_by(col(BoardMemory.board_id), col(BoardMemory.created_at).desc())
+        # Postgres: DISTINCT ON (board_id) to get latest command per board.
+        .distinct(col(BoardMemory.board_id))
+    )
+
+    paused: set[UUID] = set()
+    for board_id, content in await session.exec(statement):
+        cmd = (content or "").strip().lower()
+        if cmd == "/pause":
+            paused.add(board_id)
+    return paused
+
+
 async def sync_gateway_templates(
     session: AsyncSession,
     gateway: Gateway,
@@ -235,6 +312,21 @@ async def sync_gateway_templates(
         return result
 
     client_config = GatewayClientConfig(url=gateway.url, token=gateway.token)
+    backoff = _GatewayBackoff(timeout_s=10 * 60)
+
+    # First, wait for the gateway to be reachable (e.g. while it is restarting).
+    try:
+
+        async def _do_ping() -> object:
+            return await openclaw_call("agents.list", config=client_config)
+
+        await backoff.run(_do_ping)
+    except TimeoutError as exc:
+        result.errors.append(GatewayTemplatesSyncError(message=str(exc)))
+        return result
+    except OpenClawGatewayError as exc:
+        result.errors.append(GatewayTemplatesSyncError(message=str(exc)))
+        return result
 
     boards = list(await session.exec(select(Board).where(col(Board.gateway_id) == gateway.id)))
     boards_by_id = {board.id: board for board in boards}
@@ -250,6 +342,8 @@ async def sync_gateway_templates(
             return result
         boards_by_id = {board_id: board}
 
+    paused_board_ids = await _paused_board_ids(session, list(boards_by_id.keys()))
+
     if boards_by_id:
         agents = list(
             await session.exec(
@@ -275,10 +369,27 @@ async def sync_gateway_templates(
             )
             continue
 
+        if board.id in paused_board_ids:
+            result.agents_skipped += 1
+            continue
+
         agent_gateway_id = _gateway_agent_id(agent)
-        auth_token = await _get_existing_auth_token(
-            agent_gateway_id=agent_gateway_id, config=client_config
-        )
+        try:
+            auth_token = await _get_existing_auth_token(
+                agent_gateway_id=agent_gateway_id,
+                config=client_config,
+                backoff=backoff,
+            )
+        except TimeoutError as exc:
+            result.errors.append(
+                GatewayTemplatesSyncError(
+                    agent_id=agent.id,
+                    agent_name=agent.name,
+                    board_id=board.id,
+                    message=str(exc),
+                )
+            )
+            return result
 
         if not auth_token:
             if not rotate_tokens:
@@ -321,6 +432,7 @@ async def sync_gateway_templates(
                 )
 
         try:
+
             async def _do_provision() -> None:
                 await provision_agent(
                     agent,
@@ -333,8 +445,19 @@ async def sync_gateway_templates(
                     reset_session=reset_sessions,
                 )
 
-            await _with_gateway_retry(_do_provision)
+            await _with_gateway_retry(_do_provision, backoff=backoff)
             result.agents_updated += 1
+        except TimeoutError as exc:  # pragma: no cover - gateway/network dependent
+            result.agents_skipped += 1
+            result.errors.append(
+                GatewayTemplatesSyncError(
+                    agent_id=agent.id,
+                    agent_name=agent.name,
+                    board_id=board.id,
+                    message=str(exc),
+                )
+            )
+            return result
         except Exception as exc:  # pragma: no cover - gateway/network dependent
             result.agents_skipped += 1
             result.errors.append(
@@ -360,10 +483,21 @@ async def sync_gateway_templates(
             )
             return result
 
-        main_gateway_agent_id = await _gateway_default_agent_id(
-            client_config,
-            fallback_session_key=gateway.main_session_key,
-        )
+        try:
+            main_gateway_agent_id = await _gateway_default_agent_id(
+                client_config,
+                fallback_session_key=gateway.main_session_key,
+                backoff=backoff,
+            )
+        except TimeoutError as exc:
+            result.errors.append(
+                GatewayTemplatesSyncError(
+                    agent_id=main_agent.id,
+                    agent_name=main_agent.name,
+                    message=str(exc),
+                )
+            )
+            return result
         if not main_gateway_agent_id:
             result.errors.append(
                 GatewayTemplatesSyncError(
@@ -374,9 +508,21 @@ async def sync_gateway_templates(
             )
             return result
 
-        main_token = await _get_existing_auth_token(
-            agent_gateway_id=main_gateway_agent_id, config=client_config
-        )
+        try:
+            main_token = await _get_existing_auth_token(
+                agent_gateway_id=main_gateway_agent_id,
+                config=client_config,
+                backoff=backoff,
+            )
+        except TimeoutError as exc:
+            result.errors.append(
+                GatewayTemplatesSyncError(
+                    agent_id=main_agent.id,
+                    agent_name=main_agent.name,
+                    message=str(exc),
+                )
+            )
+            return result
         if not main_token:
             if rotate_tokens:
                 raw_token = generate_agent_token()
@@ -417,6 +563,7 @@ async def sync_gateway_templates(
                 )
 
         try:
+
             async def _do_provision_main() -> None:
                 await provision_main_agent(
                     main_agent,
@@ -428,8 +575,17 @@ async def sync_gateway_templates(
                     reset_session=reset_sessions,
                 )
 
-            await _with_gateway_retry(_do_provision_main)
+            await _with_gateway_retry(_do_provision_main, backoff=backoff)
             result.main_updated = True
+        except TimeoutError as exc:  # pragma: no cover - gateway/network dependent
+            result.errors.append(
+                GatewayTemplatesSyncError(
+                    agent_id=main_agent.id,
+                    agent_name=main_agent.name,
+                    message=str(exc),
+                )
+            )
+            return result
         except Exception as exc:  # pragma: no cover - gateway/network dependent
             result.errors.append(
                 GatewayTemplatesSyncError(