Fix non-Docket background tasks for MCP servers

Dockerfile

@@ -112,13 +112,14 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
 # You may override with DISABLE_AUTH=true in development.
 ENV DISABLE_AUTH=false
 
-# Default to development mode (no separate worker needed).
-# For production, override the command to remove --no-worker and run a separate task-worker container.
+# Default to development mode using the API's default backend (Docket). For
+# single-process development without a worker, add `--task-backend=asyncio` to
+# the api command.
 # Examples:
-#   Development: docker run -p 8000:8000 redislabs/agent-memory-server
+#   Development: docker run -p 8000:8000 redislabs/agent-memory-server agent-memory api --host 0.0.0.0 --port 8000 --task-backend=asyncio
 #   Production API: docker run -p 8000:8000 redislabs/agent-memory-server agent-memory api --host 0.0.0.0 --port 8000
 #   Production Worker: docker run redislabs/agent-memory-server agent-memory task-worker --concurrency 10
-CMD ["agent-memory", "api", "--host", "0.0.0.0", "--port", "8000", "--no-worker"]
+CMD ["agent-memory", "api", "--host", "0.0.0.0", "--port", "8000"]
 
 # ============================================
 # AWS VARIANT - Includes AWS Bedrock support
@@ -142,10 +143,11 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
 # You may override with DISABLE_AUTH=true in development.
 ENV DISABLE_AUTH=false
 
-# Default to development mode (no separate worker needed).
-# For production, override the command to remove --no-worker and run a separate task-worker container.
+# Default to development mode using the API's default backend (Docket). For
+# single-process development without a worker, add `--task-backend=asyncio` to
+# the api command.
 # Examples:
-#   Development: docker run -p 8000:8000 redislabs/agent-memory-server:aws
+#   Development: docker run -p 8000:8000 redislabs/agent-memory-server:aws agent-memory api --host 0.0.0.0 --port 8000 --task-backend=asyncio
 #   Production API: docker run -p 8000:8000 redislabs/agent-memory-server:aws agent-memory api --host 0.0.0.0 --port 8000
 #   Production Worker: docker run redislabs/agent-memory-server:aws agent-memory task-worker --concurrency 10
-CMD ["agent-memory", "api", "--host", "0.0.0.0", "--port", "8000", "--no-worker"]
+CMD ["agent-memory", "api", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -31,10 +31,14 @@ docker-compose up
 docker run -p 8000:8000 \
   -e REDIS_URL=redis://your-redis:6379 \
   -e OPENAI_API_KEY=your-key \
-  redislabs/agent-memory-server:latest
+  redislabs/agent-memory-server:latest \
+  agent-memory api --host 0.0.0.0 --port 8000 --task-backend=asyncio
 ```
 
-The default image runs in development mode (`--no-worker`), which is perfect for testing and development.
+By default, the image runs the API with the **Docket** task backend, which
+expects a separate `agent-memory task-worker` process for non-blocking
+background tasks. The example above shows how to override this to use the
+asyncio backend for a single-container development setup.
 
 **Production Deployment**:
 
@@ -74,8 +78,8 @@ uv install --all-extras
 # Start Redis
 docker-compose up redis
 
-# Start the server (development mode)
-uv run agent-memory api --no-worker
+# Start the server (development mode, default asyncio backend)
+uv run agent-memory api
 ```
 
 ### 2. Python SDK
@@ -155,8 +159,8 @@ result = await executor.ainvoke({"input": "Remember that I love pizza"})
 # Start MCP server (stdio mode - recommended for Claude Desktop)
 uv run agent-memory mcp
 
-# Or with SSE mode (development mode)
-uv run agent-memory mcp --mode sse --port 9000 --no-worker
+# Or with SSE mode (development mode, default asyncio backend)
+uv run agent-memory mcp --mode sse --port 9000
 ```
 
 ### MCP config via uvx (recommended)

agent_memory_server/__init__.py

@@ -1,3 +1,3 @@
 """Redis Agent Memory Server - A memory system for conversational AI."""
 
-__version__ = "0.12.5"
+__version__ = "0.12.6"

agent_memory_server/cli.py

@@ -257,7 +257,7 @@ async def run_migration():
             )
         else:
             click.echo(
-                "\nMigration completed with errors. " "Run again to retry failed keys."
+                "\nMigration completed with errors. Run again to retry failed keys."
             )
 
     asyncio.run(run_migration())
@@ -268,16 +268,41 @@ async def run_migration():
 @click.option("--host", default="0.0.0.0", help="Host to run the server on")
 @click.option("--reload", is_flag=True, help="Enable auto-reload")
 @click.option(
-    "--no-worker", is_flag=True, help="Use FastAPI background tasks instead of Docket"
+    "--no-worker",
+    is_flag=True,
+    help=(
+        "(DEPRECATED) Use --task-backend=asyncio instead. "
+        "If present, force FastAPI/asyncio background tasks instead of Docket."
+    ),
+    deprecated=True,
+)
+@click.option(
+    "--task-backend",
+    default="docket",
+    type=click.Choice(["asyncio", "docket"]),
+    help=(
+        "Background task backend (asyncio, docket). "
+        "Default is 'docket' to preserve existing behavior using Docket-based "
+        "workers (requires a running `agent-memory task-worker` for "
+        "non-blocking background tasks). Use 'asyncio' (or deprecated "
+        "--no-worker) for single-process development without a worker."
+    ),
 )
-def api(port: int, host: str, reload: bool, no_worker: bool):
+def api(port: int, host: str, reload: bool, no_worker: bool, task_backend: str):
     """Run the REST API server."""
     from agent_memory_server.main import on_start_logger
 
     configure_logging()
 
-    # Set use_docket based on the --no-worker flag
-    if no_worker:
+    # Determine effective backend.
+    # - Default is 'docket' to preserve prior behavior (Docket workers).
+    # - --task-backend=asyncio opts into single-process asyncio background tasks.
+    # - Deprecated --no-worker flag forces asyncio for backward compatibility.
+    effective_backend = "asyncio" if no_worker else task_backend
+
+    if effective_backend == "docket":
+        settings.use_docket = True
+    else:  # "asyncio"
         settings.use_docket = False
 
     on_start_logger(port)
@@ -298,9 +323,17 @@ def api(port: int, host: str, reload: bool, no_worker: bool):
     type=click.Choice(["stdio", "sse"]),
 )
 @click.option(
-    "--no-worker", is_flag=True, help="Use FastAPI background tasks instead of Docket"
+    "--task-backend",
+    default="asyncio",
+    type=click.Choice(["asyncio", "docket"]),
+    help=(
+        "Background task backend (asyncio, docket). "
+        "Default is 'asyncio' (no separate worker needed). "
+        "Use 'docket' for production setups with a running task worker "
+        "(see `agent-memory task-worker`)."
+    ),
 )
-def mcp(port: int, mode: str, no_worker: bool):
+def mcp(port: int, mode: str, task_backend: str):
     """Run the MCP server."""
     import asyncio
 
@@ -317,14 +350,12 @@ def mcp(port: int, mode: str, no_worker: bool):
     from agent_memory_server.mcp import mcp_app
 
     async def setup_and_run():
-        # Redis setup is handled by the MCP app before it starts
-
-        # Set use_docket based on mode and --no-worker flag
-        if mode == "stdio":
-            # Don't run a task worker in stdio mode by default
-            settings.use_docket = False
-        elif no_worker:
-            # Use --no-worker flag for SSE mode
+        # Configure background task backend for MCP.
+        # Default is asyncio (no separate worker required). Use 'docket' to
+        # send tasks to a separate worker process.
+        if task_backend == "docket":
+            settings.use_docket = True
+        else:  # "asyncio"
             settings.use_docket = False
 
         # Run the MCP server

agent_memory_server/dependencies.py

@@ -1,8 +1,10 @@
+import asyncio
 import concurrent.futures
 from collections.abc import Callable
 from typing import Any
 
 from fastapi import BackgroundTasks
+from starlette.concurrency import run_in_threadpool
 
 from agent_memory_server.config import settings
 from agent_memory_server.logging import get_logger
@@ -12,11 +14,26 @@
 
 
 class HybridBackgroundTasks(BackgroundTasks):
-    """A BackgroundTasks implementation that can use either Docket or FastAPI background tasks."""
+    """A BackgroundTasks implementation that can use either Docket or asyncio tasks.
+
+    When use_docket=True, tasks are scheduled through Docket's Redis-based queue
+    for processing by a separate worker process.
+
+    When use_docket=False, tasks are scheduled using asyncio.create_task() to run
+    in the current event loop. This works in both FastAPI and MCP contexts, unlike
+    the parent class's approach which relies on Starlette's response lifecycle
+    (which doesn't exist in MCP's stdio/SSE modes).
+    """
 
     def add_task(self, func: Callable[..., Any], *args: Any, **kwargs: Any) -> None:
-        """Run tasks either directly, through Docket, or through FastAPI background tasks"""
-        logger.info("Adding task to background tasks...")
+        """Schedule a background task for execution.
+
+        Args:
+            func: The function to run (can be sync or async)
+            *args: Positional arguments to pass to the function
+            **kwargs: Keyword arguments to pass to the function
+        """
+        logger.info(f"Adding background task: {func.__name__}")
 
         if settings.use_docket:
             logger.info("Scheduling task through Docket")
@@ -28,7 +45,6 @@ def add_task(self, func: Callable[..., Any], *args: Any, **kwargs: Any) -> None:
             # This runs in a thread to avoid event loop conflicts
             def run_in_thread():
                 """Run the async Docket operations in a separate thread"""
-                import asyncio
 
                 async def schedule_task():
                     async with Docket(
@@ -48,9 +64,34 @@ async def schedule_task():
 
             # When using Docket, we don't add anything to FastAPI background tasks
         else:
-            logger.info("Using FastAPI background tasks")
-            # Use FastAPI's background tasks directly
-            super().add_task(func, *args, **kwargs)
+            logger.info("Scheduling task with asyncio.create_task")
+            # Use asyncio.create_task to schedule the task in the event loop.
+            # This works universally in both FastAPI and MCP contexts.
+            #
+            # Note: We don't use super().add_task() because Starlette's BackgroundTasks
+            # relies on being attached to a response object and run after the response
+            # is sent. In MCP mode (stdio/SSE), there's no Starlette response lifecycle,
+            # so tasks added via super().add_task() would never execute.
+            asyncio.create_task(self._run_task(func, *args, **kwargs))
+
+    async def _run_task(
+        self, func: Callable[..., Any], *args: Any, **kwargs: Any
+    ) -> None:
+        """Execute a background task, handling both sync and async functions.
+
+        Args:
+            func: The function to run (can be sync or async)
+            *args: Positional arguments to pass to the function
+            **kwargs: Keyword arguments to pass to the function
+        """
+        try:
+            if asyncio.iscoroutinefunction(func):
+                await func(*args, **kwargs)
+            else:
+                # Run sync functions in a thread pool to avoid blocking the event loop
+                await run_in_threadpool(func, *args, **kwargs)
+        except Exception as e:
+            logger.error(f"Background task {func.__name__} failed: {e}", exc_info=True)
 
 
 # Backwards compatibility alias

docker-compose-task-workers.yml

@@ -10,8 +10,8 @@ networks:
 
 services:
   # For testing a production-like setup, you can run this API and the
-  # task-worker container. This API container does NOT use --no-worker, so when
-  # it starts background work, the task-worker will process those tasks.
+  # task-worker container. This API container uses --task-backend=docket, so
+  # when it starts background work, the task-worker will process those tasks.
 
   # =============================================================================
   # STANDARD (OpenAI/Anthropic only)
@@ -44,7 +44,7 @@ services:
       interval: 30s
       timeout: 10s
       retries: 3
-    command: ["agent-memory", "api", "--host", "0.0.0.0", "--port", "8000"]
+    command: ["agent-memory", "api", "--host", "0.0.0.0", "--port", "8000", "--task-backend", "docket"]
 
   mcp:
     profiles: ["standard", ""]
@@ -64,7 +64,7 @@ services:
       - "9050:9000"
     depends_on:
       - redis
-    command: ["agent-memory", "mcp", "--mode", "sse"]
+    command: ["agent-memory", "mcp", "--mode", "sse", "--task-backend", "docket"]
     networks:
       - server-network
 
@@ -126,7 +126,7 @@ services:
       interval: 30s
       timeout: 10s
       retries: 3
-    command: ["agent-memory", "api", "--host", "0.0.0.0", "--port", "8000"]
+    command: ["agent-memory", "api", "--host", "0.0.0.0", "--port", "8000", "--task-backend", "docket"]
 
   mcp-aws:
     profiles: ["aws"]
@@ -151,7 +151,7 @@ services:
       - "9050:9000"
     depends_on:
       - redis
-    command: ["agent-memory", "mcp", "--mode", "sse"]
+    command: ["agent-memory", "mcp", "--mode", "sse", "--task-backend", "docket"]
     networks:
       - server-network
 

docs/cli.md

@@ -27,15 +27,16 @@ agent-memory api [OPTIONS]
 - `--port INTEGER`: Port to run the server on. (Default: value from `settings.port`, usually 8000)
 - `--host TEXT`: Host to run the server on. (Default: "0.0.0.0")
 - `--reload`: Enable auto-reload for development.
-- `--no-worker`: Use FastAPI background tasks instead of Docket workers. Ideal for development and testing.
+- `--task-backend [asyncio|docket]`: Background task backend. `docket` (default) uses Docket-based background workers (requires a running `agent-memory task-worker` for non-blocking tasks). `asyncio` runs tasks inline in the API process and does **not** require a separate worker.
+- `--no-worker` (**deprecated**): Backwards-compatible alias for `--task-backend=asyncio`. Maintained for older scripts; prefer `--task-backend`.
 
 **Examples:**
 
 ```bash
-# Development mode (no separate worker needed)
-agent-memory api --port 8080 --reload --no-worker
+# Development mode (no separate worker needed, asyncio backend)
+agent-memory api --port 8080 --reload --task-backend asyncio
 
-# Production mode (requires separate worker process)
+# Production mode (default Docket backend; requires separate worker process)
 agent-memory api --port 8080
 ```
 
@@ -51,22 +52,22 @@ agent-memory mcp [OPTIONS]
 
 - `--port INTEGER`: Port to run the MCP server on. (Default: value from `settings.mcp_port`, usually 9000)
 - `--mode [stdio|sse]`: Run the MCP server in stdio or SSE mode. (Default: stdio)
-- `--no-worker`: Use FastAPI background tasks instead of Docket workers. Ideal for development and testing.
+- `--task-backend [asyncio|docket]`: Background task backend. `asyncio` (default) runs tasks inline in the MCP process with no separate worker. `docket` sends tasks to a Docket queue, which requires running `agent-memory task-worker`.
 
 **Examples:**
 
 ```bash
-# Stdio mode (recommended for Claude Desktop) - automatically uses --no-worker
+# Stdio mode (recommended for Claude Desktop) - default asyncio backend
 agent-memory mcp
 
 # SSE mode for development (no separate worker needed)
-agent-memory mcp --mode sse --port 9001 --no-worker
+agent-memory mcp --mode sse --port 9001
 
 # SSE mode for production (requires separate worker process)
-agent-memory mcp --mode sse --port 9001
+agent-memory mcp --mode sse --port 9001 --task-backend docket
 ```
 
-**Note:** Stdio mode automatically disables Docket workers as they're not needed when Claude Desktop manages the process lifecycle.
+**Note:** Stdio mode is designed for tools like Claude Desktop and, by default, uses the asyncio backend (no worker). Use `--task-backend docket` if you want MCP to enqueue background work into a shared Docket worker.
 
 ### `schedule-task`
 

docs/getting-started.md

@@ -28,10 +28,10 @@ But you can also run these components via the CLI commands. Here's how you
 run the REST API server:
 
 ```bash
-# Development mode (no separate worker needed)
-uv run agent-memory api --no-worker
+# Development mode (no separate worker needed, asyncio backend)
+uv run agent-memory api --task-backend asyncio
 
-# Production mode (requires separate worker process)
+# Production mode (default Docket backend; requires separate worker process)
 uv run agent-memory api
 ```
 
@@ -42,10 +42,10 @@ Or the MCP server:
 uv run agent-memory mcp
 
 # SSE mode for development
-uv run agent-memory mcp --mode sse --no-worker
-
-# SSE mode for production
 uv run agent-memory mcp --mode sse
+
+# SSE mode for production (use Docket backend)
+uv run agent-memory mcp --mode sse --task-backend docket
 ```
 
 ### Using uvx in MCP clients
@@ -80,7 +80,7 @@ Notes:
 uv run agent-memory task-worker
 ```
 
-**For development**, use the `--no-worker` flag to run tasks inline without needing a separate worker process.
+**For development**, the default `--task-backend=asyncio` on the `mcp` command runs tasks inline without needing a separate worker process. For the `api` command, use `--task-backend=asyncio` explicitly when you want single-process behavior.
 
 **NOTE:** With uv, prefix the command with `uv`, e.g.: `uv run agent-memory --mode sse`. If you installed from source, you'll probably need to add `--directory` to tell uv where to find the code: `uv run --directory <path/to/checkout> run agent-memory --mode stdio`.