Add stdio daemon mode with configurable runtime logging

Author: danias

.github/workflows/ci.yml

@@ -33,3 +33,4 @@ jobs:
         run: |
           bitloops-embeddings --help
           bitloops-embeddings describe --model bge-m3
+          bitloops-embeddings daemon --help

README.md

@@ -4,6 +4,7 @@
 
 - a one-shot CLI for simple embedding requests
 - a long-lived local HTTP server for repeated requests
+- a long-lived stdio daemon for process-managed IPC
 - release packaging for major desktop and server operating systems
 
 The first release is intentionally operational rather than retrieval-quality-complete. It focuses on a stable interface, model bootstrapping, hello-world inference, and releasable artefacts.
@@ -105,6 +106,15 @@ Override the bind target:
 bitloops-embeddings serve --model bge-m3 --host 127.0.0.1 --port 7719
 ```
 
+Configure logging for long-lived modes:
+
+```bash
+bitloops-embeddings serve \
+  --model bge-m3 \
+  --log-level debug \
+  --log-file ./bitloops-embeddings.log
+```
+
 ### HTTP API
 
 Health:
@@ -146,6 +156,56 @@ Error shape:
 }
 ```
 
+## Daemon usage
+
+Start the stdio daemon:
+
+```bash
+bitloops-embeddings daemon --model bge-m3
+```
+
+The daemon:
+
+- loads the model once and keeps it warm
+- reads newline-delimited JSON requests from `stdin`
+- writes newline-delimited JSON protocol responses only to `stdout`
+- writes logs and diagnostics to the configured log sink or, if needed, to `stderr`
+
+Use a custom log file:
+
+```bash
+bitloops-embeddings daemon \
+  --model bge-m3 \
+  --log-level info \
+  --log-file ./bitloops-embeddings-daemon.log
+```
+
+Ready event:
+
+```json
+{"event":"ready","protocol":1,"capabilities":["embed","ping","health","shutdown"]}
+```
+
+Example request:
+
+```json
+{"id":"1","cmd":"embed","texts":["hello","world"],"model":"bge-m3"}
+```
+
+Example response:
+
+```json
+{"id":"1","ok":true,"vectors":[[0.12,0.98],[-0.44,0.07]],"model":"bge-m3"}
+```
+
+Example error:
+
+```json
+{"id":"7","ok":false,"error":{"code":"UNKNOWN_COMMAND","message":"unsupported cmd: frobnicate"}}
+```
+
+The daemon exits cleanly on `shutdown` or when `stdin` reaches EOF.
+
 ## Cache directory resolution
 
 Model cache resolution order:
@@ -198,5 +258,7 @@ The repository includes two workflows:
 ## Troubleshooting
 
 - The first `embed` or `serve` invocation downloads model files into the local cache. This can take a while on a cold machine.
+- The first `daemon` invocation also downloads model files into the local cache if they are not already present.
 - If model loading fails, check network access to Hugging Face and confirm the cache directory is writable.
+- Long-lived modes support `--log-level` and `--log-file`. Without `--log-file`, `serve` and `daemon` use a best-effort OS log sink and fall back to `stderr` if the native sink is unavailable.
 - The runtime does not log input texts by default.

scripts/real_backend_smoke.py

@@ -5,6 +5,7 @@
 import socket
 import subprocess
 import sys
+import tempfile
 import time
 from pathlib import Path
 from urllib import error, request
@@ -24,6 +25,7 @@ def main() -> None:
 
     run_embed_smoke(binary)
     run_server_smoke(binary, port)
+    run_daemon_smoke(binary)
 
 
 def run_embed_smoke(binary: str) -> None:
@@ -48,39 +50,109 @@ def run_embed_smoke(binary: str) -> None:
 
 
 def run_server_smoke(binary: str, port: int) -> None:
-    process = subprocess.Popen(
-        [
-            binary,
-            "serve",
-            "--model",
-            "bge-m3",
-            "--host",
-            "127.0.0.1",
-            "--port",
-            str(port),
-        ],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        text=True,
-    )
+    with tempfile.TemporaryDirectory(prefix="bitloops-embeddings-serve-logs-") as temp_dir:
+        log_file = Path(temp_dir) / "serve.log"
+        process = subprocess.Popen(
+            [
+                binary,
+                "serve",
+                "--model",
+                "bge-m3",
+                "--host",
+                "127.0.0.1",
+                "--port",
+                str(port),
+                "--log-file",
+                str(log_file),
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
 
-    try:
-        wait_for_health(process, port)
-        embed_response = http_post_json(
-            f"http://127.0.0.1:{port}/embed",
-            {"texts": ["Hello World"]},
+        try:
+            wait_for_health(process, port)
+            embed_response = http_post_json(
+                f"http://127.0.0.1:{port}/embed",
+                {"texts": ["Hello World"]},
+            )
+            if embed_response["model_id"] != "bge-m3":
+                raise RuntimeError("Server smoke returned an unexpected model id.")
+            if not embed_response["embeddings"] or not embed_response["embeddings"][0]:
+                raise RuntimeError("Server smoke returned an empty embedding vector.")
+        finally:
+            process.terminate()
+            try:
+                process.wait(timeout=20)
+            except subprocess.TimeoutExpired:
+                process.kill()
+                process.wait(timeout=5)
+
+
+def run_daemon_smoke(binary: str) -> None:
+    with tempfile.TemporaryDirectory(prefix="bitloops-embeddings-daemon-logs-") as temp_dir:
+        log_file = Path(temp_dir) / "daemon.log"
+        process = subprocess.Popen(
+            [
+                binary,
+                "daemon",
+                "--model",
+                "bge-m3",
+                "--log-file",
+                str(log_file),
+            ],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
         )
-        if embed_response["model_id"] != "bge-m3":
-            raise RuntimeError("Server smoke returned an unexpected model id.")
-        if not embed_response["embeddings"] or not embed_response["embeddings"][0]:
-            raise RuntimeError("Server smoke returned an empty embedding vector.")
-    finally:
-        process.terminate()
+
         try:
-            process.wait(timeout=20)
-        except subprocess.TimeoutExpired:
-            process.kill()
-            process.wait(timeout=5)
+            ready_event = read_json_line(process.stdout)
+            if ready_event.get("event") != "ready":
+                raise RuntimeError("Daemon smoke did not emit a ready event.")
+
+            write_json_line(process.stdin, {"id": "1", "cmd": "ping"})
+            ping_response = read_json_line(process.stdout)
+            if ping_response != {"id": "1", "ok": True, "pong": True}:
+                raise RuntimeError(f"Unexpected daemon ping response: {ping_response}")
+
+            write_json_line(
+                process.stdin,
+                {"id": "2", "cmd": "embed", "texts": ["Hello World"]},
+            )
+            embed_response = read_json_line(process.stdout)
+            if embed_response.get("model") != "bge-m3":
+                raise RuntimeError("Daemon smoke returned an unexpected model id.")
+            if not embed_response.get("vectors") or not embed_response["vectors"][0]:
+                raise RuntimeError("Daemon smoke returned an empty embedding vector.")
+
+            write_json_line(process.stdin, {"id": "3", "cmd": "shutdown"})
+            shutdown_response = read_json_line(process.stdout)
+            if shutdown_response != {"id": "3", "ok": True}:
+                raise RuntimeError(f"Unexpected daemon shutdown response: {shutdown_response}")
+            if process.wait(timeout=20) != 0:
+                raise RuntimeError("Daemon smoke exited with a non-zero status.")
+        finally:
+            if process.poll() is None:
+                process.terminate()
+                try:
+                    process.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    process.kill()
+                    process.wait(timeout=5)
+
+
+def write_json_line(stream, payload: dict[str, object]) -> None:
+    stream.write(json.dumps(payload) + "\n")
+    stream.flush()
+
+
+def read_json_line(stream) -> dict[str, object]:
+    line = stream.readline()
+    if not line:
+        raise RuntimeError("Expected a protocol message but reached EOF.")
+    return json.loads(line)
 
 
 def wait_for_health(process: subprocess.Popen[str], port: int, timeout_seconds: int = 180) -> None:

src/bitloops_embeddings/backend/base.py

@@ -13,7 +13,11 @@ def backend_name(self) -> str: ...
     @property
     def dimensions(self) -> int: ...
 
+    @property
+    def is_loaded(self) -> bool: ...
+
     def load(self) -> None: ...
 
     def embed(self, texts: list[str]) -> list[list[float]]: ...
 
+    def close(self) -> None: ...

src/bitloops_embeddings/backend/sentence_transformers_backend.py

@@ -34,8 +34,12 @@ def backend_name(self) -> str:
     def dimensions(self) -> int:
         return self._dimensions
 
+    @property
+    def is_loaded(self) -> bool:
+        return self._model is not None
+
     def load(self) -> None:
-        if self._model is not None:
+        if self.is_loaded:
             return
 
         try:
@@ -90,3 +94,6 @@ def embed(self, texts: list[str]) -> list[list[float]]:
         if hasattr(vectors, "tolist"):
             return vectors.tolist()
         return [[float(value) for value in vector] for vector in vectors]
+
+    def close(self) -> None:
+        self._model = None

src/bitloops_embeddings/cli.py

@@ -8,6 +8,7 @@
 
 from bitloops_embeddings.backend.base import EmbeddingBackend
 from bitloops_embeddings.cache import ensure_cache_dir, resolve_cache_dir
+from bitloops_embeddings.daemon import run_daemon
 from bitloops_embeddings.errors import BitloopsEmbeddingsError
 from bitloops_embeddings.logging_utils import configure_logging, log_event
 from bitloops_embeddings.models import EmbeddingResponse, RuntimeInfo
@@ -39,6 +40,10 @@ class LogLevel(str, Enum):
     ERROR = "error"
 
 
+class Transport(str, Enum):
+    STDIO = "stdio"
+
+
 def main() -> None:
     app(prog_name=RUNTIME_NAME)
 
@@ -109,14 +114,22 @@ def serve(
         LogLevel,
         typer.Option("--log-level", help="Server log verbosity.", case_sensitive=False),
     ] = LogLevel.INFO,
+    log_file: Annotated[
+        Optional[Path],
+        typer.Option(
+            "--log-file",
+            help="Optional log file path. Defaults to the OS log sink for long-lived modes.",
+            dir_okay=False,
+            writable=True,
+        ),
+    ] = None,
     max_batch_size: Annotated[
         int,
         typer.Option("--max-batch-size", help="Maximum texts accepted by the /embed endpoint."),
     ] = 32,
 ) -> None:
-    configure_logging(log_level.value)
-
     try:
+        configure_logging(log_level.value, log_file=log_file, prefer_os_log=True)
         backend = _build_backend(model=model, cache_dir=cache_dir)
         backend.load()
         app_instance = create_app(backend, max_batch_size=max_batch_size)
@@ -134,6 +147,53 @@ def serve(
         _exit_with_error(BitloopsEmbeddingsError(f"Unexpected runtime error: {exc}"))
 
 
+@app.command()
+def daemon(
+    model: Annotated[str, typer.Option("--model", help="Public model identifier.")],
+    transport: Annotated[
+        Transport,
+        typer.Option("--transport", help="IPC transport.", case_sensitive=False),
+    ] = Transport.STDIO,
+    cache_dir: Annotated[
+        Optional[Path],
+        typer.Option(
+            "--cache-dir",
+            help="Override the model cache directory.",
+            file_okay=False,
+            dir_okay=True,
+            writable=True,
+        ),
+    ] = None,
+    log_level: Annotated[
+        LogLevel,
+        typer.Option("--log-level", help="Daemon log verbosity.", case_sensitive=False),
+    ] = LogLevel.INFO,
+    log_file: Annotated[
+        Optional[Path],
+        typer.Option(
+            "--log-file",
+            help="Optional log file path. Defaults to the OS log sink for long-lived modes.",
+            dir_okay=False,
+            writable=True,
+        ),
+    ] = None,
+) -> None:
+    try:
+        configure_logging(log_level.value, log_file=log_file, prefer_os_log=True)
+        if transport is not Transport.STDIO:
+            raise typer.BadParameter("Only stdio transport is supported in v1.")
+        backend = _build_backend(model=model, cache_dir=cache_dir)
+        raise typer.Exit(code=run_daemon(backend))
+    except typer.BadParameter:
+        raise
+    except typer.Exit:
+        raise
+    except BitloopsEmbeddingsError as exc:
+        _exit_with_error(exc)
+    except Exception as exc:
+        _exit_with_error(BitloopsEmbeddingsError(f"Unexpected runtime error: {exc}"))
+
+
 @app.command()
 def describe(
     model: Annotated[str, typer.Option("--model", help="Public model identifier.")],
@@ -180,6 +240,9 @@ def _emit_json(payload: str, *, output: Optional[Path]) -> None:
 
 
 def _exit_with_error(exc: BitloopsEmbeddingsError) -> None:
-    log_event("fatal_error", code=exc.code, message=str(exc))
+    try:
+        log_event("fatal_error", code=exc.code, message=str(exc))
+    except Exception:
+        pass
     typer.echo(f"Error: {exc}", err=True)
     raise typer.Exit(code=1)