feat(storage): 新增 SQLiteBackend 状态后端 — 基于 sqlite3 按键查询, 支持 TTL/batch/线程安全, 解决 JSONBackend 全量加载瓶颈, 作为 drop-in 替换

tests/test_storage.py

@@ -4,7 +4,9 @@ from __future__ import annotations
 
 import json
 import os
+import sqlite3
 import tempfile
+import threading
 import time
 from pathlib import Path
 from typing import Any
@@ -12,7 +14,7 @@ from typing import Any
 import pytest
 
 from pyflowx.errors import StorageError
-from pyflowx.storage import JSONBackend, MemoryBackend, StateBackend, resolve_backend
+from pyflowx.storage import JSONBackend, MemoryBackend, SQLiteBackend, StateBackend, resolve_backend
 
 
 @pytest.fixture
@@ -294,6 +296,284 @@ def test_json_backend_save_value_error(monkeypatch: pytest.MonkeyPatch) -> None:
         monkeypatch.setattr(_json, "dumps", original_dumps)
 
 
+# ---------------------------------------------------------------------- #
+# SQLiteBackend
+# ---------------------------------------------------------------------- #
+def test_sqlite_backend_save_and_load() -> None:
+    """save 后重新打开同一文件应读到已保存内容。"""
+    with tempfile.TemporaryDirectory() as tmp:
+        path = str(Path(tmp) / "state.db")
+        b = SQLiteBackend(path)
+        b.save("a", {"x": 1})
+        b.save("b", [1, 2, 3])
+        b.close()
+        b2 = SQLiteBackend(path)
+        assert b2.has("a")
+        assert b2.get("a") == {"x": 1}
+        assert b2.get("b") == [1, 2, 3]
+        assert dict(b2.load()) == {"a": {"x": 1}, "b": [1, 2, 3]}
+        b2.close()
+
+
+def test_sqlite_backend_overwrite_existing() -> None:
+    """对同一 key 再次 save 应覆盖旧值。"""
+    with tempfile.TemporaryDirectory() as tmp:
+        path = str(Path(tmp) / "state.db")
+        b = SQLiteBackend(path)
+        b.save("a", 1)
+        b.save("a", 2)
+        assert b.get("a") == 2
+        b.close()
+
+
+def test_sqlite_backend_clear() -> None:
+    """clear 后所有键清空，且持久化到文件。"""
+    with tempfile.TemporaryDirectory() as tmp:
+        path = str(Path(tmp) / "state.db")
+        b = SQLiteBackend(path)
+        b.save("a", 1)
+        b.clear()
+        assert not b.has("a")
+        assert dict(b.load()) == {}
+        b.close()
+        # 重新打开仍为空
+        b2 = SQLiteBackend(path)
+        assert dict(b2.load()) == {}
+        b2.close()
+
+
+def test_sqlite_backend_nonexistent_file_starts_empty() -> None:
+    """文件不存在时应正常初始化为空。"""
+    with tempfile.TemporaryDirectory() as tmp:
+        path = str(Path(tmp) / "absent.db")
+        b = SQLiteBackend(path)
+        assert dict(b.load()) == {}
+        assert not b.has("anything")
+        b.close()
+
+
+def test_sqlite_backend_non_serialisable_raises() -> None:
+    """不可 JSON 序列化的值应抛 StorageError，且不写入数据库。"""
+    with tempfile.TemporaryDirectory() as tmp:
+        path = str(Path(tmp) / "state.db")
+        b = SQLiteBackend(path)
+        with pytest.raises(StorageError):
+            b.save("a", object())  # object() 不可序列化
+        assert not b.has("a")
+        b.close()
+
+
+def test_sqlite_backend_get_missing_raises() -> None:
+    """get 不存在的键应抛 KeyError。"""
+    with tempfile.TemporaryDirectory() as tmp:
+        path = str(Path(tmp) / "state.db")
+        b = SQLiteBackend(path)
+        with pytest.raises(KeyError):
+            b.get("nope")
+        b.close()
+
+
+def test_sqlite_backend_memory_db() -> None:
+    """:memory: 内存数据库基础生命周期。"""
+    b = SQLiteBackend(":memory:")
+    b.save("a", 1)
+    assert b.has("a")
+    assert b.get("a") == 1
+    assert dict(b.load()) == {"a": 1}
+    b.clear()
+    assert not b.has("a")
+    b.close()
+
+
+def test_sqlite_backend_batch_defers_commit(tmp_path: Path) -> None:
+    """batch 期间 save 不立即 commit，第二个连接看不到未提交数据；退出后可见。"""
+    path = str(tmp_path / "state.db")
+    b = SQLiteBackend(path)
+    with b.batch():
+        b.save("a", 1)
+        b.save("b", 2)
+        # 第二个连接在 WAL 模式下只读已提交数据，应看不到 batch 内未提交的写入
+        other = sqlite3.connect(path)
+        row = other.execute("SELECT value FROM state WHERE key = ?", ("a",)).fetchone()
+        other.close()
+        assert row is None
+    # batch 退出 commit 后，第二个连接应能看到
+    other = sqlite3.connect(path)
+    rows = other.execute("SELECT key FROM state ORDER BY key").fetchall()
+    other.close()
+    assert rows == [("a",), ("b",)]
+    b.close()
+
+
+def test_sqlite_backend_no_batch_commits_each_save(tmp_path: Path) -> None:
+    """无 batch 时每次 save 立即 commit，第二个连接立即可见。"""
+    path = str(tmp_path / "state.db")
+    b = SQLiteBackend(path)
+    b.save("a", 1)
+    # 第二个连接应立即看到已提交数据
+    other = sqlite3.connect(path)
+    row = other.execute("SELECT value FROM state WHERE key = ?", ("a",)).fetchone()
+    other.close()
+    assert row is not None
+    assert json.loads(row[0]) == 1
+    b.save("b", 2)
+    other = sqlite3.connect(path)
+    rows = other.execute("SELECT key FROM state ORDER BY key").fetchall()
+    other.close()
+    assert rows == [("a",), ("b",)]
+    b.close()
+
+
+def test_sqlite_backend_concurrent_access(tmp_path: Path) -> None:
+    """多线程并发 save/get 不出错，结果一致。"""
+    path = str(tmp_path / "state.db")
+    b = SQLiteBackend(path)
+    errors: list[BaseException] = []
+
+    def worker(start: int) -> None:
+        try:
+            for i in range(start, start + 20):
+                b.save(f"task_{i}", i)
+                assert b.get(f"task_{i}") == i
+        except BaseException as exc:
+            errors.append(exc)
+
+    threads = [threading.Thread(target=worker, args=(i * 20,)) for i in range(4)]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join()
+    assert not errors, f"并发访问出错: {errors}"
+    # 共写入 80 个键
+    assert len(dict(b.load())) == 80
+    b.close()
+
+
+def test_sqlite_backend_ttl_expired_has_returns_false() -> None:
+    """SQLiteBackend TTL 过期后 has 返回 False。"""
+    with tempfile.TemporaryDirectory() as tmp:
+        path = str(Path(tmp) / "state.db")
+        b = SQLiteBackend(path, ttl=0.1)
+        b.save("a", 1)
+        assert b.has("a")
+        time.sleep(0.15)
+        assert not b.has("a")
+        b.close()
+
+
+def test_sqlite_backend_ttl_expired_get_raises_keyerror() -> None:
+    """SQLiteBackend TTL 过期后 get 抛 KeyError。"""
+    with tempfile.TemporaryDirectory() as tmp:
+        path = str(Path(tmp) / "state.db")
+        b = SQLiteBackend(path, ttl=0.1)
+        b.save("a", 1)
+        time.sleep(0.15)
+        with pytest.raises(KeyError):
+            b.get("a")
+        b.close()
+
+
+def test_sqlite_backend_ttl_load_filters_expired() -> None:
+    """SQLiteBackend.load() 应过滤过期的条目。"""
+    with tempfile.TemporaryDirectory() as tmp:
+        path = str(Path(tmp) / "state.db")
+        b = SQLiteBackend(path, ttl=0.1)
+        b.save("a", 1)
+        b.save("b", 2)
+        time.sleep(0.15)
+        assert dict(b.load()) == {}
+        b.close()
+
+
+def test_sqlite_backend_no_ttl_never_expired(tmp_path: Path) -> None:
+    """无 TTL 时永不过期。"""
+    path = str(tmp_path / "state.db")
+    b = SQLiteBackend(path)
+    b.save("a", 1)
+    # 手动修改 ts 为很久以前
+    with b._lock:
+        b._conn.execute("UPDATE state SET ts = ? WHERE key = ?", (time.time() - 1000, "a"))
+        b._conn.commit()
+    assert b.has("a")
+    assert b.get("a") == 1
+    b.close()
+
+
+def test_sqlite_backend_expired_with_ttl(tmp_path: Path) -> None:
+    """有 TTL 时过期键 has 返回 False。"""
+    path = str(tmp_path / "state.db")
+    b = SQLiteBackend(path, ttl=1.0)
+    b.save("a", 1)
+    # 手动修改 ts 为很久以前
+    with b._lock:
+        b._conn.execute("UPDATE state SET ts = ? WHERE key = ?", (time.time() - 10, "a"))
+        b._conn.commit()
+    assert not b.has("a")
+    b.close()
+
+
+def test_sqlite_backend_corrupt_value_raises(tmp_path: Path) -> None:
+    """value 列被外部写入非法 JSON 时 get 抛 StorageError。"""
+    path = str(tmp_path / "state.db")
+    b = SQLiteBackend(path)
+    b.save("a", 1)
+    b.close()
+    # 直接用 sqlite3 写入非法 JSON
+    conn = sqlite3.connect(path)
+    conn.execute("UPDATE state SET value = ? WHERE key = ?", ("{not valid json", "a"))
+    conn.commit()
+    conn.close()
+    b2 = SQLiteBackend(path)
+    with pytest.raises(StorageError, match="cannot decode"):
+        b2.get("a")
+    b2.close()
+
+
+def test_sqlite_backend_iter_raw_decode_error_raises(tmp_path: Path) -> None:
+    """_iter_raw 遇到非法 JSON 时 load 抛 StorageError。"""
+    path = str(tmp_path / "state.db")
+    b = SQLiteBackend(path)
+    b.save("good", 1)
+    b.close()
+    # 写入非法 JSON
+    conn = sqlite3.connect(path)
+    conn.execute("INSERT INTO state (key, value, ts) VALUES (?, ?, ?)", ("bad", "{broken", 1.0))
+    conn.commit()
+    conn.close()
+    b2 = SQLiteBackend(path)
+    with pytest.raises(StorageError, match="cannot decode"):
+        dict(b2.load())
+    b2.close()
+
+
+def test_sqlite_backend_close_releases(tmp_path: Path) -> None:
+    """close 后连接关闭，所有操作抛 StorageError（包装 ProgrammingError）。"""
+    path = str(tmp_path / "state.db")
+    b = SQLiteBackend(path)
+    b.save("a", 1)
+    b.close()
+    with pytest.raises(StorageError, match="cannot read"):
+        b.has("a")
+    with pytest.raises(StorageError, match="cannot read"):
+        b.get("a")
+    with pytest.raises(StorageError, match="cannot write"):
+        b.save("b", 2)
+    with pytest.raises(StorageError, match="cannot iterate"):
+        dict(b.load())
+    with pytest.raises(StorageError, match="cannot clear"):
+        b.clear()
+    with pytest.raises(StorageError, match="cannot flush"):
+        b.flush()
+
+
+def test_sqlite_backend_corrupt_file_raises(tmp_path: Path) -> None:
+    """打开非 SQLite 格式的文件应抛 StorageError。"""
+    path = tmp_path / "not_a_db.db"
+    _ = path.write_bytes(b"not a sqlite database file content")
+    with pytest.raises(StorageError, match="cannot open sqlite state file"):
+        _ = SQLiteBackend(str(path))
+
+
 # ---------------------------------------------------------------------- #
 # resolve_backend
 # ---------------------------------------------------------------------- #