chore: 发布 pyflowx 0.2.10,新增性能计时器与多项重构

1. 新增 perf_timer 工具与配套测试用例
2. 重构任务条件跳过逻辑,优化失败条件展示
3. 重构 Graph 子图生成逻辑,提取公共依赖修剪函数
4. 重构条件模块,统一条件名称与失败原因获取逻辑
5. 重构存储后端,提取 TTL 共享逻辑并优化实现
6. 重构执行器模块,使用 Mixin 复用代码,拆分任务与层执行逻辑
7. 删除冗余的 which 命令测试文件
8. 更新依赖锁文件
This commit is contained in:
2026-06-27 20:15:35 +08:00
parent c3b86b603d
commit d58fc5536e
9 changed files with 701 additions and 607 deletions
+27 -13
View File
@@ -42,6 +42,19 @@ def _static(predicate: Callable[[], bool], name: str) -> Condition:
return _cond
def _cond_reason(cond: Condition) -> str | list[str] | None:
"""获取条件的失败原因:优先返回 ``_reason``,否则返回 ``__name__``。"""
reason = getattr(cond, "_reason", None)
if reason is not None:
return reason
return getattr(cond, "__name__", repr(cond))
def _cond_name(cond: Condition) -> str:
"""获取条件的可读名称。"""
return getattr(cond, "__name__", repr(cond))
# ---------------------------------------------------------------------- #
# 模块级静态条件常量
# ---------------------------------------------------------------------- #
@@ -61,21 +74,25 @@ class BuiltinConditions:
# ------------------------------------------------------------------ #
# 静态条件
# ------------------------------------------------------------------ #
@staticmethod
def IS_WINDOWS() -> Condition:
"""检查是否为 Windows 平台."""
return _static(lambda: Constants.IS_WINDOWS, "IS_WINDOWS")
return IS_WINDOWS
@staticmethod
def IS_LINUX() -> Condition:
"""检查是否为 Linux 平台."""
return _static(lambda: Constants.IS_LINUX, "IS_LINUX")
return IS_LINUX
@staticmethod
def IS_MACOS() -> Condition:
"""检查是否为 macOS 平台."""
return _static(lambda: Constants.IS_MACOS, "IS_MACOS")
return IS_MACOS
@staticmethod
def IS_POSIX() -> Condition:
"""检查是否为 POSIX 平台."""
return _static(lambda: Constants.IS_POSIX, "IS_POSIX")
return IS_POSIX
@staticmethod
def PYTHON_VERSION(major: int, minor: int | None = None) -> Condition:
@@ -214,12 +231,12 @@ class BuiltinConditions:
result = condition(ctx)
if result:
# inner 为 True 时 NOT 会失败,记录 inner 的具体原因
inner_reason = getattr(condition, "_reason", None)
inner_reason = _cond_reason(condition)
if inner_reason is not None:
_cond._reason = inner_reason # type: ignore[attr-defined]
return not result
_cond.__name__ = f"NOT({getattr(condition, '__name__', repr(condition))})"
_cond.__name__ = f"NOT({_cond_name(condition)})"
return _cond
@staticmethod
@@ -229,8 +246,7 @@ class BuiltinConditions:
def _cond(ctx: Context) -> bool:
return all(c(ctx) for c in conditions)
names = [getattr(c, "__name__", repr(c)) for c in conditions]
_cond.__name__ = f"AND({', '.join(names)})"
_cond.__name__ = f"AND({', '.join(_cond_name(c) for c in conditions)})"
return _cond
@staticmethod
@@ -241,14 +257,12 @@ class BuiltinConditions:
matched: list[str] = []
for c in conditions:
if c(ctx):
matched.append(
getattr(c, "_reason", None) or getattr(c, "__name__", repr(c)),
)
reason = _cond_reason(c)
matched.append(reason if isinstance(reason, str) else str(reason))
if matched:
_cond._reason = matched # type: ignore[attr-defined]
return True
return False
names = [getattr(c, "__name__", repr(c)) for c in conditions]
_cond.__name__ = f"OR({', '.join(names)})"
_cond.__name__ = f"OR({', '.join(_cond_name(c) for c in conditions)})"
return _cond
+423 -468
View File
File diff suppressed because it is too large Load Diff
+19 -16
View File
@@ -49,6 +49,15 @@ class GraphDefaults:
verbose: bool = False
def _prune_deps(spec: TaskSpec[Any], keep: Callable[[str], bool]) -> TaskSpec[Any]:
"""返回新 spec,其 ``depends_on`` / ``soft_depends_on`` 仅保留 ``keep(dep)`` 为真的依赖。"""
return replace(
spec,
depends_on=tuple(d for d in spec.depends_on if keep(d)),
soft_depends_on=tuple(d for d in spec.soft_depends_on if keep(d)),
)
@dataclass
class Graph:
"""校验后的有向无环任务图。
@@ -225,16 +234,13 @@ class Graph:
def subgraph(self, tags: Iterable[str]) -> Graph:
"""返回仅包含匹配任意标签的任务的新图。依赖边被修剪。"""
wanted: set[str] = set(tags)
kept: list[TaskSpec[Any]] = []
for spec in self.specs.values():
if wanted & set(spec.tags):
pruned_deps = tuple(
d for d in spec.depends_on if d in self.specs and (wanted & set(self.specs[d].tags))
)
pruned_soft = tuple(
d for d in spec.soft_depends_on if d in self.specs and (wanted & set(self.specs[d].tags))
)
kept.append(replace(spec, depends_on=pruned_deps, soft_depends_on=pruned_soft))
def _dep_kept(dep: str) -> bool:
return dep in self.specs and bool(wanted & set(self.specs[dep].tags))
kept: list[TaskSpec[Any]] = [
_prune_deps(spec, _dep_kept) for spec in self.specs.values() if wanted & set(spec.tags)
]
return Graph.from_specs(kept, defaults=self.defaults)
def subgraph_by_names(self, names: Iterable[str]) -> Graph:
@@ -243,12 +249,9 @@ class Graph:
for n in wanted:
if n not in self.specs:
raise KeyError(f"Unknown task name: {n!r}")
kept: list[TaskSpec[Any]] = []
for spec in self.specs.values():
if spec.name in wanted:
pruned_deps = tuple(d for d in spec.depends_on if d in wanted)
pruned_soft = tuple(d for d in spec.soft_depends_on if d in wanted)
kept.append(replace(spec, depends_on=pruned_deps, soft_depends_on=pruned_soft))
kept: list[TaskSpec[Any]] = [
_prune_deps(spec, lambda d: d in wanted) for spec in self.specs.values() if spec.name in wanted
]
return Graph.from_specs(kept, defaults=self.defaults)
# ------------------------------------------------------------------ #
+110 -40
View File
@@ -17,6 +17,7 @@ import json
import sys
import time
from abc import ABC, abstractmethod
from collections.abc import Iterator
from pathlib import Path
from typing import Any, Mapping
@@ -55,7 +56,74 @@ class StateBackend(ABC):
"""清除所有存储状态。"""
class MemoryBackend(StateBackend):
class _TTLStateBackendMixin(StateBackend):
"""TTL 状态后端共享逻辑。
将 ``has`` / ``get`` / ``load`` / ``save`` / ``clear`` 的统一实现
委托给四个原始存取原语::meth:`_get_raw`、:meth:`_put_raw`、
:meth:`_iter_raw`、:meth:`_clear_raw`,并基于 :meth:`_now` 与
``self._ttl`` 提供统一的过期判断 :meth:`_is_expired`。
子类需设置 ``self._ttl`` 并实现上述四个原语;如需自定义时间源
(如 ``time.monotonic``)可覆盖 :meth:`_now`。
"""
_ttl: float | None
# ---- 原语:由子类实现 ---- #
@abstractmethod
def _get_raw(self, key: str) -> tuple[Any, float] | None:
"""返回 ``(value, ts)``;键不存在时返回 ``None``。"""
@abstractmethod
def _put_raw(self, key: str, value: Any, ts: float) -> None:
"""写入一条记录。"""
@abstractmethod
def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
"""迭代所有记录(不做过期过滤),yield ``(key, value, ts)``。"""
@abstractmethod
def _clear_raw(self) -> None:
"""清空所有记录。"""
# ---- 共享实现 ---- #
def _now(self) -> float:
"""当前时间戳,默认为 wall-clock 秒。"""
return time.time()
def _is_expired(self, ts: float) -> bool:
"""时间戳 ``ts`` 是否已过期。"""
if self._ttl is None:
return False
return (self._now() - ts) > self._ttl
@override
def load(self) -> Mapping[str, Any]:
return {k: v for k, v, ts in self._iter_raw() if not self._is_expired(ts)}
@override
def save(self, key: str, value: Any) -> None:
self._put_raw(key, value, self._now())
@override
def has(self, key: str) -> bool:
entry = self._get_raw(key)
return entry is not None and not self._is_expired(entry[1])
@override
def get(self, key: str) -> Any:
entry = self._get_raw(key)
if entry is None or self._is_expired(entry[1]):
raise KeyError(key)
return entry[0]
@override
def clear(self) -> None:
self._clear_raw()
class MemoryBackend(_TTLStateBackendMixin):
"""进程内 dict 后端。进程退出即丢失。
Parameters
@@ -70,35 +138,35 @@ class MemoryBackend(StateBackend):
self._ttl = ttl
@override
def load(self) -> Mapping[str, Any]:
return {k: v for k, (v, _ts) in self._store.items() if not self._expired(k)}
def _now(self) -> float:
return time.monotonic()
@override
def save(self, key: str, value: Any) -> None:
self._store[key] = (value, time.monotonic())
def _get_raw(self, key: str) -> tuple[Any, float] | None:
return self._store.get(key)
@override
def has(self, key: str) -> bool:
return key in self._store and not self._expired(key)
def _put_raw(self, key: str, value: Any, ts: float) -> None:
self._store[key] = (value, ts)
@override
def get(self, key: str) -> Any:
if key not in self._store or self._expired(key):
raise KeyError(key)
return self._store[key][0]
def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
for k, (v, ts) in self._store.items():
yield k, v, ts
@override
def clear(self) -> None:
def _clear_raw(self) -> None:
self._store.clear()
def _expired(self, key: str) -> bool:
if self._ttl is None or key not in self._store:
"""键是否已过期(兼容旧测试 API)。"""
entry = self._get_raw(key)
if entry is None:
return False
_value, ts = self._store[key]
return (time.monotonic() - ts) > self._ttl
return self._is_expired(entry[1])
class JSONBackend(StateBackend):
class JSONBackend(_TTLStateBackendMixin):
"""基于文件的 JSON 存储,用于跨进程续跑。
存储格式:``{key: {"value": v, "ts": epoch_seconds}}``。
@@ -144,17 +212,30 @@ class JSONBackend(StateBackend):
except (OSError, TypeError) as exc:
raise StorageError(f"cannot write state file {self._path!r}", exc) from exc
def _now(self) -> float:
return time.time()
def _expired(self, entry: dict[str, Any]) -> bool:
if self._ttl is None:
return False
return (self._now() - float(entry.get("ts", 0))) > self._ttl
@override
def _get_raw(self, key: str) -> tuple[Any, float] | None:
entry = self._store.get(key)
if entry is None:
return None
return entry["value"], float(entry.get("ts", 0))
@override
def load(self) -> Mapping[str, Any]:
return {k: v["value"] for k, v in self._store.items() if not self._expired(v)}
def _put_raw(self, key: str, value: Any, ts: float) -> None:
self._store[key] = {"value": value, "ts": ts}
@override
def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
for k, entry in self._store.items():
yield k, entry["value"], float(entry.get("ts", 0))
@override
def _clear_raw(self) -> None:
self._store.clear()
@override
def clear(self) -> None:
super().clear()
self._flush()
@override
def save(self, key: str, value: Any) -> None:
@@ -162,23 +243,12 @@ class JSONBackend(StateBackend):
_ = json.dumps(value)
except (TypeError, ValueError) as exc:
raise StorageError(f"result of key {key!r} is not JSON-serialisable", exc) from exc
self._store[key] = {"value": value, "ts": self._now()}
super().save(key, value)
self._flush()
@override
def has(self, key: str) -> bool:
return key in self._store and not self._expired(self._store[key])
@override
def get(self, key: str) -> Any:
if key not in self._store or self._expired(self._store[key]):
raise KeyError(key)
return self._store[key]["value"]
@override
def clear(self) -> None:
self._store.clear()
self._flush()
def _expired(self, entry: Mapping[str, Any]) -> bool:
"""带元数据的条目是否已过期(兼容旧测试 API)。"""
return self._is_expired(float(entry.get("ts", 0)))
def resolve_backend(backend: StateBackend | None) -> StateBackend:
+10 -3
View File
@@ -74,6 +74,13 @@ Condition = Callable[[Context], bool]
CacheKeyFn = Callable[[Context], str]
def _format_skip_reason(failed_conditions: list[str]) -> str:
"""格式化跳过原因:≤2 个全展示,>2 个仅展示前 2 个并附总数。"""
if len(failed_conditions) <= 2:
return f"条件不满足: {', '.join(failed_conditions)}"
return f"条件不满足: {', '.join(failed_conditions[:2])}{len(failed_conditions)}个条件"
# ---------------------------------------------------------------------- #
# 重试策略
# ---------------------------------------------------------------------- #
@@ -315,6 +322,7 @@ class TaskSpec(Generic[T]):
-------
(should_run, skip_reason)
``should_run`` 为 False 时 ``skip_reason`` 描述跳过原因。
失败条件超过 2 个时仅展示前 2 个并附总数。
"""
# 逐个求值条件,记录失败项。
failed_conditions: list[str] = []
@@ -323,8 +331,7 @@ class TaskSpec(Generic[T]):
ok = condition(context)
except Exception:
ok = False
name = getattr(condition, "__name__", None) or "匿名条件(执行错误)"
failed_conditions.append(name)
failed_conditions.append("匿名条件(执行错误)")
continue
if not ok:
reason = getattr(condition, "_reason", None)
@@ -336,7 +343,7 @@ class TaskSpec(Generic[T]):
failed_conditions.append(getattr(condition, "__name__", None) or "匿名条件")
if failed_conditions:
return False, f"条件不满足: {', '.join(failed_conditions)}"
return False, _format_skip_reason(failed_conditions)
if self.skip_if_missing and not self._is_cmd_available():
cmd_name = self.cmd[0] if isinstance(self.cmd, list) and self.cmd else "unknown"
+70
View File
@@ -0,0 +1,70 @@
"""常用工具函数."""
__all__ = ["perf_timer"]
import functools
import logging
import time
from collections import defaultdict
from typing import Callable, ParamSpec, TypedDict
from typing_extensions import TypeVar
P = ParamSpec("P")
R = TypeVar("R")
class _PerformanceMetrics(TypedDict):
"""性能指标."""
count: int
total_time: float
_perf_metrics: defaultdict[str, _PerformanceMetrics] = defaultdict(
lambda: _PerformanceMetrics(
count=0,
total_time=0.0,
)
)
def perf_timer(unit: str = "ms", precision: int = 4, report: bool = False):
"""性能计时器装饰器."""
scale: dict[str, float] = {
"s": 1.0,
"ms": 1000.0,
"us": 1000000.0,
}
def decorator(func: Callable[P, R]) -> Callable[P, R]:
@functools.wraps(func)
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
start_time = time.time()
result = func(*args, **kwargs)
end_time = time.time()
_perf_metrics[func.__name__]["count"] += 1
_perf_metrics[func.__name__]["total_time"] += (end_time - start_time) * scale[unit]
if not report:
logging.info(
f"{func.__name__} {unit}: {_perf_metrics[func.__name__]['total_time']:.{precision}f}{unit}"
)
return result
return wrapper
if report:
import atexit
logging.basicConfig(level=logging.INFO)
logging.info(f"Performance metrics report enabled with unit {unit} and precision {precision}")
@atexit.register
def _() -> None:
for name, metrics in _perf_metrics.items():
logging.info(f"{name}: {metrics['count']} times, {metrics['total_time']:.{precision}f}{unit}")
return decorator
-66
View File
@@ -1,66 +0,0 @@
"""Tests for cli.which module."""
from __future__ import annotations
import shutil
from unittest.mock import patch
import pytest
import pyflowx as px
from pyflowx.cli import which
# ---------------------------------------------------------------------- #
# main function
# ---------------------------------------------------------------------- #
class TestMain:
"""Test main function."""
def test_main_with_single_command(self) -> None:
"""main() should handle single command argument."""
with patch("sys.argv", ["which", "python"]), patch.object(
shutil, "which", return_value="/usr/bin/python"
), patch.object(px, "run") as mock_run:
which.main()
# Should create a graph with one task
assert mock_run.called
graph = mock_run.call_args[0][0]
assert isinstance(graph, px.Graph)
def test_main_with_multiple_commands(self) -> None:
"""main() should handle multiple command arguments."""
with patch("sys.argv", ["which", "python", "pip", "node"]), patch.object(
shutil, "which", return_value="/usr/bin/cmd"
), patch.object(px, "run") as mock_run:
which.main()
# Should create a graph with three tasks
assert mock_run.called
graph = mock_run.call_args[0][0]
assert isinstance(graph, px.Graph)
def test_main_with_no_args_shows_help(self) -> None:
"""main() with no args should show help and exit."""
with patch("sys.argv", ["which"]), pytest.raises(SystemExit) as exc_info:
which.main()
assert exc_info.value.code == 2
def test_main_creates_task_specs_with_correct_names(self) -> None:
"""main() should create TaskSpecs with correct names."""
with patch("sys.argv", ["which", "git", "npm"]), patch.object(
shutil, "which", return_value="/usr/bin/cmd"
), patch.object(px, "run") as mock_run:
which.main()
graph = mock_run.call_args[0][0]
# Check that task names are correct
task_names = list(graph.all_specs().keys())
assert "which_git" in task_names
assert "which_npm" in task_names
def test_main_uses_thread_strategy(self) -> None:
"""main() should use thread strategy."""
with patch("sys.argv", ["which", "python"]), patch.object(
shutil, "which", return_value="/usr/bin/python"
), patch.object(px, "run") as mock_run:
which.main()
assert mock_run.call_args[1]["strategy"] == "thread"
+41
View File
@@ -0,0 +1,41 @@
import time
import pytest
from pytest_mock import MockerFixture
from pyflowx.utils import _perf_metrics, perf_timer
@pytest.fixture(autouse=True)
def reset_perf_metrics():
"""重置性能指标."""
_perf_metrics.clear()
class TestPerformanceTimer:
def test_perf_timer(self):
@perf_timer()
def test_func():
time.sleep(0.1)
test_func()
assert _perf_metrics["test_func"] is not None
assert _perf_metrics["test_func"]["count"] == 1
assert _perf_metrics["test_func"]["total_time"] >= 0.1
def test_perf_timer_report(self, mocker: MockerFixture):
mock_log = mocker.patch("logging.info")
@perf_timer(report=True, unit="ms", precision=3)
def test_func():
time.sleep(0.1)
test_func()
assert _perf_metrics["test_func"] is not None
assert _perf_metrics["test_func"]["count"] == 1
assert _perf_metrics["test_func"]["total_time"] >= 0.1
assert mock_log.call_count == 1
Generated
+1 -1
View File
@@ -5603,7 +5603,7 @@ pycountry = [
[[package]]
name = "pyflowx"
version = "0.2.9"
version = "0.2.10"
source = { editable = "." }
dependencies = [
{ name = "graphlib-backport", marker = "python_full_version < '3.9'" },