chore: 发布 pyflowx 0.2.10,新增性能计时器与多项重构

1. 新增 perf_timer 工具与配套测试用例
2. 重构任务条件跳过逻辑,优化失败条件展示
3. 重构 Graph 子图生成逻辑,提取公共依赖修剪函数
4. 重构条件模块,统一条件名称与失败原因获取逻辑
5. 重构存储后端,提取 TTL 共享逻辑并优化实现
6. 重构执行器模块,使用 Mixin 复用代码,拆分任务与层执行逻辑
7. 删除冗余的 which 命令测试文件
8. 更新依赖锁文件
This commit is contained in:
2026-06-27 20:15:35 +08:00
parent c3b86b603d
commit d58fc5536e
9 changed files with 701 additions and 607 deletions
+27 -13
View File
@@ -42,6 +42,19 @@ def _static(predicate: Callable[[], bool], name: str) -> Condition:
return _cond return _cond
def _cond_reason(cond: Condition) -> str | list[str] | None:
"""获取条件的失败原因:优先返回 ``_reason``,否则返回 ``__name__``。"""
reason = getattr(cond, "_reason", None)
if reason is not None:
return reason
return getattr(cond, "__name__", repr(cond))
def _cond_name(cond: Condition) -> str:
"""获取条件的可读名称。"""
return getattr(cond, "__name__", repr(cond))
# ---------------------------------------------------------------------- # # ---------------------------------------------------------------------- #
# 模块级静态条件常量 # 模块级静态条件常量
# ---------------------------------------------------------------------- # # ---------------------------------------------------------------------- #
@@ -61,21 +74,25 @@ class BuiltinConditions:
# ------------------------------------------------------------------ # # ------------------------------------------------------------------ #
# 静态条件 # 静态条件
# ------------------------------------------------------------------ # # ------------------------------------------------------------------ #
@staticmethod
def IS_WINDOWS() -> Condition: def IS_WINDOWS() -> Condition:
"""检查是否为 Windows 平台.""" """检查是否为 Windows 平台."""
return _static(lambda: Constants.IS_WINDOWS, "IS_WINDOWS") return IS_WINDOWS
@staticmethod
def IS_LINUX() -> Condition: def IS_LINUX() -> Condition:
"""检查是否为 Linux 平台.""" """检查是否为 Linux 平台."""
return _static(lambda: Constants.IS_LINUX, "IS_LINUX") return IS_LINUX
@staticmethod
def IS_MACOS() -> Condition: def IS_MACOS() -> Condition:
"""检查是否为 macOS 平台.""" """检查是否为 macOS 平台."""
return _static(lambda: Constants.IS_MACOS, "IS_MACOS") return IS_MACOS
@staticmethod
def IS_POSIX() -> Condition: def IS_POSIX() -> Condition:
"""检查是否为 POSIX 平台.""" """检查是否为 POSIX 平台."""
return _static(lambda: Constants.IS_POSIX, "IS_POSIX") return IS_POSIX
@staticmethod @staticmethod
def PYTHON_VERSION(major: int, minor: int | None = None) -> Condition: def PYTHON_VERSION(major: int, minor: int | None = None) -> Condition:
@@ -214,12 +231,12 @@ class BuiltinConditions:
result = condition(ctx) result = condition(ctx)
if result: if result:
# inner 为 True 时 NOT 会失败,记录 inner 的具体原因 # inner 为 True 时 NOT 会失败,记录 inner 的具体原因
inner_reason = getattr(condition, "_reason", None) inner_reason = _cond_reason(condition)
if inner_reason is not None: if inner_reason is not None:
_cond._reason = inner_reason # type: ignore[attr-defined] _cond._reason = inner_reason # type: ignore[attr-defined]
return not result return not result
_cond.__name__ = f"NOT({getattr(condition, '__name__', repr(condition))})" _cond.__name__ = f"NOT({_cond_name(condition)})"
return _cond return _cond
@staticmethod @staticmethod
@@ -229,8 +246,7 @@ class BuiltinConditions:
def _cond(ctx: Context) -> bool: def _cond(ctx: Context) -> bool:
return all(c(ctx) for c in conditions) return all(c(ctx) for c in conditions)
names = [getattr(c, "__name__", repr(c)) for c in conditions] _cond.__name__ = f"AND({', '.join(_cond_name(c) for c in conditions)})"
_cond.__name__ = f"AND({', '.join(names)})"
return _cond return _cond
@staticmethod @staticmethod
@@ -241,14 +257,12 @@ class BuiltinConditions:
matched: list[str] = [] matched: list[str] = []
for c in conditions: for c in conditions:
if c(ctx): if c(ctx):
matched.append( reason = _cond_reason(c)
getattr(c, "_reason", None) or getattr(c, "__name__", repr(c)), matched.append(reason if isinstance(reason, str) else str(reason))
)
if matched: if matched:
_cond._reason = matched # type: ignore[attr-defined] _cond._reason = matched # type: ignore[attr-defined]
return True return True
return False return False
names = [getattr(c, "__name__", repr(c)) for c in conditions] _cond.__name__ = f"OR({', '.join(_cond_name(c) for c in conditions)})"
_cond.__name__ = f"OR({', '.join(names)})"
return _cond return _cond
+423 -468
View File
File diff suppressed because it is too large Load Diff
+19 -16
View File
@@ -49,6 +49,15 @@ class GraphDefaults:
verbose: bool = False verbose: bool = False
def _prune_deps(spec: TaskSpec[Any], keep: Callable[[str], bool]) -> TaskSpec[Any]:
"""返回新 spec,其 ``depends_on`` / ``soft_depends_on`` 仅保留 ``keep(dep)`` 为真的依赖。"""
return replace(
spec,
depends_on=tuple(d for d in spec.depends_on if keep(d)),
soft_depends_on=tuple(d for d in spec.soft_depends_on if keep(d)),
)
@dataclass @dataclass
class Graph: class Graph:
"""校验后的有向无环任务图。 """校验后的有向无环任务图。
@@ -225,16 +234,13 @@ class Graph:
def subgraph(self, tags: Iterable[str]) -> Graph: def subgraph(self, tags: Iterable[str]) -> Graph:
"""返回仅包含匹配任意标签的任务的新图。依赖边被修剪。""" """返回仅包含匹配任意标签的任务的新图。依赖边被修剪。"""
wanted: set[str] = set(tags) wanted: set[str] = set(tags)
kept: list[TaskSpec[Any]] = []
for spec in self.specs.values(): def _dep_kept(dep: str) -> bool:
if wanted & set(spec.tags): return dep in self.specs and bool(wanted & set(self.specs[dep].tags))
pruned_deps = tuple(
d for d in spec.depends_on if d in self.specs and (wanted & set(self.specs[d].tags)) kept: list[TaskSpec[Any]] = [
) _prune_deps(spec, _dep_kept) for spec in self.specs.values() if wanted & set(spec.tags)
pruned_soft = tuple( ]
d for d in spec.soft_depends_on if d in self.specs and (wanted & set(self.specs[d].tags))
)
kept.append(replace(spec, depends_on=pruned_deps, soft_depends_on=pruned_soft))
return Graph.from_specs(kept, defaults=self.defaults) return Graph.from_specs(kept, defaults=self.defaults)
def subgraph_by_names(self, names: Iterable[str]) -> Graph: def subgraph_by_names(self, names: Iterable[str]) -> Graph:
@@ -243,12 +249,9 @@ class Graph:
for n in wanted: for n in wanted:
if n not in self.specs: if n not in self.specs:
raise KeyError(f"Unknown task name: {n!r}") raise KeyError(f"Unknown task name: {n!r}")
kept: list[TaskSpec[Any]] = [] kept: list[TaskSpec[Any]] = [
for spec in self.specs.values(): _prune_deps(spec, lambda d: d in wanted) for spec in self.specs.values() if spec.name in wanted
if spec.name in wanted: ]
pruned_deps = tuple(d for d in spec.depends_on if d in wanted)
pruned_soft = tuple(d for d in spec.soft_depends_on if d in wanted)
kept.append(replace(spec, depends_on=pruned_deps, soft_depends_on=pruned_soft))
return Graph.from_specs(kept, defaults=self.defaults) return Graph.from_specs(kept, defaults=self.defaults)
# ------------------------------------------------------------------ # # ------------------------------------------------------------------ #
+110 -40
View File
@@ -17,6 +17,7 @@ import json
import sys import sys
import time import time
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections.abc import Iterator
from pathlib import Path from pathlib import Path
from typing import Any, Mapping from typing import Any, Mapping
@@ -55,7 +56,74 @@ class StateBackend(ABC):
"""清除所有存储状态。""" """清除所有存储状态。"""
class MemoryBackend(StateBackend): class _TTLStateBackendMixin(StateBackend):
"""TTL 状态后端共享逻辑。
将 ``has`` / ``get`` / ``load`` / ``save`` / ``clear`` 的统一实现
委托给四个原始存取原语::meth:`_get_raw`、:meth:`_put_raw`、
:meth:`_iter_raw`、:meth:`_clear_raw`,并基于 :meth:`_now` 与
``self._ttl`` 提供统一的过期判断 :meth:`_is_expired`。
子类需设置 ``self._ttl`` 并实现上述四个原语;如需自定义时间源
(如 ``time.monotonic``)可覆盖 :meth:`_now`。
"""
_ttl: float | None
# ---- 原语:由子类实现 ---- #
@abstractmethod
def _get_raw(self, key: str) -> tuple[Any, float] | None:
"""返回 ``(value, ts)``;键不存在时返回 ``None``。"""
@abstractmethod
def _put_raw(self, key: str, value: Any, ts: float) -> None:
"""写入一条记录。"""
@abstractmethod
def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
"""迭代所有记录(不做过期过滤),yield ``(key, value, ts)``。"""
@abstractmethod
def _clear_raw(self) -> None:
"""清空所有记录。"""
# ---- 共享实现 ---- #
def _now(self) -> float:
"""当前时间戳,默认为 wall-clock 秒。"""
return time.time()
def _is_expired(self, ts: float) -> bool:
"""时间戳 ``ts`` 是否已过期。"""
if self._ttl is None:
return False
return (self._now() - ts) > self._ttl
@override
def load(self) -> Mapping[str, Any]:
return {k: v for k, v, ts in self._iter_raw() if not self._is_expired(ts)}
@override
def save(self, key: str, value: Any) -> None:
self._put_raw(key, value, self._now())
@override
def has(self, key: str) -> bool:
entry = self._get_raw(key)
return entry is not None and not self._is_expired(entry[1])
@override
def get(self, key: str) -> Any:
entry = self._get_raw(key)
if entry is None or self._is_expired(entry[1]):
raise KeyError(key)
return entry[0]
@override
def clear(self) -> None:
self._clear_raw()
class MemoryBackend(_TTLStateBackendMixin):
"""进程内 dict 后端。进程退出即丢失。 """进程内 dict 后端。进程退出即丢失。
Parameters Parameters
@@ -70,35 +138,35 @@ class MemoryBackend(StateBackend):
self._ttl = ttl self._ttl = ttl
@override @override
def load(self) -> Mapping[str, Any]: def _now(self) -> float:
return {k: v for k, (v, _ts) in self._store.items() if not self._expired(k)} return time.monotonic()
@override @override
def save(self, key: str, value: Any) -> None: def _get_raw(self, key: str) -> tuple[Any, float] | None:
self._store[key] = (value, time.monotonic()) return self._store.get(key)
@override @override
def has(self, key: str) -> bool: def _put_raw(self, key: str, value: Any, ts: float) -> None:
return key in self._store and not self._expired(key) self._store[key] = (value, ts)
@override @override
def get(self, key: str) -> Any: def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
if key not in self._store or self._expired(key): for k, (v, ts) in self._store.items():
raise KeyError(key) yield k, v, ts
return self._store[key][0]
@override @override
def clear(self) -> None: def _clear_raw(self) -> None:
self._store.clear() self._store.clear()
def _expired(self, key: str) -> bool: def _expired(self, key: str) -> bool:
if self._ttl is None or key not in self._store: """键是否已过期(兼容旧测试 API)。"""
entry = self._get_raw(key)
if entry is None:
return False return False
_value, ts = self._store[key] return self._is_expired(entry[1])
return (time.monotonic() - ts) > self._ttl
class JSONBackend(StateBackend): class JSONBackend(_TTLStateBackendMixin):
"""基于文件的 JSON 存储,用于跨进程续跑。 """基于文件的 JSON 存储,用于跨进程续跑。
存储格式:``{key: {"value": v, "ts": epoch_seconds}}``。 存储格式:``{key: {"value": v, "ts": epoch_seconds}}``。
@@ -144,17 +212,30 @@ class JSONBackend(StateBackend):
except (OSError, TypeError) as exc: except (OSError, TypeError) as exc:
raise StorageError(f"cannot write state file {self._path!r}", exc) from exc raise StorageError(f"cannot write state file {self._path!r}", exc) from exc
def _now(self) -> float: @override
return time.time() def _get_raw(self, key: str) -> tuple[Any, float] | None:
entry = self._store.get(key)
def _expired(self, entry: dict[str, Any]) -> bool: if entry is None:
if self._ttl is None: return None
return False return entry["value"], float(entry.get("ts", 0))
return (self._now() - float(entry.get("ts", 0))) > self._ttl
@override @override
def load(self) -> Mapping[str, Any]: def _put_raw(self, key: str, value: Any, ts: float) -> None:
return {k: v["value"] for k, v in self._store.items() if not self._expired(v)} self._store[key] = {"value": value, "ts": ts}
@override
def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
for k, entry in self._store.items():
yield k, entry["value"], float(entry.get("ts", 0))
@override
def _clear_raw(self) -> None:
self._store.clear()
@override
def clear(self) -> None:
super().clear()
self._flush()
@override @override
def save(self, key: str, value: Any) -> None: def save(self, key: str, value: Any) -> None:
@@ -162,23 +243,12 @@ class JSONBackend(StateBackend):
_ = json.dumps(value) _ = json.dumps(value)
except (TypeError, ValueError) as exc: except (TypeError, ValueError) as exc:
raise StorageError(f"result of key {key!r} is not JSON-serialisable", exc) from exc raise StorageError(f"result of key {key!r} is not JSON-serialisable", exc) from exc
self._store[key] = {"value": value, "ts": self._now()} super().save(key, value)
self._flush() self._flush()
@override def _expired(self, entry: Mapping[str, Any]) -> bool:
def has(self, key: str) -> bool: """带元数据的条目是否已过期(兼容旧测试 API)。"""
return key in self._store and not self._expired(self._store[key]) return self._is_expired(float(entry.get("ts", 0)))
@override
def get(self, key: str) -> Any:
if key not in self._store or self._expired(self._store[key]):
raise KeyError(key)
return self._store[key]["value"]
@override
def clear(self) -> None:
self._store.clear()
self._flush()
def resolve_backend(backend: StateBackend | None) -> StateBackend: def resolve_backend(backend: StateBackend | None) -> StateBackend:
+10 -3
View File
@@ -74,6 +74,13 @@ Condition = Callable[[Context], bool]
CacheKeyFn = Callable[[Context], str] CacheKeyFn = Callable[[Context], str]
def _format_skip_reason(failed_conditions: list[str]) -> str:
"""格式化跳过原因:≤2 个全展示,>2 个仅展示前 2 个并附总数。"""
if len(failed_conditions) <= 2:
return f"条件不满足: {', '.join(failed_conditions)}"
return f"条件不满足: {', '.join(failed_conditions[:2])}{len(failed_conditions)}个条件"
# ---------------------------------------------------------------------- # # ---------------------------------------------------------------------- #
# 重试策略 # 重试策略
# ---------------------------------------------------------------------- # # ---------------------------------------------------------------------- #
@@ -315,6 +322,7 @@ class TaskSpec(Generic[T]):
------- -------
(should_run, skip_reason) (should_run, skip_reason)
``should_run`` 为 False 时 ``skip_reason`` 描述跳过原因。 ``should_run`` 为 False 时 ``skip_reason`` 描述跳过原因。
失败条件超过 2 个时仅展示前 2 个并附总数。
""" """
# 逐个求值条件,记录失败项。 # 逐个求值条件,记录失败项。
failed_conditions: list[str] = [] failed_conditions: list[str] = []
@@ -323,8 +331,7 @@ class TaskSpec(Generic[T]):
ok = condition(context) ok = condition(context)
except Exception: except Exception:
ok = False ok = False
name = getattr(condition, "__name__", None) or "匿名条件(执行错误)" failed_conditions.append("匿名条件(执行错误)")
failed_conditions.append(name)
continue continue
if not ok: if not ok:
reason = getattr(condition, "_reason", None) reason = getattr(condition, "_reason", None)
@@ -336,7 +343,7 @@ class TaskSpec(Generic[T]):
failed_conditions.append(getattr(condition, "__name__", None) or "匿名条件") failed_conditions.append(getattr(condition, "__name__", None) or "匿名条件")
if failed_conditions: if failed_conditions:
return False, f"条件不满足: {', '.join(failed_conditions)}" return False, _format_skip_reason(failed_conditions)
if self.skip_if_missing and not self._is_cmd_available(): if self.skip_if_missing and not self._is_cmd_available():
cmd_name = self.cmd[0] if isinstance(self.cmd, list) and self.cmd else "unknown" cmd_name = self.cmd[0] if isinstance(self.cmd, list) and self.cmd else "unknown"
+70
View File
@@ -0,0 +1,70 @@
"""常用工具函数."""
__all__ = ["perf_timer"]
import functools
import logging
import time
from collections import defaultdict
from typing import Callable, ParamSpec, TypedDict
from typing_extensions import TypeVar
P = ParamSpec("P")
R = TypeVar("R")
class _PerformanceMetrics(TypedDict):
"""性能指标."""
count: int
total_time: float
_perf_metrics: defaultdict[str, _PerformanceMetrics] = defaultdict(
lambda: _PerformanceMetrics(
count=0,
total_time=0.0,
)
)
def perf_timer(unit: str = "ms", precision: int = 4, report: bool = False):
"""性能计时器装饰器."""
scale: dict[str, float] = {
"s": 1.0,
"ms": 1000.0,
"us": 1000000.0,
}
def decorator(func: Callable[P, R]) -> Callable[P, R]:
@functools.wraps(func)
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
start_time = time.time()
result = func(*args, **kwargs)
end_time = time.time()
_perf_metrics[func.__name__]["count"] += 1
_perf_metrics[func.__name__]["total_time"] += (end_time - start_time) * scale[unit]
if not report:
logging.info(
f"{func.__name__} {unit}: {_perf_metrics[func.__name__]['total_time']:.{precision}f}{unit}"
)
return result
return wrapper
if report:
import atexit
logging.basicConfig(level=logging.INFO)
logging.info(f"Performance metrics report enabled with unit {unit} and precision {precision}")
@atexit.register
def _() -> None:
for name, metrics in _perf_metrics.items():
logging.info(f"{name}: {metrics['count']} times, {metrics['total_time']:.{precision}f}{unit}")
return decorator
-66
View File
@@ -1,66 +0,0 @@
"""Tests for cli.which module."""
from __future__ import annotations
import shutil
from unittest.mock import patch
import pytest
import pyflowx as px
from pyflowx.cli import which
# ---------------------------------------------------------------------- #
# main function
# ---------------------------------------------------------------------- #
class TestMain:
"""Test main function."""
def test_main_with_single_command(self) -> None:
"""main() should handle single command argument."""
with patch("sys.argv", ["which", "python"]), patch.object(
shutil, "which", return_value="/usr/bin/python"
), patch.object(px, "run") as mock_run:
which.main()
# Should create a graph with one task
assert mock_run.called
graph = mock_run.call_args[0][0]
assert isinstance(graph, px.Graph)
def test_main_with_multiple_commands(self) -> None:
"""main() should handle multiple command arguments."""
with patch("sys.argv", ["which", "python", "pip", "node"]), patch.object(
shutil, "which", return_value="/usr/bin/cmd"
), patch.object(px, "run") as mock_run:
which.main()
# Should create a graph with three tasks
assert mock_run.called
graph = mock_run.call_args[0][0]
assert isinstance(graph, px.Graph)
def test_main_with_no_args_shows_help(self) -> None:
"""main() with no args should show help and exit."""
with patch("sys.argv", ["which"]), pytest.raises(SystemExit) as exc_info:
which.main()
assert exc_info.value.code == 2
def test_main_creates_task_specs_with_correct_names(self) -> None:
"""main() should create TaskSpecs with correct names."""
with patch("sys.argv", ["which", "git", "npm"]), patch.object(
shutil, "which", return_value="/usr/bin/cmd"
), patch.object(px, "run") as mock_run:
which.main()
graph = mock_run.call_args[0][0]
# Check that task names are correct
task_names = list(graph.all_specs().keys())
assert "which_git" in task_names
assert "which_npm" in task_names
def test_main_uses_thread_strategy(self) -> None:
"""main() should use thread strategy."""
with patch("sys.argv", ["which", "python"]), patch.object(
shutil, "which", return_value="/usr/bin/python"
), patch.object(px, "run") as mock_run:
which.main()
assert mock_run.call_args[1]["strategy"] == "thread"
+41
View File
@@ -0,0 +1,41 @@
import time
import pytest
from pytest_mock import MockerFixture
from pyflowx.utils import _perf_metrics, perf_timer
@pytest.fixture(autouse=True)
def reset_perf_metrics():
"""重置性能指标."""
_perf_metrics.clear()
class TestPerformanceTimer:
def test_perf_timer(self):
@perf_timer()
def test_func():
time.sleep(0.1)
test_func()
assert _perf_metrics["test_func"] is not None
assert _perf_metrics["test_func"]["count"] == 1
assert _perf_metrics["test_func"]["total_time"] >= 0.1
def test_perf_timer_report(self, mocker: MockerFixture):
mock_log = mocker.patch("logging.info")
@perf_timer(report=True, unit="ms", precision=3)
def test_func():
time.sleep(0.1)
test_func()
assert _perf_metrics["test_func"] is not None
assert _perf_metrics["test_func"]["count"] == 1
assert _perf_metrics["test_func"]["total_time"] >= 0.1
assert mock_log.call_count == 1
Generated
+1 -1
View File
@@ -5603,7 +5603,7 @@ pycountry = [
[[package]] [[package]]
name = "pyflowx" name = "pyflowx"
version = "0.2.9" version = "0.2.10"
source = { editable = "." } source = { editable = "." }
dependencies = [ dependencies = [
{ name = "graphlib-backport", marker = "python_full_version < '3.9'" }, { name = "graphlib-backport", marker = "python_full_version < '3.9'" },