chore: 发布 pyflowx 0.2.10,新增性能计时器与多项重构
1. 新增 perf_timer 工具与配套测试用例 2. 重构任务条件跳过逻辑,优化失败条件展示 3. 重构 Graph 子图生成逻辑,提取公共依赖修剪函数 4. 重构条件模块,统一条件名称与失败原因获取逻辑 5. 重构存储后端,提取 TTL 共享逻辑并优化实现 6. 重构执行器模块,使用 Mixin 复用代码,拆分任务与层执行逻辑 7. 删除冗余的 which 命令测试文件 8. 更新依赖锁文件
This commit is contained in:
+27
-13
@@ -42,6 +42,19 @@ def _static(predicate: Callable[[], bool], name: str) -> Condition:
|
|||||||
return _cond
|
return _cond
|
||||||
|
|
||||||
|
|
||||||
|
def _cond_reason(cond: Condition) -> str | list[str] | None:
|
||||||
|
"""获取条件的失败原因:优先返回 ``_reason``,否则返回 ``__name__``。"""
|
||||||
|
reason = getattr(cond, "_reason", None)
|
||||||
|
if reason is not None:
|
||||||
|
return reason
|
||||||
|
return getattr(cond, "__name__", repr(cond))
|
||||||
|
|
||||||
|
|
||||||
|
def _cond_name(cond: Condition) -> str:
|
||||||
|
"""获取条件的可读名称。"""
|
||||||
|
return getattr(cond, "__name__", repr(cond))
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------- #
|
# ---------------------------------------------------------------------- #
|
||||||
# 模块级静态条件常量
|
# 模块级静态条件常量
|
||||||
# ---------------------------------------------------------------------- #
|
# ---------------------------------------------------------------------- #
|
||||||
@@ -61,21 +74,25 @@ class BuiltinConditions:
|
|||||||
# ------------------------------------------------------------------ #
|
# ------------------------------------------------------------------ #
|
||||||
# 静态条件
|
# 静态条件
|
||||||
# ------------------------------------------------------------------ #
|
# ------------------------------------------------------------------ #
|
||||||
|
@staticmethod
|
||||||
def IS_WINDOWS() -> Condition:
|
def IS_WINDOWS() -> Condition:
|
||||||
"""检查是否为 Windows 平台."""
|
"""检查是否为 Windows 平台."""
|
||||||
return _static(lambda: Constants.IS_WINDOWS, "IS_WINDOWS")
|
return IS_WINDOWS
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
def IS_LINUX() -> Condition:
|
def IS_LINUX() -> Condition:
|
||||||
"""检查是否为 Linux 平台."""
|
"""检查是否为 Linux 平台."""
|
||||||
return _static(lambda: Constants.IS_LINUX, "IS_LINUX")
|
return IS_LINUX
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
def IS_MACOS() -> Condition:
|
def IS_MACOS() -> Condition:
|
||||||
"""检查是否为 macOS 平台."""
|
"""检查是否为 macOS 平台."""
|
||||||
return _static(lambda: Constants.IS_MACOS, "IS_MACOS")
|
return IS_MACOS
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
def IS_POSIX() -> Condition:
|
def IS_POSIX() -> Condition:
|
||||||
"""检查是否为 POSIX 平台."""
|
"""检查是否为 POSIX 平台."""
|
||||||
return _static(lambda: Constants.IS_POSIX, "IS_POSIX")
|
return IS_POSIX
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def PYTHON_VERSION(major: int, minor: int | None = None) -> Condition:
|
def PYTHON_VERSION(major: int, minor: int | None = None) -> Condition:
|
||||||
@@ -214,12 +231,12 @@ class BuiltinConditions:
|
|||||||
result = condition(ctx)
|
result = condition(ctx)
|
||||||
if result:
|
if result:
|
||||||
# inner 为 True 时 NOT 会失败,记录 inner 的具体原因
|
# inner 为 True 时 NOT 会失败,记录 inner 的具体原因
|
||||||
inner_reason = getattr(condition, "_reason", None)
|
inner_reason = _cond_reason(condition)
|
||||||
if inner_reason is not None:
|
if inner_reason is not None:
|
||||||
_cond._reason = inner_reason # type: ignore[attr-defined]
|
_cond._reason = inner_reason # type: ignore[attr-defined]
|
||||||
return not result
|
return not result
|
||||||
|
|
||||||
_cond.__name__ = f"NOT({getattr(condition, '__name__', repr(condition))})"
|
_cond.__name__ = f"NOT({_cond_name(condition)})"
|
||||||
return _cond
|
return _cond
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -229,8 +246,7 @@ class BuiltinConditions:
|
|||||||
def _cond(ctx: Context) -> bool:
|
def _cond(ctx: Context) -> bool:
|
||||||
return all(c(ctx) for c in conditions)
|
return all(c(ctx) for c in conditions)
|
||||||
|
|
||||||
names = [getattr(c, "__name__", repr(c)) for c in conditions]
|
_cond.__name__ = f"AND({', '.join(_cond_name(c) for c in conditions)})"
|
||||||
_cond.__name__ = f"AND({', '.join(names)})"
|
|
||||||
return _cond
|
return _cond
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -241,14 +257,12 @@ class BuiltinConditions:
|
|||||||
matched: list[str] = []
|
matched: list[str] = []
|
||||||
for c in conditions:
|
for c in conditions:
|
||||||
if c(ctx):
|
if c(ctx):
|
||||||
matched.append(
|
reason = _cond_reason(c)
|
||||||
getattr(c, "_reason", None) or getattr(c, "__name__", repr(c)),
|
matched.append(reason if isinstance(reason, str) else str(reason))
|
||||||
)
|
|
||||||
if matched:
|
if matched:
|
||||||
_cond._reason = matched # type: ignore[attr-defined]
|
_cond._reason = matched # type: ignore[attr-defined]
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
names = [getattr(c, "__name__", repr(c)) for c in conditions]
|
_cond.__name__ = f"OR({', '.join(_cond_name(c) for c in conditions)})"
|
||||||
_cond.__name__ = f"OR({', '.join(names)})"
|
|
||||||
return _cond
|
return _cond
|
||||||
|
|||||||
+423
-468
File diff suppressed because it is too large
Load Diff
+19
-16
@@ -49,6 +49,15 @@ class GraphDefaults:
|
|||||||
verbose: bool = False
|
verbose: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
def _prune_deps(spec: TaskSpec[Any], keep: Callable[[str], bool]) -> TaskSpec[Any]:
|
||||||
|
"""返回新 spec,其 ``depends_on`` / ``soft_depends_on`` 仅保留 ``keep(dep)`` 为真的依赖。"""
|
||||||
|
return replace(
|
||||||
|
spec,
|
||||||
|
depends_on=tuple(d for d in spec.depends_on if keep(d)),
|
||||||
|
soft_depends_on=tuple(d for d in spec.soft_depends_on if keep(d)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class Graph:
|
class Graph:
|
||||||
"""校验后的有向无环任务图。
|
"""校验后的有向无环任务图。
|
||||||
@@ -225,16 +234,13 @@ class Graph:
|
|||||||
def subgraph(self, tags: Iterable[str]) -> Graph:
|
def subgraph(self, tags: Iterable[str]) -> Graph:
|
||||||
"""返回仅包含匹配任意标签的任务的新图。依赖边被修剪。"""
|
"""返回仅包含匹配任意标签的任务的新图。依赖边被修剪。"""
|
||||||
wanted: set[str] = set(tags)
|
wanted: set[str] = set(tags)
|
||||||
kept: list[TaskSpec[Any]] = []
|
|
||||||
for spec in self.specs.values():
|
def _dep_kept(dep: str) -> bool:
|
||||||
if wanted & set(spec.tags):
|
return dep in self.specs and bool(wanted & set(self.specs[dep].tags))
|
||||||
pruned_deps = tuple(
|
|
||||||
d for d in spec.depends_on if d in self.specs and (wanted & set(self.specs[d].tags))
|
kept: list[TaskSpec[Any]] = [
|
||||||
)
|
_prune_deps(spec, _dep_kept) for spec in self.specs.values() if wanted & set(spec.tags)
|
||||||
pruned_soft = tuple(
|
]
|
||||||
d for d in spec.soft_depends_on if d in self.specs and (wanted & set(self.specs[d].tags))
|
|
||||||
)
|
|
||||||
kept.append(replace(spec, depends_on=pruned_deps, soft_depends_on=pruned_soft))
|
|
||||||
return Graph.from_specs(kept, defaults=self.defaults)
|
return Graph.from_specs(kept, defaults=self.defaults)
|
||||||
|
|
||||||
def subgraph_by_names(self, names: Iterable[str]) -> Graph:
|
def subgraph_by_names(self, names: Iterable[str]) -> Graph:
|
||||||
@@ -243,12 +249,9 @@ class Graph:
|
|||||||
for n in wanted:
|
for n in wanted:
|
||||||
if n not in self.specs:
|
if n not in self.specs:
|
||||||
raise KeyError(f"Unknown task name: {n!r}")
|
raise KeyError(f"Unknown task name: {n!r}")
|
||||||
kept: list[TaskSpec[Any]] = []
|
kept: list[TaskSpec[Any]] = [
|
||||||
for spec in self.specs.values():
|
_prune_deps(spec, lambda d: d in wanted) for spec in self.specs.values() if spec.name in wanted
|
||||||
if spec.name in wanted:
|
]
|
||||||
pruned_deps = tuple(d for d in spec.depends_on if d in wanted)
|
|
||||||
pruned_soft = tuple(d for d in spec.soft_depends_on if d in wanted)
|
|
||||||
kept.append(replace(spec, depends_on=pruned_deps, soft_depends_on=pruned_soft))
|
|
||||||
return Graph.from_specs(kept, defaults=self.defaults)
|
return Graph.from_specs(kept, defaults=self.defaults)
|
||||||
|
|
||||||
# ------------------------------------------------------------------ #
|
# ------------------------------------------------------------------ #
|
||||||
|
|||||||
+110
-40
@@ -17,6 +17,7 @@ import json
|
|||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
from collections.abc import Iterator
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Mapping
|
from typing import Any, Mapping
|
||||||
|
|
||||||
@@ -55,7 +56,74 @@ class StateBackend(ABC):
|
|||||||
"""清除所有存储状态。"""
|
"""清除所有存储状态。"""
|
||||||
|
|
||||||
|
|
||||||
class MemoryBackend(StateBackend):
|
class _TTLStateBackendMixin(StateBackend):
|
||||||
|
"""TTL 状态后端共享逻辑。
|
||||||
|
|
||||||
|
将 ``has`` / ``get`` / ``load`` / ``save`` / ``clear`` 的统一实现
|
||||||
|
委托给四个原始存取原语::meth:`_get_raw`、:meth:`_put_raw`、
|
||||||
|
:meth:`_iter_raw`、:meth:`_clear_raw`,并基于 :meth:`_now` 与
|
||||||
|
``self._ttl`` 提供统一的过期判断 :meth:`_is_expired`。
|
||||||
|
|
||||||
|
子类需设置 ``self._ttl`` 并实现上述四个原语;如需自定义时间源
|
||||||
|
(如 ``time.monotonic``)可覆盖 :meth:`_now`。
|
||||||
|
"""
|
||||||
|
|
||||||
|
_ttl: float | None
|
||||||
|
|
||||||
|
# ---- 原语:由子类实现 ---- #
|
||||||
|
@abstractmethod
|
||||||
|
def _get_raw(self, key: str) -> tuple[Any, float] | None:
|
||||||
|
"""返回 ``(value, ts)``;键不存在时返回 ``None``。"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def _put_raw(self, key: str, value: Any, ts: float) -> None:
|
||||||
|
"""写入一条记录。"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
|
||||||
|
"""迭代所有记录(不做过期过滤),yield ``(key, value, ts)``。"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def _clear_raw(self) -> None:
|
||||||
|
"""清空所有记录。"""
|
||||||
|
|
||||||
|
# ---- 共享实现 ---- #
|
||||||
|
def _now(self) -> float:
|
||||||
|
"""当前时间戳,默认为 wall-clock 秒。"""
|
||||||
|
return time.time()
|
||||||
|
|
||||||
|
def _is_expired(self, ts: float) -> bool:
|
||||||
|
"""时间戳 ``ts`` 是否已过期。"""
|
||||||
|
if self._ttl is None:
|
||||||
|
return False
|
||||||
|
return (self._now() - ts) > self._ttl
|
||||||
|
|
||||||
|
@override
|
||||||
|
def load(self) -> Mapping[str, Any]:
|
||||||
|
return {k: v for k, v, ts in self._iter_raw() if not self._is_expired(ts)}
|
||||||
|
|
||||||
|
@override
|
||||||
|
def save(self, key: str, value: Any) -> None:
|
||||||
|
self._put_raw(key, value, self._now())
|
||||||
|
|
||||||
|
@override
|
||||||
|
def has(self, key: str) -> bool:
|
||||||
|
entry = self._get_raw(key)
|
||||||
|
return entry is not None and not self._is_expired(entry[1])
|
||||||
|
|
||||||
|
@override
|
||||||
|
def get(self, key: str) -> Any:
|
||||||
|
entry = self._get_raw(key)
|
||||||
|
if entry is None or self._is_expired(entry[1]):
|
||||||
|
raise KeyError(key)
|
||||||
|
return entry[0]
|
||||||
|
|
||||||
|
@override
|
||||||
|
def clear(self) -> None:
|
||||||
|
self._clear_raw()
|
||||||
|
|
||||||
|
|
||||||
|
class MemoryBackend(_TTLStateBackendMixin):
|
||||||
"""进程内 dict 后端。进程退出即丢失。
|
"""进程内 dict 后端。进程退出即丢失。
|
||||||
|
|
||||||
Parameters
|
Parameters
|
||||||
@@ -70,35 +138,35 @@ class MemoryBackend(StateBackend):
|
|||||||
self._ttl = ttl
|
self._ttl = ttl
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def load(self) -> Mapping[str, Any]:
|
def _now(self) -> float:
|
||||||
return {k: v for k, (v, _ts) in self._store.items() if not self._expired(k)}
|
return time.monotonic()
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def save(self, key: str, value: Any) -> None:
|
def _get_raw(self, key: str) -> tuple[Any, float] | None:
|
||||||
self._store[key] = (value, time.monotonic())
|
return self._store.get(key)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def has(self, key: str) -> bool:
|
def _put_raw(self, key: str, value: Any, ts: float) -> None:
|
||||||
return key in self._store and not self._expired(key)
|
self._store[key] = (value, ts)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def get(self, key: str) -> Any:
|
def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
|
||||||
if key not in self._store or self._expired(key):
|
for k, (v, ts) in self._store.items():
|
||||||
raise KeyError(key)
|
yield k, v, ts
|
||||||
return self._store[key][0]
|
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def clear(self) -> None:
|
def _clear_raw(self) -> None:
|
||||||
self._store.clear()
|
self._store.clear()
|
||||||
|
|
||||||
def _expired(self, key: str) -> bool:
|
def _expired(self, key: str) -> bool:
|
||||||
if self._ttl is None or key not in self._store:
|
"""键是否已过期(兼容旧测试 API)。"""
|
||||||
|
entry = self._get_raw(key)
|
||||||
|
if entry is None:
|
||||||
return False
|
return False
|
||||||
_value, ts = self._store[key]
|
return self._is_expired(entry[1])
|
||||||
return (time.monotonic() - ts) > self._ttl
|
|
||||||
|
|
||||||
|
|
||||||
class JSONBackend(StateBackend):
|
class JSONBackend(_TTLStateBackendMixin):
|
||||||
"""基于文件的 JSON 存储,用于跨进程续跑。
|
"""基于文件的 JSON 存储,用于跨进程续跑。
|
||||||
|
|
||||||
存储格式:``{key: {"value": v, "ts": epoch_seconds}}``。
|
存储格式:``{key: {"value": v, "ts": epoch_seconds}}``。
|
||||||
@@ -144,17 +212,30 @@ class JSONBackend(StateBackend):
|
|||||||
except (OSError, TypeError) as exc:
|
except (OSError, TypeError) as exc:
|
||||||
raise StorageError(f"cannot write state file {self._path!r}", exc) from exc
|
raise StorageError(f"cannot write state file {self._path!r}", exc) from exc
|
||||||
|
|
||||||
def _now(self) -> float:
|
@override
|
||||||
return time.time()
|
def _get_raw(self, key: str) -> tuple[Any, float] | None:
|
||||||
|
entry = self._store.get(key)
|
||||||
def _expired(self, entry: dict[str, Any]) -> bool:
|
if entry is None:
|
||||||
if self._ttl is None:
|
return None
|
||||||
return False
|
return entry["value"], float(entry.get("ts", 0))
|
||||||
return (self._now() - float(entry.get("ts", 0))) > self._ttl
|
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def load(self) -> Mapping[str, Any]:
|
def _put_raw(self, key: str, value: Any, ts: float) -> None:
|
||||||
return {k: v["value"] for k, v in self._store.items() if not self._expired(v)}
|
self._store[key] = {"value": value, "ts": ts}
|
||||||
|
|
||||||
|
@override
|
||||||
|
def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
|
||||||
|
for k, entry in self._store.items():
|
||||||
|
yield k, entry["value"], float(entry.get("ts", 0))
|
||||||
|
|
||||||
|
@override
|
||||||
|
def _clear_raw(self) -> None:
|
||||||
|
self._store.clear()
|
||||||
|
|
||||||
|
@override
|
||||||
|
def clear(self) -> None:
|
||||||
|
super().clear()
|
||||||
|
self._flush()
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def save(self, key: str, value: Any) -> None:
|
def save(self, key: str, value: Any) -> None:
|
||||||
@@ -162,23 +243,12 @@ class JSONBackend(StateBackend):
|
|||||||
_ = json.dumps(value)
|
_ = json.dumps(value)
|
||||||
except (TypeError, ValueError) as exc:
|
except (TypeError, ValueError) as exc:
|
||||||
raise StorageError(f"result of key {key!r} is not JSON-serialisable", exc) from exc
|
raise StorageError(f"result of key {key!r} is not JSON-serialisable", exc) from exc
|
||||||
self._store[key] = {"value": value, "ts": self._now()}
|
super().save(key, value)
|
||||||
self._flush()
|
self._flush()
|
||||||
|
|
||||||
@override
|
def _expired(self, entry: Mapping[str, Any]) -> bool:
|
||||||
def has(self, key: str) -> bool:
|
"""带元数据的条目是否已过期(兼容旧测试 API)。"""
|
||||||
return key in self._store and not self._expired(self._store[key])
|
return self._is_expired(float(entry.get("ts", 0)))
|
||||||
|
|
||||||
@override
|
|
||||||
def get(self, key: str) -> Any:
|
|
||||||
if key not in self._store or self._expired(self._store[key]):
|
|
||||||
raise KeyError(key)
|
|
||||||
return self._store[key]["value"]
|
|
||||||
|
|
||||||
@override
|
|
||||||
def clear(self) -> None:
|
|
||||||
self._store.clear()
|
|
||||||
self._flush()
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_backend(backend: StateBackend | None) -> StateBackend:
|
def resolve_backend(backend: StateBackend | None) -> StateBackend:
|
||||||
|
|||||||
+10
-3
@@ -74,6 +74,13 @@ Condition = Callable[[Context], bool]
|
|||||||
CacheKeyFn = Callable[[Context], str]
|
CacheKeyFn = Callable[[Context], str]
|
||||||
|
|
||||||
|
|
||||||
|
def _format_skip_reason(failed_conditions: list[str]) -> str:
|
||||||
|
"""格式化跳过原因:≤2 个全展示,>2 个仅展示前 2 个并附总数。"""
|
||||||
|
if len(failed_conditions) <= 2:
|
||||||
|
return f"条件不满足: {', '.join(failed_conditions)}"
|
||||||
|
return f"条件不满足: {', '.join(failed_conditions[:2])} 等{len(failed_conditions)}个条件"
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------- #
|
# ---------------------------------------------------------------------- #
|
||||||
# 重试策略
|
# 重试策略
|
||||||
# ---------------------------------------------------------------------- #
|
# ---------------------------------------------------------------------- #
|
||||||
@@ -315,6 +322,7 @@ class TaskSpec(Generic[T]):
|
|||||||
-------
|
-------
|
||||||
(should_run, skip_reason)
|
(should_run, skip_reason)
|
||||||
``should_run`` 为 False 时 ``skip_reason`` 描述跳过原因。
|
``should_run`` 为 False 时 ``skip_reason`` 描述跳过原因。
|
||||||
|
失败条件超过 2 个时仅展示前 2 个并附总数。
|
||||||
"""
|
"""
|
||||||
# 逐个求值条件,记录失败项。
|
# 逐个求值条件,记录失败项。
|
||||||
failed_conditions: list[str] = []
|
failed_conditions: list[str] = []
|
||||||
@@ -323,8 +331,7 @@ class TaskSpec(Generic[T]):
|
|||||||
ok = condition(context)
|
ok = condition(context)
|
||||||
except Exception:
|
except Exception:
|
||||||
ok = False
|
ok = False
|
||||||
name = getattr(condition, "__name__", None) or "匿名条件(执行错误)"
|
failed_conditions.append("匿名条件(执行错误)")
|
||||||
failed_conditions.append(name)
|
|
||||||
continue
|
continue
|
||||||
if not ok:
|
if not ok:
|
||||||
reason = getattr(condition, "_reason", None)
|
reason = getattr(condition, "_reason", None)
|
||||||
@@ -336,7 +343,7 @@ class TaskSpec(Generic[T]):
|
|||||||
failed_conditions.append(getattr(condition, "__name__", None) or "匿名条件")
|
failed_conditions.append(getattr(condition, "__name__", None) or "匿名条件")
|
||||||
|
|
||||||
if failed_conditions:
|
if failed_conditions:
|
||||||
return False, f"条件不满足: {', '.join(failed_conditions)}"
|
return False, _format_skip_reason(failed_conditions)
|
||||||
|
|
||||||
if self.skip_if_missing and not self._is_cmd_available():
|
if self.skip_if_missing and not self._is_cmd_available():
|
||||||
cmd_name = self.cmd[0] if isinstance(self.cmd, list) and self.cmd else "unknown"
|
cmd_name = self.cmd[0] if isinstance(self.cmd, list) and self.cmd else "unknown"
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
"""常用工具函数."""
|
||||||
|
|
||||||
|
__all__ = ["perf_timer"]
|
||||||
|
|
||||||
|
|
||||||
|
import functools
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
from collections import defaultdict
|
||||||
|
from typing import Callable, ParamSpec, TypedDict
|
||||||
|
|
||||||
|
from typing_extensions import TypeVar
|
||||||
|
|
||||||
|
P = ParamSpec("P")
|
||||||
|
R = TypeVar("R")
|
||||||
|
|
||||||
|
|
||||||
|
class _PerformanceMetrics(TypedDict):
|
||||||
|
"""性能指标."""
|
||||||
|
|
||||||
|
count: int
|
||||||
|
total_time: float
|
||||||
|
|
||||||
|
|
||||||
|
_perf_metrics: defaultdict[str, _PerformanceMetrics] = defaultdict(
|
||||||
|
lambda: _PerformanceMetrics(
|
||||||
|
count=0,
|
||||||
|
total_time=0.0,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def perf_timer(unit: str = "ms", precision: int = 4, report: bool = False):
|
||||||
|
"""性能计时器装饰器."""
|
||||||
|
scale: dict[str, float] = {
|
||||||
|
"s": 1.0,
|
||||||
|
"ms": 1000.0,
|
||||||
|
"us": 1000000.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
def decorator(func: Callable[P, R]) -> Callable[P, R]:
|
||||||
|
@functools.wraps(func)
|
||||||
|
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
|
||||||
|
start_time = time.time()
|
||||||
|
result = func(*args, **kwargs)
|
||||||
|
end_time = time.time()
|
||||||
|
|
||||||
|
_perf_metrics[func.__name__]["count"] += 1
|
||||||
|
_perf_metrics[func.__name__]["total_time"] += (end_time - start_time) * scale[unit]
|
||||||
|
|
||||||
|
if not report:
|
||||||
|
logging.info(
|
||||||
|
f"{func.__name__} {unit}: {_perf_metrics[func.__name__]['total_time']:.{precision}f}{unit}"
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
if report:
|
||||||
|
import atexit
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
logging.info(f"Performance metrics report enabled with unit {unit} and precision {precision}")
|
||||||
|
|
||||||
|
@atexit.register
|
||||||
|
def _() -> None:
|
||||||
|
for name, metrics in _perf_metrics.items():
|
||||||
|
logging.info(f"{name}: {metrics['count']} times, {metrics['total_time']:.{precision}f}{unit}")
|
||||||
|
|
||||||
|
return decorator
|
||||||
@@ -1,66 +0,0 @@
|
|||||||
"""Tests for cli.which module."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import shutil
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
import pyflowx as px
|
|
||||||
from pyflowx.cli import which
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------- #
|
|
||||||
# main function
|
|
||||||
# ---------------------------------------------------------------------- #
|
|
||||||
class TestMain:
|
|
||||||
"""Test main function."""
|
|
||||||
|
|
||||||
def test_main_with_single_command(self) -> None:
|
|
||||||
"""main() should handle single command argument."""
|
|
||||||
with patch("sys.argv", ["which", "python"]), patch.object(
|
|
||||||
shutil, "which", return_value="/usr/bin/python"
|
|
||||||
), patch.object(px, "run") as mock_run:
|
|
||||||
which.main()
|
|
||||||
# Should create a graph with one task
|
|
||||||
assert mock_run.called
|
|
||||||
graph = mock_run.call_args[0][0]
|
|
||||||
assert isinstance(graph, px.Graph)
|
|
||||||
|
|
||||||
def test_main_with_multiple_commands(self) -> None:
|
|
||||||
"""main() should handle multiple command arguments."""
|
|
||||||
with patch("sys.argv", ["which", "python", "pip", "node"]), patch.object(
|
|
||||||
shutil, "which", return_value="/usr/bin/cmd"
|
|
||||||
), patch.object(px, "run") as mock_run:
|
|
||||||
which.main()
|
|
||||||
# Should create a graph with three tasks
|
|
||||||
assert mock_run.called
|
|
||||||
graph = mock_run.call_args[0][0]
|
|
||||||
assert isinstance(graph, px.Graph)
|
|
||||||
|
|
||||||
def test_main_with_no_args_shows_help(self) -> None:
|
|
||||||
"""main() with no args should show help and exit."""
|
|
||||||
with patch("sys.argv", ["which"]), pytest.raises(SystemExit) as exc_info:
|
|
||||||
which.main()
|
|
||||||
assert exc_info.value.code == 2
|
|
||||||
|
|
||||||
def test_main_creates_task_specs_with_correct_names(self) -> None:
|
|
||||||
"""main() should create TaskSpecs with correct names."""
|
|
||||||
with patch("sys.argv", ["which", "git", "npm"]), patch.object(
|
|
||||||
shutil, "which", return_value="/usr/bin/cmd"
|
|
||||||
), patch.object(px, "run") as mock_run:
|
|
||||||
which.main()
|
|
||||||
graph = mock_run.call_args[0][0]
|
|
||||||
# Check that task names are correct
|
|
||||||
task_names = list(graph.all_specs().keys())
|
|
||||||
assert "which_git" in task_names
|
|
||||||
assert "which_npm" in task_names
|
|
||||||
|
|
||||||
def test_main_uses_thread_strategy(self) -> None:
|
|
||||||
"""main() should use thread strategy."""
|
|
||||||
with patch("sys.argv", ["which", "python"]), patch.object(
|
|
||||||
shutil, "which", return_value="/usr/bin/python"
|
|
||||||
), patch.object(px, "run") as mock_run:
|
|
||||||
which.main()
|
|
||||||
assert mock_run.call_args[1]["strategy"] == "thread"
|
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from pytest_mock import MockerFixture
|
||||||
|
|
||||||
|
from pyflowx.utils import _perf_metrics, perf_timer
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def reset_perf_metrics():
|
||||||
|
"""重置性能指标."""
|
||||||
|
_perf_metrics.clear()
|
||||||
|
|
||||||
|
|
||||||
|
class TestPerformanceTimer:
|
||||||
|
def test_perf_timer(self):
|
||||||
|
|
||||||
|
@perf_timer()
|
||||||
|
def test_func():
|
||||||
|
time.sleep(0.1)
|
||||||
|
|
||||||
|
test_func()
|
||||||
|
|
||||||
|
assert _perf_metrics["test_func"] is not None
|
||||||
|
assert _perf_metrics["test_func"]["count"] == 1
|
||||||
|
assert _perf_metrics["test_func"]["total_time"] >= 0.1
|
||||||
|
|
||||||
|
def test_perf_timer_report(self, mocker: MockerFixture):
|
||||||
|
mock_log = mocker.patch("logging.info")
|
||||||
|
|
||||||
|
@perf_timer(report=True, unit="ms", precision=3)
|
||||||
|
def test_func():
|
||||||
|
time.sleep(0.1)
|
||||||
|
|
||||||
|
test_func()
|
||||||
|
|
||||||
|
assert _perf_metrics["test_func"] is not None
|
||||||
|
assert _perf_metrics["test_func"]["count"] == 1
|
||||||
|
assert _perf_metrics["test_func"]["total_time"] >= 0.1
|
||||||
|
|
||||||
|
assert mock_log.call_count == 1
|
||||||
@@ -5603,7 +5603,7 @@ pycountry = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pyflowx"
|
name = "pyflowx"
|
||||||
version = "0.2.9"
|
version = "0.2.10"
|
||||||
source = { editable = "." }
|
source = { editable = "." }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "graphlib-backport", marker = "python_full_version < '3.9'" },
|
{ name = "graphlib-backport", marker = "python_full_version < '3.9'" },
|
||||||
|
|||||||
Reference in New Issue
Block a user