10 Commits

Author SHA1 Message Date
zhou 6f93e6eb6d bump version to 0.3.0
Release / build (push) Failing after 31s
Release / release (push) Has been skipped
Release / publish-pypi (push) Has been skipped
CI / Test (macos-latest) (push) Has been cancelled
CI / Test (ubuntu-latest) (push) Has been cancelled
CI / Test (windows-latest) (push) Has been cancelled
CI / Lint & Typecheck (push) Has been cancelled
2026-06-28 21:38:37 +08:00
zhou 43e1aad1fe chore: 发布版本0.2.13并完善任务执行环境配置
本次提交更新了版本号至0.2.13,同时完成多项改进:
1.  在.gitignore中新增忽略性能分析文件*_profile.html
2.  修复测试用例中echo命令在Windows下无法被正确检测的问题,改用python命令
3.  优化测试用例确保性能统计数据有效,添加耗时模拟函数
4.  为所有CLI任务统一配置项目根目录作为工作目录,解决跨平台执行路径问题
5.  新增测试验证所有任务的cwd配置正确性
2026-06-28 21:38:18 +08:00
zhou 467634f8c7 bump version to 0.2.13
Release / build (push) Failing after 11m59s
Release / release (push) Has been skipped
Release / publish-pypi (push) Has been skipped
2026-06-28 20:30:54 +08:00
zhou ce31f60441 feat(cli): add pxp performance profiler command
1. 新增pxp CLI工具用于分析PyFlowX脚本生成性能报告
2. 新增ProfileReport.to_html方法生成自包含HTML报告
3. 新增完整的profiler功能测试用例
4. 更新pyproject.toml添加pxp入口点
5. 版本升级至0.2.12
2026-06-28 20:30:17 +08:00
zhou 3d6d769685 feat(profiling): 添加工作流性能分析模块与测试用例
新增了性能剖面分析能力,支持从运行报告生成任务级、图级性能指标,包括关键路径、并行度分析和瓶颈识别,同时补充了完整的单元测试覆盖。
2026-06-28 19:59:25 +08:00
zhou 3f9c52e6f1 bump version to 0.2.12
Release / build (push) Failing after 23m3s
Release / publish-pypi (push) Has been skipped
Release / release (push) Has been skipped
2026-06-28 18:56:42 +08:00
zhou 8fadf6edd8 fix(executors): 修复进程池退出阻塞问题
1. 新增_shutdown_process_pool函数,在run()结束时主动关闭进程池
2. 通过atexit注册兜底清理逻辑,防止进程池泄漏
3. 先调用shutdown(wait=False)通知管理线程退出,再强制kill工作进程,避免Python退出时threading._shutdown等待join导致数秒阻塞
4. 新增测试规范文档说明测试相关规则
2026-06-28 18:56:27 +08:00
zhou abc1152538 refactor(cli): 统一使用@px.task装饰器定义任务,重构任务注册和别名管理
1. 将folderzip/folderback/gittool中的旧TaskSpec定义替换为@px.task装饰器
2. 重构pymake模块,将maturin_build_cmd转为常量定义,合并别名配置
3. 精简测试文件中的冗余测试用例
2026-06-28 18:12:30 +08:00
zhou 5e561b4b3a refactor: 重构CliRunner,新增cmd工厂函数优化任务定义
1. 新增cmd工厂函数,简化TaskSpec创建并自动推导名称
2. 重构CliRunner,将graphs参数替换为tasks+aliases,支持扁平任务注册与别名映射
3. 替换所有cli工具中的旧版任务定义方式,使用新API简化代码
4. 补充对应测试用例,适配新的运行器API
2026-06-28 17:52:52 +08:00
zhou 40f641611b feat: 新增多项核心功能并优化默认执行策略
1.  将CliRunner默认执行策略从sequential改为dependency
2.  新增RunReport的任务状态查询和时长统计方法
3.  实现task装饰器并补充executor参数文档
4.  新增进程池执行器支持CPU密集型任务
5.  新增Graph.chain链式构建和add_subgraph子图合并功能
6.  新增流式任务传递、进程池执行、命名空间等多类测试用例
7.  补充tests目录路径导入配置
2026-06-28 15:10:15 +08:00
30 changed files with 3589 additions and 348 deletions
+1
View File
@@ -10,3 +10,4 @@ wheels/
.venv
.coverage
.idea
*_profile.html
+2 -1
View File
@@ -21,7 +21,7 @@ license = { text = "MIT" }
name = "pyflowx"
readme = "README.md"
requires-python = ">=3.8"
version = "0.2.11"
version = "0.3.0"
[project.scripts]
autofmt = "pyflowx.cli.autofmt:main"
@@ -38,6 +38,7 @@ packtool = "pyflowx.cli.packtool:main"
pdftool = "pyflowx.cli.pdftool:main"
piptool = "pyflowx.cli.piptool:main"
pymake = "pyflowx.cli.pymake:main"
pxp = "pyflowx.cli.profiler:main"
reseticon = "pyflowx.cli.reseticoncache:main"
scrcap = "pyflowx.cli.screenshot:main"
sglang = "pyflowx.cli.llm.sglang:main"
+8 -1
View File
@@ -82,6 +82,7 @@ from .errors import (
)
from .executors import Strategy, run
from .graph import Graph, GraphDefaults
from .profiling import ProfileReport, TaskProfile
from .report import RunReport
from .runner import CliExitCode, CliRunner
from .storage import JSONBackend, MemoryBackend, StateBackend
@@ -94,10 +95,12 @@ from .task import (
TaskResult,
TaskSpec,
TaskStatus,
cmd,
task,
task_template,
)
__version__ = "0.3.5"
__version__ = "0.4.0"
__all__ = [
"IS_LINUX",
@@ -120,6 +123,7 @@ __all__ = [
"JSONBackend",
"MemoryBackend",
"MissingDependencyError",
"ProfileReport",
"PyFlowXError",
"RetryPolicy",
"RunReport",
@@ -130,14 +134,17 @@ __all__ = [
"TaskEvent",
"TaskFailedError",
"TaskHooks",
"TaskProfile",
"TaskResult",
"TaskSpec",
"TaskStatus",
"TaskTimeoutError",
"build_call_args",
"cmd",
"compose",
"describe_injection",
"run",
"run_command",
"task",
"task_template",
]
+6 -15
View File
@@ -66,19 +66,10 @@ def backup_folder(src: str, dst: str, max_zip: int = 5) -> None:
zip_target(src_path, dst_path, max_zip)
# ============================================================================
# TaskSpec 定义
# ============================================================================
folderback_default: px.TaskSpec = px.TaskSpec(
"folderback_default",
fn=lambda: backup_folder(".", "./backup", 5),
)
# ============================================================================
# CLI Runner
# ============================================================================
@px.task
def folderback_default() -> None:
"""备份当前目录到 ./backup."""
backup_folder(".", "./backup", 5)
def main() -> None:
@@ -86,9 +77,9 @@ def main() -> None:
runner = px.CliRunner(
strategy="thread",
description="FolderBack - 文件夹备份工具",
graphs={
aliases={
# 备份当前目录到 ./backup
"b": px.Graph.from_specs([folderback_default]),
"b": folderback_default,
},
)
runner.run_cli()
+6 -12
View File
@@ -57,16 +57,10 @@ def zip_folders(cwd: str = ".") -> None:
archive_folder(dir_path)
# ============================================================================
# TaskSpec 定义
# ============================================================================
folderzip_default: px.TaskSpec = px.TaskSpec("folderzip_default", fn=lambda: zip_folders("."))
# ============================================================================
# CLI Runner
# ============================================================================
@px.task
def folderzip_default() -> None:
"""压缩当前目录下的所有文件夹."""
zip_folders(".")
def main() -> None:
@@ -74,9 +68,9 @@ def main() -> None:
runner = px.CliRunner(
strategy="thread",
description="FolderZip - 文件夹压缩工具",
graphs={
aliases={
# 压缩当前目录下的所有文件夹
"z": px.Graph.from_specs([folderzip_default]),
"z": folderzip_default,
},
)
runner.run_cli()
+15 -10
View File
@@ -46,7 +46,12 @@ def init_sub_dirs() -> None:
)
isub: px.TaskSpec = px.TaskSpec("isub", fn=init_sub_dirs)
@px.task(name="isub")
def isub() -> None:
"""初始化子目录的Git仓库."""
init_sub_dirs()
push: px.TaskSpec = px.TaskSpec("push", cmd=["git", "push"])
pull: px.TaskSpec = px.TaskSpec("pull", cmd=["git", "pull"])
kill_tgit: px.TaskSpec = px.TaskSpec("task_kill", cmd=["taskkill", "/f", "/t", "/im", "tgitcache.exe"])
@@ -67,17 +72,17 @@ def main() -> None:
runner = px.CliRunner(
strategy="thread",
description="Gittool - Git 执行工具.",
graphs={
aliases={
# 添加并提交
"a": px.Graph.from_specs([
px.TaskSpec("add", cmd=["git", "add", "."], conditions=(lambda _: has_files(),)),
px.TaskSpec("commit", cmd=["git", "commit", "-m", "chore: update"], depends_on=("add",)),
]),
# 清理
"c": px.Graph.from_specs([
# 清理chain: clean → status
"c": px.Graph().chain(
px.TaskSpec("clean", cmd=["git", "clean", "-xfd", *EXCLUDE_CMDS]),
px.TaskSpec("status", cmd=["git", "status", "--porcelain"], depends_on=("clean",)),
]),
px.TaskSpec("status", cmd=["git", "status", "--porcelain"]),
),
# 初始化、添加并提交
"i": px.Graph.from_specs([
px.TaskSpec("init", cmd=["git", "init"], conditions=(lambda _: not_has_git_repo(),)),
@@ -90,13 +95,13 @@ def main() -> None:
),
]),
# 初始化子目录
"isub": px.Graph.from_specs([isub]),
"isub": isub,
# 推送
"p": px.Graph.from_specs([push]),
"p": push,
# 拉取
"pl": px.Graph.from_specs([pull]),
"pl": pull,
# 重启TGit缓存
"r": px.Graph.from_specs([kill_tgit]),
"r": kill_tgit,
},
)
runner.run_cli()
+272
View File
@@ -0,0 +1,272 @@
"""pxp —— PyFlowX 性能分析器.
分析包含 ``px`` 调用的 Python 脚本,生成工作流执行性能剖面报告。
工作原理
--------
1. 注入 hookmonkey-patch ``pyflowx.run`` / ``pyflowx.executors.run`` /
``pyflowx.runner.run``,捕获最后一次执行的 ``Graph`` 与 ``RunReport``。
2. 执行目标脚本:用 ``runpy.run_path`` 以 ``__main__`` 身份执行,
捕获 ``SystemExit``(脚本可能调 ``sys.exit``)。
3. 生成报告:从捕获的 report + graph 构建 :class:`ProfileReport`
默认输出 HTML 并自动打开浏览器。
使用方式
--------
# 分析 pymake.py,生成 HTML 报告并打开浏览器
pxp pymake.py
# 传递参数给被分析脚本(用 -- 分隔)
pxp pymake.py -- t
# 指定输出文件
pxp pymake.py -o report.html
# 不打开浏览器
pxp pymake.py --no-browser
# 输出纯文本报告
pxp pymake.py -E text
"""
from __future__ import annotations
__all__ = ["main"]
import argparse
import runpy
import sys
import webbrowser
from pathlib import Path
from typing import Any
from .. import executors as _executors
from .. import runner as _runner
from ..profiling import ProfileReport
from ..report import RunReport
def _build_parser() -> argparse.ArgumentParser:
"""构建参数解析器。"""
parser = argparse.ArgumentParser(
prog="pxp",
description="PyFlowX 性能分析器:分析包含 px 调用的脚本,生成性能剖面报告。",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=(
"示例:\n"
" pxp pymake.py # 分析并打开 HTML 报告\n"
" pxp pymake.py -- t # 传递参数 t 给脚本\n"
" pxp pymake.py -E text # 输出纯文本报告\n"
" pxp pymake.py -o out.html # 指定输出文件\n"
),
)
_ = parser.add_argument(
"--export",
"-E",
choices=["html", "text"],
default="html",
help="导出格式(默认: html",
)
_ = parser.add_argument(
"--no-browser",
action="store_true",
help="不自动打开浏览器(仅 HTML 格式有效)",
)
_ = parser.add_argument(
"-o",
"--output",
help="输出文件路径(默认: <script>_profile.html",
)
return parser
def _capture_px_run() -> dict[str, Any]:
"""注入 hook 捕获 px.run() 调用。
返回一个字典,``run()`` 执行后填充 ``graph`` 与 ``report``。
同时返回还原函数用于 finally 块。
Note
-----
需同时 patch 三处引用:
* ``pyflowx.executors.run`` —— 实际实现
* ``pyflowx.runner.run`` —— ``CliRunner`` 直接 import 的引用
* ``pyflowx.run`` —— 顶层包导出的引用(用户脚本常用 ``px.run()``
另外 patch ``RunReport.__init__`` 以捕获 ``run()`` 内部创建的 report 实例。
这对于 ``run()`` 抛出 ``TaskFailedError`` 的场景至关重要:此时 ``run()``
不会正常返回 report,但 report 对象已在内部创建并填充了已执行任务的结果。
通过 ``capture_enabled`` 标志确保只在 ``patched_run`` 调用期间捕获。
"""
captured: dict[str, Any] = {}
original_exec_run = _executors.run
original_runner_run = _runner.run
# 惰性获取顶层 pyflowx.run 引用(避免循环导入)
import pyflowx as px_mod
original_px_run = px_mod.run
original_report_init = RunReport.__init__
capture_enabled = [False]
def patched_report_init(self: RunReport, *args: Any, **kwargs: Any) -> None:
original_report_init(self, *args, **kwargs)
if capture_enabled[0]:
captured["report"] = self
RunReport.__init__ = patched_report_init # type: ignore[assignment]
def patched_run(graph: Any, *args: Any, **kwargs: Any) -> RunReport:
captured["graph"] = graph
capture_enabled[0] = True
try:
report = original_exec_run(graph, *args, **kwargs)
# 正常返回时确保 captured["report"] 是返回的 report
captured["report"] = report
return report
finally:
capture_enabled[0] = False
# patch 所有引用 run 的入口
_executors.run = patched_run # type: ignore[assignment]
_runner.run = patched_run # type: ignore[assignment]
px_mod.run = patched_run # type: ignore[assignment]
def _restore() -> None:
_executors.run = original_exec_run # type: ignore[assignment]
_runner.run = original_runner_run # type: ignore[assignment]
px_mod.run = original_px_run # type: ignore[assignment]
RunReport.__init__ = original_report_init # type: ignore[assignment]
captured["_restore"] = _restore
return captured
def _run_target_script(script: Path, script_args: list[str]) -> dict[str, Any]:
"""执行目标脚本。
将脚本所在目录加入 ``sys.path``,设置 ``sys.argv``,然后用
``runpy.run_path`` 以 ``__main__`` 身份执行。捕获 ``SystemExit``。
Returns
-------
dict[str, Any]
脚本模块的全局变量字典(含 ``main`` 等定义)。
"""
sys.argv = [str(script), *script_args]
script_dir = str(script.parent.resolve())
if script_dir not in sys.path:
sys.path.insert(0, script_dir)
return runpy.run_path(str(script), run_name="__main__")
def _try_call_main(module_globals: dict[str, Any]) -> None:
"""若模块定义了 ``main`` 可调用对象,调用它。
用于脚本无 ``if __name__ == "__main__"`` 块的场景(如通过 entry points
注册的 CLI 工具脚本)。``main`` 通常调用 ``CliRunner.run_cli()``
后者读取 ``sys.argv[1:]`` 执行对应命令。
"""
main_fn = module_globals.get("main")
if callable(main_fn):
main_fn()
def _output_report(
profile: ProfileReport,
export: str,
output: str | None,
script_stem: str,
no_browser: bool,
) -> None:
"""输出性能报告。"""
if export == "text":
print(profile.describe())
return
# HTML 格式
html = profile.to_html()
if output:
out_path = Path(output)
else:
out_path = Path.cwd() / f"{script_stem}_profile.html"
out_path.write_text(html, encoding="utf-8")
print(f"HTML 报告已生成: {out_path}")
if not no_browser:
try:
webbrowser.open(f"file://{out_path.resolve()}")
except Exception as e:
print(f"警告:无法打开浏览器: {e}", file=sys.stderr)
def main() -> None:
"""pxp CLI 入口。"""
parser = _build_parser()
pxp_args, remaining = parser.parse_known_args()
if not remaining:
parser.print_help()
sys.exit(2)
script_str = remaining[0]
script_args = remaining[1:]
script_path = Path(script_str).resolve()
if not script_path.is_file():
print(f"错误:脚本不存在: {script_path}", file=sys.stderr)
sys.exit(2)
# 注入 hook
captured = _capture_px_run()
# 执行目标脚本
print(f"正在分析: {script_path}")
if script_args:
print(f"脚本参数: {script_args}")
print("-" * 60)
module_globals: dict[str, Any] = {}
try:
module_globals = _run_target_script(script_path, script_args)
except SystemExit:
# 脚本调用了 sys.exit,正常情况
pass
except Exception as e:
print(f"警告:脚本执行抛出异常: {e}", file=sys.stderr)
# 若脚本执行未捕获到 run(),尝试调用模块的 main() 函数
# (适用于无 ``if __name__ == "__main__"`` 块的 CLI 脚本)
if captured.get("report") is None and module_globals:
try:
_try_call_main(module_globals)
except SystemExit:
pass
except Exception as e:
print(f"警告:调用 main() 抛出异常: {e}", file=sys.stderr)
# 还原 hook
restore = captured.pop("_restore", None)
if restore is not None:
restore()
# 检查是否捕获到 run() 调用
report = captured.get("report")
graph = captured.get("graph")
if report is None or graph is None:
print("错误:未捕获到 px.run() 调用,无法生成性能报告", file=sys.stderr)
print("请确保脚本通过 px.run() 或 CliRunner 执行任务流图。", file=sys.stderr)
sys.exit(1)
# 生成报告
profile = ProfileReport.from_report(report, graph)
_output_report(
profile,
export=pxp_args.export,
output=pxp_args.output,
script_stem=script_path.stem,
no_browser=pxp_args.no_browser,
)
if __name__ == "__main__":
main()
+67 -67
View File
@@ -6,51 +6,77 @@
from __future__ import annotations
from pathlib import Path
import pyflowx as px
from pyflowx.conditions import Constants
# 项目根目录(pymake.py 在 src/pyflowx/cli,向上四层到达根目录)
ROOT_DIR = Path(__file__).parent.parent.parent.parent
def maturin_build_cmd() -> list[str]:
"""获取 maturin 构建命令(根据平台自动添加参数).
MATURIN_BUILD_COMMAND = ["maturin", "build", "-r"]
if Constants.IS_WINDOWS:
MATURIN_BUILD_COMMAND.extend(["--target", "x86_64-win7-windows-msvc", "-Zbuild-std", "-i", "python3.8"])
Returns
-------
list[str]
完整的 maturin 构建命令列表.
"""
command = ["maturin", "build", "-r"].copy()
if Constants.IS_WINDOWS:
command.extend(["--target", "x86_64-win7-windows-msvc", "-Zbuild-std", "-i", "python3.8"])
return command
# 扁平注册所有任务(px.cmd 自动从命令前两段推导 name)
# 所有任务指定 cwd=ROOT_DIR,确保在项目根目录执行
tasks: list[px.TaskSpec] = [
px.cmd(["uv", "build"], cwd=ROOT_DIR),
px.cmd(MATURIN_BUILD_COMMAND, cwd=ROOT_DIR),
px.cmd(["uv", "sync"], cwd=ROOT_DIR),
px.cmd(["gitt", "c"], name="git_clean", cwd=ROOT_DIR),
px.cmd(
["pytest", "-m", "not slow", "-n", "8", "--dist", "loadfile", "--color=yes", "--durations=10"],
name="test",
cwd=ROOT_DIR,
),
px.cmd(
["pytest", "-m", "not slow", "--dist", "loadfile", "--color=yes", "--durations=10"],
name="test_fast",
cwd=ROOT_DIR,
),
px.cmd(
["pytest", "--cov", "-n", "8", "--dist", "loadfile", "--tb=short", "-v", "--color=yes", "--durations=10"],
name="test_coverage",
cwd=ROOT_DIR,
),
px.cmd(["pyrefly", "check", "."], cwd=ROOT_DIR),
px.cmd(["git", "add", "-A"], name="git_add_all", cwd=ROOT_DIR),
px.cmd(["bumpversion"], cwd=ROOT_DIR),
px.cmd(["bumpversion", "minor"], cwd=ROOT_DIR),
px.cmd(["git", "push"], cwd=ROOT_DIR),
px.cmd(["git", "push", "--tags"], name="git_push_tags", cwd=ROOT_DIR),
px.cmd(["hatch", "publish"], name="publish_python", cwd=ROOT_DIR),
px.cmd(["twine", "upload", "--disable-progress-bar"], name="twine_publish", cwd=ROOT_DIR),
]
# 单任务别名(alias 名与任务名相同):直接内联 TaskSpec,避免 str 自引用
aliases: dict[str, str | list[str | px.TaskSpec] | px.TaskSpec | px.Graph] = {
# 构建命令
"b": "uv_build",
"bc": "maturin_build",
"ba": ["b", "bc"],
# 安装命令
"sync": "uv_sync",
# 清理命令
"c": "git_clean",
# 开发工具
"bump": ["c", "tc", "git_add_all", "bumpversion"],
"bumpmi": "bumpversion_minor",
"cov": ["git_clean", "test_coverage"],
"doc": px.cmd(["sphinx-build", "-b", "html", "docs", "docs/_build"], name="doc", cwd=ROOT_DIR),
"lint": px.cmd(["ruff", "check", "--fix", "--unsafe-fixes"], name="lint", cwd=ROOT_DIR),
"pb": ["twine_publish", "publish_python"],
"t": "test",
"tf": "test_fast",
"tc": ["pyrefly_check", "lint"],
"tox": px.cmd(["tox", "-p", "auto"], name="tox", cwd=ROOT_DIR),
# 发布命令
"p": ["git_clean", "git_push", "git_push_tags"],
}
uv_build: px.TaskSpec = px.TaskSpec("uv_build", cmd=["uv", "build"])
maturin_build: px.TaskSpec = px.TaskSpec("maturin_build", cmd=maturin_build_cmd())
uv_sync: px.TaskSpec = px.TaskSpec("uv_sync", cmd=["uv", "sync"])
git_clean: px.TaskSpec = px.TaskSpec("git_clean", cmd=["gitt", "c"])
test: px.TaskSpec = px.TaskSpec(
"test", cmd=["pytest", "-m", "not slow", "-n", "8", "--dist", "loadfile", "--color=yes", "--durations=10"]
)
test_fast: px.TaskSpec = px.TaskSpec(
"test_fast", cmd=["pytest", "-m", "not slow", "--dist", "loadfile", "--color=yes", "--durations=10"]
)
test_coverage: px.TaskSpec = px.TaskSpec(
"test_coverage",
cmd=["pytest", "--cov", "-n", "8", "--dist", "loadfile", "--tb=short", "-v", "--color=yes", "--durations=10"],
)
ruff_lint: px.TaskSpec = px.TaskSpec("lint", cmd=["ruff", "check", "--fix", "--unsafe-fixes"])
typecheck: px.TaskSpec = px.TaskSpec("pyrefly_check", cmd=["pyrefly", "check", "."])
git_add_all: px.TaskSpec = px.TaskSpec("git_add_all", cmd=["git", "add", "-A"])
bump: px.TaskSpec = px.TaskSpec("bumpversion", cmd=["bumpversion"])
doc: px.TaskSpec = px.TaskSpec("doc", cmd=["sphinx-build", "-b", "html", "docs", "docs/_build"])
git_push: px.TaskSpec = px.TaskSpec("git_push", cmd=["git", "push"])
git_push_tags: px.TaskSpec = px.TaskSpec("git_push_tags", cmd=["git", "push", "--tags"])
hatch_publish: px.TaskSpec = px.TaskSpec("publish_python", cmd=["hatch", "publish"])
twine_publish: px.TaskSpec = px.TaskSpec("twine_publish", cmd=["twine", "upload", "--disable-progress-bar"])
tox: px.TaskSpec = px.TaskSpec("tox", cmd=["tox", "-p", "auto"])
def main():
def main() -> None:
"""pymake 构建工具.
🔨 构建命令:
@@ -78,10 +104,10 @@ def main():
📦 发布命令:
pymake pb - 发布到 PyPI (twine + hatch)
版本管理:
🔖 版本管理:
pymake bump - 自动升级版本号并提交修改 (清理 + 检查 + 格式化 + git add + bumpversion)
💡 常用工作流:
💡 常用工作流:
1. 日常开发: pymake lint && pymake t
2. 构建发布包: pymake ba
3. 多版本兼容性测试: pymake tox
@@ -95,31 +121,5 @@ def main():
pymake lint # 格式化代码
pymake type # 类型检查
"""
runner = px.CliRunner(
strategy="sequential",
description="PyMake - Python 构建工具",
graphs={
# 构建命令
"b": px.Graph.from_specs([uv_build]),
"bc": px.Graph.from_specs([maturin_build]),
"ba": px.Graph.from_specs(["b", "bc"]),
# 安装命令
"sync": px.Graph.from_specs([uv_sync]),
# 清理命令
"c": px.Graph.from_specs([git_clean]),
# 开发工具
"bump": px.Graph.from_specs(["c", "tc", git_add_all, bump]),
"bumpmi": px.Graph.from_specs([px.TaskSpec("bumpversion_minor", cmd=["bumpversion", "minor"])]),
"cov": px.Graph.from_specs([git_clean, test_coverage]),
"doc": px.Graph.from_specs([doc]),
"lint": px.Graph.from_specs([ruff_lint]),
"pb": px.Graph.from_specs([twine_publish, hatch_publish]),
"t": px.Graph.from_specs([test]),
"tf": px.Graph.from_specs([test_fast]),
"tc": px.Graph.from_specs([typecheck, "lint"]),
"tox": px.Graph.from_specs([tox]),
# 发布命令
"p": px.Graph.from_specs([git_clean, git_push, git_push_tags]),
},
)
runner = px.CliRunner(strategy="sequential", description="PyMake - Python 构建工具", tasks=tasks, aliases=aliases)
runner.run_cli()
+99 -9
View File
@@ -41,7 +41,9 @@
from __future__ import annotations
import asyncio
import atexit
import concurrent.futures
import contextlib
import inspect
import logging
import threading
@@ -58,6 +60,59 @@ from .task import TaskEvent, TaskHooks, TaskResult, TaskSpec, TaskStatus
logger = logging.getLogger(__name__)
# 进程池复用:同一次 run() 内的 process 任务共享一个 ProcessPoolExecutor。
# 模块级缓存避免每次任务都创建/销毁进程池的开销。
# run() 结束后通过 _shutdown_process_pool() 关闭(shutdown(wait=False) +
# kill 工作进程),避免 Python 退出时 threading._shutdown 等待管理线程
# join 工作进程导致数秒阻塞。
_process_pool: concurrent.futures.ProcessPoolExecutor | None = None
_process_pool_lock = threading.Lock()
def _get_process_pool() -> concurrent.futures.ProcessPoolExecutor:
"""获取复用的进程池(惰性创建)。"""
global _process_pool # noqa: PLW0603
if _process_pool is None:
with _process_pool_lock:
if _process_pool is None:
_process_pool = concurrent.futures.ProcessPoolExecutor()
return _process_pool
def _shutdown_process_pool() -> None:
"""关闭复用的进程池。
``shutdown(wait=False)`` 通知管理线程退出(管理线程是非 daemon,
``threading._shutdown`` 会等待它);同时 kill 工作进程,避免管理线程
在退出前逐个 join 工作进程导致数秒阻塞。
"""
global _process_pool # noqa: PLW0603
if _process_pool is not None:
pool = _process_pool
_process_pool = None
# 在 shutdown 前获取进程列表(管理线程退出会清空 _processes)。
# _processes 是 ProcessPoolExecutor 的私有属性,无公开 API 替代。
procs = list((getattr(pool, "_processes", None) or {}).values())
pool.shutdown(wait=False)
# 强制终止工作进程(SIGKILL),避免管理线程 join 导致 ~7s 阻塞。
for proc in procs:
with contextlib.suppress(ProcessLookupError, AttributeError):
proc.kill() # type: ignore[attr-defined]
# 兜底:防止未经 run() 直接使用执行器的场景导致进程池泄漏。
atexit.register(_shutdown_process_pool)
def _run_in_process(fn: Any, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any:
"""模块级函数:在进程池中执行任务(须可 pickle)。
env_context 等上下文管理器无法跨进程传递,进程池任务的 ``env``/``cwd``
不生效;如需设置环境,应在 ``fn`` 内部自行处理。
"""
return fn(*args, **kwargs)
# 观察者回调类型。
EventCallback = Callable[[TaskEvent], None]
Strategy = Literal["sequential", "thread", "async", "dependency"]
@@ -391,19 +446,50 @@ async def _execute_async_task(
loop: asyncio.AbstractEventLoop,
) -> Any:
"""执行异步或同步任务(带超时处理)。"""
# 异步任务直接 await
if _is_async_fn(spec):
coro = cast(Awaitable[Any], spec.effective_fn(*args, **kwargs))
if spec.timeout is not None:
return await asyncio.wait_for(coro, timeout=spec.timeout)
return await coro
return await asyncio.wait_for(coro, timeout=spec.timeout) if spec.timeout is not None else await coro
# 同步任务:根据 executor 选择执行器
fut = _submit_sync_task(spec, args, kwargs, loop)
return await asyncio.wait_for(fut, timeout=spec.timeout) if spec.timeout is not None else await fut
def _submit_sync_task(
spec: TaskSpec[Any],
args: tuple[Any, ...],
kwargs: dict[str, Any],
loop: asyncio.AbstractEventLoop,
) -> asyncio.Future[Any]:
"""提交同步任务到对应执行器,返回 Future。
* ``inline``:直接在事件循环线程调用(阻塞循环,最快)。
* ``process``:进程池执行(绕过 GILfn 须可 pickle)。
* ``thread``(默认):线程池执行。
"""
def fn_call() -> Any:
with spec.env_context():
return spec.effective_fn(*args, **kwargs)
if spec.timeout is not None:
return await asyncio.wait_for(loop.run_in_executor(None, fn_call), timeout=spec.timeout)
return await loop.run_in_executor(None, fn_call)
# inline:直接在事件循环线程调用,无线程池开销,但会阻塞循环。
if spec.executor == "inline":
result = fn_call()
fut: asyncio.Future[Any] = loop.create_future()
fut.set_result(result)
return fut
# process:进程池执行,绕过 GIL,适合 CPU 密集型任务(fn 须可 pickle)。
if spec.executor == "process":
from functools import partial
pool = _get_process_pool()
proc_fn = partial(_run_in_process, spec.effective_fn, args, kwargs)
return loop.run_in_executor(pool, proc_fn)
# thread(默认):线程池执行。
return loop.run_in_executor(None, fn_call)
# ---------------------------------------------------------------------- #
@@ -662,7 +748,7 @@ def _make_verbose_callback(on_event: EventCallback | None) -> EventCallback:
def run(
graph: Graph,
strategy: Strategy = "sequential",
strategy: Strategy = "dependency",
*,
max_workers: int | None = None,
dry_run: bool = False,
@@ -678,8 +764,8 @@ def run(
graph:
待执行的已校验 :class:`Graph`。
strategy:
执行策略: ``"sequential"`` / ``"thread"`` / ``"async"`` /
``"dependency"````"dependency"`` 为依赖驱动调度,无层屏障
执行策略: ``"dependency"``(默认,依赖驱动无层屏障,最大并行度)/
``"sequential"`` / ``"thread"`` / ``"async"``(层屏障模型)
max_workers:
``"thread"`` 的线程池大小。默认 ``min(32, len(layer))``。
dry_run:
@@ -737,6 +823,10 @@ def run(
except TaskFailedError:
report.success = False
raise
finally:
# 关闭进程池:通知管理线程退出 + kill 工作进程,避免
# threading._shutdown 等待管理线程 join 工作进程导致 ~7s 阻塞。
_shutdown_process_pool()
return report
+138 -2
View File
@@ -17,12 +17,13 @@ __all__ = [
"GraphDefaults",
]
import inspect
import sys
from dataclasses import dataclass, field, replace
from typing import Any, Callable, Iterable, Mapping, Sequence
from .errors import CycleError, DuplicateTaskError, MissingDependencyError
from .task import RetryPolicy, TaskSpec
from .task import Context, RetryPolicy, TaskSpec
if sys.version_info >= (3, 9): # pragma: no cover
import graphlib # pyright: ignore[reportUnreachable]
@@ -63,6 +64,74 @@ def _prune_deps(spec: TaskSpec[Any], keep: Callable[[str], bool]) -> TaskSpec[An
)
def _make_namespaced_fn(orig_fn: Any, ns: str, dep_names: set[str]) -> Any:
"""包装 fn,使其能接收带 ``ns:`` 前缀的依赖名,调用时映射回原参数名。
命名空间合并后,依赖名带前缀(如 ``build:extract``),但 Python 参数名
不能含 ``:``。wrapper 用 ``**kwargs`` 接收所有依赖,内部把带前缀的依赖名
映射回原参数名后调用原 fn。
无依赖参数时直接返回原 fn。
"""
if not dep_names or orig_fn is None:
return orig_fn
try:
orig_sig = inspect.signature(orig_fn)
except (TypeError, ValueError):
return orig_fn
# 带前缀依赖名 -> 原参数名
name_map: dict[str, str] = {f"{ns}:{orig}": orig for orig in dep_names}
prefix = f"{ns}:"
# 检查原 fn 是否有 Context 标注参数
context_param_name: str | None = None
for p in orig_sig.parameters.values():
ann = p.annotation
if ann is not Context and not (isinstance(ann, str) and ann.endswith("Context")):
continue
context_param_name = p.name
break
if context_param_name is not None:
def wrapper(ctx: Any = None, **kwargs: Any) -> Any:
# ctx 是 dep_context,键为带前缀的依赖名;映射回原始键
orig_ctx: dict[str, Any] = {}
for k, v in (ctx or {}).items():
orig_ctx[name_map.get(k, k)] = v
# kwargs 中带前缀的依赖也映射回原参数名
for k, v in kwargs.items():
if k in name_map:
orig_ctx[name_map[k]] = v
return orig_fn(**{context_param_name: orig_ctx})
ctx_param = inspect.Parameter("ctx", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=Context)
kw_param = inspect.Parameter("kwargs", inspect.Parameter.VAR_KEYWORD)
wrapper.__signature__ = inspect.Signature( # type: ignore[attr-defined]
parameters=[ctx_param, kw_param],
return_annotation=orig_sig.return_annotation,
)
else:
def wrapper(**kwargs: Any) -> Any: # type: ignore[no-redef]
orig_kwargs: dict[str, Any] = {}
for k, v in kwargs.items():
if k.startswith(prefix):
orig_kwargs[k[len(prefix) :]] = v
return orig_fn(**orig_kwargs)
kw_param = inspect.Parameter("kwargs", inspect.Parameter.VAR_KEYWORD)
wrapper.__signature__ = inspect.Signature( # type: ignore[attr-defined]
parameters=[kw_param],
return_annotation=orig_sig.return_annotation,
)
wrapper.__name__ = f"{ns}_{getattr(orig_fn, '__name__', 'fn')}"
wrapper.__doc__ = getattr(orig_fn, "__doc__", None)
return wrapper
@dataclass
class Graph:
"""校验后的有向无环任务图。
@@ -78,6 +147,7 @@ class Graph:
specs: dict[str, TaskSpec[Any]] = field(default_factory=dict)
deps: dict[str, tuple[str, ...]] = field(default_factory=dict)
defaults: GraphDefaults = field(default_factory=GraphDefaults)
namespace: str | None = None
# 待解析的字符串引用列表(由 GraphComposer 消费);为空表示无引用。
_pending_refs: list[str] = field(default_factory=list)
@@ -95,6 +165,28 @@ class Graph:
self._validate_references()
return self
def chain(self, *specs: TaskSpec[Any]) -> Graph:
"""链式注册任务:每个 spec 自动依赖前一个。
``chain(a, b, c)`` 等价于 ``b`` 依赖 ``a````c`` 依赖 ``b``。
若 spec 已带 ``depends_on``,则前驱名追加到现有依赖前。
返回 ``self`` 支持链式调用。
Examples
--------
>>> graph = px.Graph().chain(extract, transform, load)
"""
prev_name: str | None = None
for s in specs:
current = s
if prev_name is not None:
# 将前驱追加到 depends_on 最前(保持显式依赖优先)
new_deps = (prev_name, *s.depends_on) if prev_name not in s.depends_on else s.depends_on
current = replace(s, depends_on=new_deps)
self.add(current)
prev_name = current.name
return self
def _register(self, spec: TaskSpec[Any]) -> None:
if spec.name in self.specs:
raise DuplicateTaskError(spec.name)
@@ -108,6 +200,8 @@ class Graph:
cls,
specs: Iterable[TaskSpec[Any] | str],
defaults: GraphDefaults | None = None,
*,
namespace: str | None = None,
) -> Graph:
"""从可迭代的 task spec 构建图。
@@ -120,8 +214,10 @@ class Graph:
TaskSpec 对象或字符串引用的列表。
defaults:
图级默认值。``None`` 使用空 :class:`GraphDefaults`。
namespace:
可选命名空间,用于 :meth:`add_subgraph` 合并时加前缀。
"""
graph = cls(defaults=defaults or GraphDefaults())
graph = cls(defaults=defaults or GraphDefaults(), namespace=namespace)
pending_refs: list[str] = []
for spec in specs:
@@ -139,6 +235,46 @@ class Graph:
graph.validate()
return graph
def add_subgraph(self, sub: Graph, *, namespace: str | None = None) -> Graph:
"""将子图合并到当前图,任务名加命名空间前缀避免冲突。
参数
----
sub:
待合并的子图。
namespace:
命名空间前缀。``None`` 时使用 ``sub.namespace``,若子图也无命名空间
则抛出 ``ValueError``。最终任务名为 ``f"{ns}:{original_name}"``。
合并后,子图内任务的依赖名也会被加前缀;与子图外部任务的依赖保持原样。
返回 ``self`` 支持链式调用。
"""
ns = namespace or sub.namespace
if not ns:
raise ValueError("add_subgraph 需要 namespace 或子图自带 namespace")
def _rename(name: str) -> str:
# 仅对子图内部任务名加前缀;外部依赖保持原样
return f"{ns}:{name}" if name in sub.specs else name
sub_names = set(sub.specs.keys())
for spec in sub.specs.values():
# 子图内部依赖名需加前缀,对应的 fn 参数也需包装
internal_deps = (set(spec.depends_on) | set(spec.soft_depends_on)) & sub_names
new_fn = _make_namespaced_fn(spec.fn, ns, internal_deps) if spec.fn else spec.fn
new_spec = replace(
spec,
name=_rename(spec.name),
fn=new_fn,
depends_on=tuple(_rename(d) for d in spec.depends_on),
soft_depends_on=tuple(_rename(d) for d in spec.soft_depends_on),
)
self._register(new_spec)
self._validate_references()
self.validate()
return self
# ------------------------------------------------------------------ #
# 校验
# ------------------------------------------------------------------ #
+705
View File
@@ -0,0 +1,705 @@
"""工作流执行性能评估。
基于 :class:`~pyflowx.report.RunReport` 中已有的 ``started_at`` /
``finished_at`` 时间戳进行离线分析,**零运行时开销**——不修改执行流程,
不注册回调,不引入额外计时器。
核心指标
--------
* **任务级**:每个任务的 wall-clock 耗时、状态、重试次数、等待时间
(从最早依赖完成到本任务开始)。
* **图级**:总耗时(wall-clock)、关键路径耗时(理论最短耗时)、
并行度效率(关键路径耗时 / 总耗时)。
* **关键路径**:从源点到汇点的最长依赖路径,识别真正的串行瓶颈。
* **并行度**:基于时间线重叠计算瞬时并行度,给出平均并行度与峰值并行度。
* **瓶颈识别**:按耗时排序的 Top-N 任务。
设计原则
--------
* 数据来源于 ``RunReport`` + ``Graph``,无副作用。
* 计算复杂度 O(V+E):拓扑排序 + 单次松弛,适合大规模图。
* 所有时间戳用 ``datetime``,与 :class:`TaskResult` 保持一致。
快速上手
--------
import pyflowx as px
report = px.run(graph)
profile = px.ProfileReport.from_report(report, graph)
print(profile.describe())
bottlenecks = profile.top_bottlenecks(3)
"""
from __future__ import annotations
__all__ = [
"ProfileReport",
"TaskProfile",
]
from dataclasses import dataclass
from datetime import datetime
from typing import Any
from .graph import Graph
from .report import RunReport
from .task import TaskResult, TaskStatus
@dataclass(frozen=True)
class TaskProfile:
"""单个任务的性能剖面。
属性
----
name:
任务名。
status:
终态(SUCCESS/FAILED/SKIPPED)。
duration:
wall-clock 执行耗时(秒)。SKIPPED 任务为 0.0。
attempts:
尝试次数(含首次)。
wait_time:
从最早硬依赖完成到本任务开始的等待时间(秒)。
无硬依赖或 SKIPPED 时为 0.0。
is_on_critical_path:
是否位于关键路径上。
deps:
硬依赖任务名列表。
"""
name: str
status: TaskStatus
duration: float
attempts: int
wait_time: float
is_on_critical_path: bool
deps: tuple[str, ...]
def to_dict(self) -> dict[str, Any]:
"""转为 JSON 友好的字典。"""
return {
"name": self.name,
"status": self.status.value,
"duration_seconds": round(self.duration, 6),
"attempts": self.attempts,
"wait_time_seconds": round(self.wait_time, 6),
"is_on_critical_path": self.is_on_critical_path,
"deps": list(self.deps),
}
@dataclass(frozen=True)
class ProfileReport:
"""工作流执行的性能剖面报告。
通过 :meth:`from_report` 从 :class:`RunReport` + :class:`Graph` 构建。
所有字段在构造时一次性计算完毕,后续访问为 O(1)。
"""
tasks: tuple[TaskProfile, ...]
"""所有任务的性能剖面(按拓扑序)。"""
total_duration: float
"""整次运行的 wall-clock 耗时(秒)。"""
critical_path_duration: float
"""关键路径耗时(秒):从最早任务开始到最晚任务结束的最长依赖路径。"""
critical_path: tuple[str, ...]
"""关键路径上的任务名序列(按执行顺序)。"""
avg_parallelism: float
"""平均并行度 = 任务总耗时 / wall-clock 总耗时。"""
peak_parallelism: int
"""峰值并行度:任一时刻同时运行的任务数最大值。"""
parallelism_efficiency: float
"""并行度效率 = 关键路径耗时 / wall-clock 总耗时。``1.0`` 表示完全串行,
越大表示并行化收益越低(瓶颈在关键路径上)。"""
# ------------------------------------------------------------------ #
# 构建
# ------------------------------------------------------------------ #
@classmethod
def from_report(cls, report: RunReport, graph: Graph) -> ProfileReport:
"""从运行报告与图构建性能剖面。
参数
----
report:
已完成的 :class:`RunReport`,需包含 ``started_at``/``finished_at``。
graph:
对应的 :class:`Graph`,用于依赖关系与关键路径分析。
Note
-----
本方法不修改 ``report`` 或 ``graph``,纯函数式计算。
"""
task_profiles = cls._build_task_profiles(report, graph)
total_duration = cls._calc_total_duration(report)
critical_path, critical_duration = cls._calc_critical_path(graph, report)
avg_par, peak_par = cls._calc_parallelism(report)
efficiency = critical_duration / total_duration if total_duration > 0 else 0.0
# 标记关键路径上的任务
critical_set = set(critical_path)
marked = tuple(
TaskProfile(
name=t.name,
status=t.status,
duration=t.duration,
attempts=t.attempts,
wait_time=t.wait_time,
is_on_critical_path=t.name in critical_set,
deps=t.deps,
)
for t in task_profiles
)
return cls(
tasks=marked,
total_duration=total_duration,
critical_path_duration=critical_duration,
critical_path=critical_path,
avg_parallelism=avg_par,
peak_parallelism=peak_par,
parallelism_efficiency=efficiency,
)
@staticmethod
def _build_task_profiles(report: RunReport, graph: Graph) -> tuple[TaskProfile, ...]:
"""构建每个任务的性能剖面。"""
profiles: list[TaskProfile] = []
for name, result in report.results.items():
spec = graph.specs.get(name)
deps = tuple(spec.depends_on) if spec is not None else ()
duration = result.duration or 0.0
wait_time = ProfileReport._calc_wait_time(result, deps, report)
profiles.append(
TaskProfile(
name=name,
status=result.status,
duration=duration,
attempts=result.attempts,
wait_time=wait_time,
is_on_critical_path=False, # 后续标记
deps=deps,
)
)
return tuple(profiles)
@staticmethod
def _calc_wait_time(
result: TaskResult[Any],
deps: tuple[str, ...],
report: RunReport,
) -> float:
"""计算等待时间:从最早依赖完成到本任务开始。
无硬依赖、SKIPPED 任务或时间戳缺失时返回 0.0。
"""
if not deps or result.started_at is None or result.status == TaskStatus.SKIPPED:
return 0.0
# 找出所有已完成依赖的最晚完成时间
dep_end_times: list[datetime] = []
for dep in deps:
dep_result = report.results.get(dep)
if dep_result is not None and dep_result.finished_at is not None:
dep_end_times.append(dep_result.finished_at)
if not dep_end_times:
return 0.0
latest_dep_end = max(dep_end_times)
delta = (result.started_at - latest_dep_end).total_seconds()
return max(0.0, delta)
@staticmethod
def _calc_total_duration(report: RunReport) -> float:
"""计算 wall-clock 总耗时:最早开始到最晚结束。"""
starts: list[datetime] = []
ends: list[datetime] = []
for r in report.results.values():
if r.started_at is not None:
starts.append(r.started_at)
if r.finished_at is not None:
ends.append(r.finished_at)
if not starts or not ends:
return 0.0
return (max(ends) - min(starts)).total_seconds()
@staticmethod
def _calc_critical_path(graph: Graph, report: RunReport) -> tuple[tuple[str, ...], float]:
"""计算关键路径:DAG 最长路径(按实际执行耗时)。
使用拓扑排序 + 动态规划,O(V+E)。SKIPPED 任务耗时按 0 计。
"""
# 构建耗时映射
durations: dict[str, float] = {}
for name, result in report.results.items():
durations[name] = result.duration or 0.0
# 拓扑序(使用 graph.layers 保证与分层一致)
try:
layers = graph.layers()
except Exception:
# 图校验失败时回退为空
return (), 0.0
# earliest_finish[name] = duration[name] + max(earliest_finish[dep] for dep in deps)
earliest_finish: dict[str, float] = {}
predecessor: dict[str, str | None] = {}
for layer in layers:
for name in layer:
spec = graph.specs.get(name)
deps = spec.depends_on if spec is not None else ()
if not deps:
earliest_finish[name] = durations.get(name, 0.0)
predecessor[name] = None
else:
best_dep: str | None = None
best_ef = 0.0
for dep in deps:
ef = earliest_finish.get(dep, 0.0)
if ef >= best_ef:
best_ef = ef
best_dep = dep
earliest_finish[name] = best_ef + durations.get(name, 0.0)
predecessor[name] = best_dep
if not earliest_finish:
return (), 0.0
# 找到 earliest_finish 最大的节点作为终点
end_node = max(earliest_finish, key=lambda n: earliest_finish[n])
total = earliest_finish[end_node]
# 回溯关键路径
path: list[str] = []
node: str | None = end_node
while node is not None:
path.append(node)
node = predecessor.get(node)
path.reverse()
return tuple(path), total
@staticmethod
def _calc_parallelism(report: RunReport) -> tuple[float, int]:
"""计算平均并行度与峰值并行度。
基于时间线扫描:将每个任务的 [started_at, finished_at] 区间
转为事件点(+1/-1),排序后扫描得到瞬时并行度序列。
返回 (avg_parallelism, peak_parallelism)。
无有效时间戳时返回 (0.0, 0)。
"""
events: list[tuple[float, int]] = [] # (timestamp, delta)
for r in report.results.values():
if r.started_at is None or r.finished_at is None:
continue
if r.status == TaskStatus.SKIPPED:
continue
start_ts = r.started_at.timestamp()
end_ts = r.finished_at.timestamp()
if end_ts <= start_ts:
continue
events.append((start_ts, 1))
events.append((end_ts, -1))
if not events:
return 0.0, 0
# 排序:同一时间点先处理结束(-1)再处理开始(+1),避免虚假峰值
events.sort(key=lambda e: (e[0], e[1]))
current = 0
peak = 0
# 加权面积用于计算平均并行度
area = 0.0
prev_ts = events[0][0]
for ts, delta in events:
if ts > prev_ts:
area += current * (ts - prev_ts)
current += delta
peak = max(peak, current)
prev_ts = ts
total_span = events[-1][0] - events[0][0]
avg = area / total_span if total_span > 0 else 0.0
return avg, peak
# ------------------------------------------------------------------ #
# 查询
# ------------------------------------------------------------------ #
def task(self, name: str) -> TaskProfile:
"""返回指定任务的剖面。不存在则 ``KeyError``。"""
for t in self.tasks:
if t.name == name:
return t
raise KeyError(name)
def top_bottlenecks(self, n: int = 5) -> tuple[TaskProfile, ...]:
"""返回耗时最长的 Top-N 任务(按 duration 降序)。
参数
----
n:
返回数量。``n <= 0`` 返回空元组。
"""
if n <= 0:
return ()
return tuple(sorted(self.tasks, key=lambda t: t.duration, reverse=True)[:n])
def critical_tasks(self) -> tuple[TaskProfile, ...]:
"""返回关键路径上的所有任务(按路径顺序)。"""
critical_set = set(self.critical_path)
# 保持关键路径顺序
order = {name: i for i, name in enumerate(self.critical_path)}
return tuple(sorted((t for t in self.tasks if t.name in critical_set), key=lambda t: order[t.name]))
def failed_tasks(self) -> tuple[TaskProfile, ...]:
"""返回 FAILED 状态的任务。"""
return tuple(t for t in self.tasks if t.status == TaskStatus.FAILED)
def skipped_tasks(self) -> tuple[TaskProfile, ...]:
"""返回 SKIPPED 状态的任务。"""
return tuple(t for t in self.tasks if t.status == TaskStatus.SKIPPED)
# ------------------------------------------------------------------ #
# 输出
# ------------------------------------------------------------------ #
def to_dict(self) -> dict[str, Any]:
"""转为 JSON 友好的字典。"""
return {
"tasks": [t.to_dict() for t in self.tasks],
"total_duration_seconds": round(self.total_duration, 6),
"critical_path_duration_seconds": round(self.critical_path_duration, 6),
"critical_path": list(self.critical_path),
"avg_parallelism": round(self.avg_parallelism, 4),
"peak_parallelism": self.peak_parallelism,
"parallelism_efficiency": round(self.parallelism_efficiency, 4),
"bottlenecks": [t.to_dict() for t in self.top_bottlenecks(5)],
}
def to_html(self) -> str:
"""生成自包含的 HTML 报告(含 CSS,无外部依赖)。
报告含:图级指标卡片、关键路径、时间线甘特图、Top 瓶颈表格、
全部任务表格。适合直接用浏览器打开查看。
"""
return _render_html(self)
def describe(self) -> str:
lines: list[str] = []
lines.append("=" * 70)
lines.append("PyFlowX 性能剖面报告")
lines.append("=" * 70)
lines.append("")
lines.append("【图级指标】")
lines.append(f" 总耗时 (wall-clock): {self.total_duration:.3f}s")
lines.append(f" 关键路径耗时: {self.critical_path_duration:.3f}s")
lines.append(f" 平均并行度: {self.avg_parallelism:.2f}")
lines.append(f" 峰值并行度: {self.peak_parallelism}")
lines.append(f" 并行度效率: {self.parallelism_efficiency:.2%}")
lines.append(f" 任务总数: {len(self.tasks)}")
lines.append("")
# 关键路径
lines.append("【关键路径】")
if self.critical_path:
lines.append(f" {' -> '.join(self.critical_path)}")
else:
lines.append(" (无)")
lines.append("")
# Top 瓶颈
bottlenecks = self.top_bottlenecks(5)
lines.append(f"【Top {len(bottlenecks)} 瓶颈任务】")
if bottlenecks:
lines.append(f" {'任务':<30} {'耗时':>10} {'等待':>10} {'尝试':>6} {'关键路径':>8} {'状态':>8}")
lines.append(f" {'-' * 30} {'-' * 10} {'-' * 10} {'-' * 6} {'-' * 8} {'-' * 8}")
for t in bottlenecks:
critical_flag = "" if t.is_on_critical_path else ""
lines.append(
f" {t.name:<30} {t.duration:>9.3f}s {t.wait_time:>9.3f}s {t.attempts:>6} "
f"{critical_flag:>8} {t.status.value:>8}",
)
else:
lines.append(" (无)")
lines.append("")
# 全部任务详情
lines.append("【全部任务】")
if self.tasks:
lines.append(f" {'任务':<30} {'耗时':>10} {'等待':>10} {'尝试':>6} {'关键路径':>8} {'状态':>8}")
lines.append(f" {'-' * 30} {'-' * 10} {'-' * 10} {'-' * 6} {'-' * 8} {'-' * 8}")
for t in self.tasks:
critical_flag = "" if t.is_on_critical_path else ""
lines.append(
f" {t.name:<30} {t.duration:>9.3f}s {t.wait_time:>9.3f}s {t.attempts:>6} "
f"{critical_flag:>8} {t.status.value:>8}",
)
else:
lines.append(" (无)")
lines.append("")
lines.append("=" * 70)
return "\n".join(lines)
def __repr__(self) -> str:
return (
f"ProfileReport(tasks={len(self.tasks)}, "
f"total={self.total_duration:.3f}s, "
f"critical={self.critical_path_duration:.3f}s, "
f"avg_par={self.avg_parallelism:.2f}, "
f"peak_par={self.peak_parallelism})"
)
# ---------------------------------------------------------------------- #
# HTML 渲染(私有,零依赖)
# ---------------------------------------------------------------------- #
_HTML_TEMPLATE = """<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>PyFlowX 性能剖面报告</title>
<style>
:root {{
--bg: #f5f5f7;
--card: #ffffff;
--border: #d2d2d7;
--text: #1d1d1f;
--muted: #6e6e73;
--accent: #0071e3;
--success: #34c759;
--warning: #ff9f0a;
--danger: #ff3b30;
--critical: #af52de;
}}
* {{ box-sizing: border-box; }}
body {{
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
margin: 0;
padding: 24px;
background: var(--bg);
color: var(--text);
line-height: 1.5;
}}
h1 {{ margin: 0 0 8px; font-size: 28px; }}
h2 {{ margin: 32px 0 12px; font-size: 20px; border-bottom: 1px solid var(--border); padding-bottom: 6px; }}
.subtitle {{ color: var(--muted); margin: 0 0 24px; font-size: 14px; }}
.cards {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 12px; margin-bottom: 8px; }}
.card {{
background: var(--card);
border: 1px solid var(--border);
border-radius: 10px;
padding: 16px;
}}
.card .label {{ font-size: 12px; color: var(--muted); margin-bottom: 4px; text-transform: uppercase; letter-spacing: 0.5px; }}
.card .value {{ font-size: 22px; font-weight: 600; }}
.card .unit {{ font-size: 13px; color: var(--muted); margin-left: 2px; }}
.critical-path {{
background: var(--card);
border: 1px solid var(--border);
border-left: 4px solid var(--critical);
border-radius: 10px;
padding: 16px;
margin-bottom: 8px;
}}
.critical-path .label {{ font-size: 12px; color: var(--muted); margin-bottom: 8px; text-transform: uppercase; letter-spacing: 0.5px; }}
.critical-path .chain {{ font-family: ui-monospace, "SF Mono", Menlo, monospace; font-size: 13px; word-break: break-all; }}
.critical-path .arrow {{ color: var(--critical); margin: 0 6px; font-weight: 600; }}
/* 甘特图 */
.gantt {{
background: var(--card);
border: 1px solid var(--border);
border-radius: 10px;
padding: 16px;
overflow-x: auto;
}}
.gantt-row {{ display: flex; align-items: center; margin-bottom: 6px; min-width: 600px; }}
.gantt-label {{ width: 200px; flex-shrink: 0; font-size: 13px; font-family: ui-monospace, monospace; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }}
.gantt-track {{ flex: 1; height: 22px; background: #f0f0f3; border-radius: 4px; position: relative; }}
.gantt-bar {{ position: absolute; height: 100%; border-radius: 4px; min-width: 2px; }}
.gantt-bar.success {{ background: var(--success); }}
.gantt-bar.failed {{ background: var(--danger); }}
.gantt-bar.skipped {{ background: var(--muted); }}
.gantt-bar.critical {{ box-shadow: 0 0 0 2px var(--critical) inset; }}
.gantt-bar:hover {{ opacity: 0.85; }}
.gantt-tooltip {{ position: absolute; bottom: 100%; left: 50%; transform: translateX(-50%); background: #1d1d1f; color: #fff; padding: 4px 8px; border-radius: 4px; font-size: 11px; white-space: nowrap; opacity: 0; pointer-events: none; transition: opacity 0.15s; }}
.gantt-bar:hover .gantt-tooltip {{ opacity: 1; }}
/* 表格 */
table {{ width: 100%; border-collapse: collapse; background: var(--card); border-radius: 10px; overflow: hidden; border: 1px solid var(--border); }}
th, td {{ padding: 10px 12px; text-align: left; font-size: 13px; }}
th {{ background: #fafafa; font-weight: 600; color: var(--muted); text-transform: uppercase; font-size: 11px; letter-spacing: 0.5px; }}
tbody tr {{ border-top: 1px solid var(--border); }}
tbody tr:hover {{ background: #fafafa; }}
td.num {{ font-family: ui-monospace, monospace; text-align: right; }}
.badge {{ display: inline-block; padding: 2px 8px; border-radius: 10px; font-size: 11px; font-weight: 500; }}
.badge.success {{ background: rgba(52,199,89,0.15); color: var(--success); }}
.badge.failed {{ background: rgba(255,59,48,0.15); color: var(--danger); }}
.badge.skipped {{ background: rgba(110,110,115,0.15); color: var(--muted); }}
.star {{ color: var(--critical); font-weight: 700; }}
.footer {{ margin-top: 32px; color: var(--muted); font-size: 12px; text-align: center; }}
</style>
</head>
<body>
<h1>PyFlowX 性能剖面报告</h1>
<p class="subtitle">由 <code>pxp</code> 生成 · {generated_at}</p>
<h2>图级指标</h2>
<div class="cards">
<div class="card"><div class="label">总耗时</div><div class="value">{total_duration:.3f}<span class="unit">s</span></div></div>
<div class="card"><div class="label">关键路径耗时</div><div class="value">{critical_duration:.3f}<span class="unit">s</span></div></div>
<div class="card"><div class="label">平均并行度</div><div class="value">{avg_par:.2f}</div></div>
<div class="card"><div class="label">峰值并行度</div><div class="value">{peak_par}</div></div>
<div class="card"><div class="label">并行度效率</div><div class="value">{efficiency:.1f}<span class="unit">%</span></div></div>
<div class="card"><div class="label">任务总数</div><div class="value">{task_count}</div></div>
</div>
<h2>关键路径</h2>
<div class="critical-path">
<div class="label">最长依赖路径(串行瓶颈)</div>
<div class="chain">{critical_chain}</div>
</div>
<h2>任务时间线</h2>
<div class="gantt">
{gantt_rows}
</div>
<h2>Top 瓶颈任务</h2>
<table>
<thead><tr><th>任务</th><th class="num">耗时</th><th class="num">等待</th><th class="num">尝试</th><th>关键路径</th><th>状态</th></tr></thead>
<tbody>
{bottleneck_rows}
</tbody>
</table>
<h2>全部任务</h2>
<table>
<thead><tr><th>任务</th><th class="num">耗时</th><th class="num">等待</th><th class="num">尝试</th><th>关键路径</th><th>状态</th><th>依赖</th></tr></thead>
<tbody>
{all_task_rows}
</tbody>
</table>
<div class="footer">由 PyFlowX · pxp 生成</div>
</body>
</html>"""
def _status_badge(status: TaskStatus) -> str:
"""生成状态徽章 HTML。"""
cls = status.value
return f'<span class="badge {cls}">{cls}</span>'
def _format_critical_chain(path: tuple[str, ...]) -> str:
"""格式化关键路径为 HTML 链。"""
if not path:
return '<em style="color:var(--muted)">(无)</em>'
arrow = '<span class="arrow">→</span>'
return arrow.join(f"<strong>{name}</strong>" for name in path)
def _render_gantt(profile: ProfileReport) -> str:
"""渲染甘特图行 HTML。
每个任务一行:标签 + 时间条。时间条位置基于 wait_time + 依赖关系
重建相对开始时间(相对最早任务起点),归一化到 0-100% 宽度。
SKIPPED 任务不显示(无时间戳)。
"""
visible = [t for t in profile.tasks if t.status != TaskStatus.SKIPPED and t.duration > 0]
if not visible:
return '<div style="color:var(--muted);padding:12px;">(无时间线数据)</div>'
# 重建相对开始时间:start[name] = max(end[dep]) + wait_time
# profile.tasks 已是拓扑序,可直接按序计算
start: dict[str, float] = {}
end: dict[str, float] = {}
for t in profile.tasks:
if t.status == TaskStatus.SKIPPED:
continue
dep_end = 0.0
for dep in t.deps:
dep_end = max(dep_end, end.get(dep, 0.0))
s = dep_end + t.wait_time
start[t.name] = s
end[t.name] = s + t.duration
# 归一化:以最早开始时间为 0,最晚结束为 100%
min_start = min(start.get(t.name, 0.0) for t in visible)
max_end = max(end.get(t.name, 0.0) for t in visible)
span = max_end - min_start
if span <= 0:
span = 1.0
rows: list[str] = []
for t in visible:
s = start.get(t.name, 0.0) - min_start
left_pct = (s / span) * 100
width_pct = (t.duration / span) * 100
cls = t.status.value
critical_cls = " critical" if t.is_on_critical_path else ""
tooltip = f"{t.name}: {t.duration:.3f}s @ +{s:.3f}s ({t.status.value})"
rows.append(
f' <div class="gantt-row">'
f'<div class="gantt-label" title="{t.name}">{t.name}</div>'
f'<div class="gantt-track">'
f'<div class="gantt-bar {cls}{critical_cls}" style="left:{left_pct:.2f}%;width:{width_pct:.2f}%">'
f'<span class="gantt-tooltip">{tooltip}</span>'
f"</div></div></div>"
)
return "\n".join(rows)
def _render_task_row(t: TaskProfile, show_deps: bool = False) -> str:
"""渲染任务表格行 HTML。"""
star = '<span class="star">★</span>' if t.is_on_critical_path else ""
deps = ", ".join(t.deps) if show_deps and t.deps else ""
deps_cell = f"<td>{deps}</td>" if show_deps else ""
return (
f" <tr>"
f"<td><code>{t.name}</code></td>"
f'<td class="num">{t.duration:.3f}s</td>'
f'<td class="num">{t.wait_time:.3f}s</td>'
f'<td class="num">{t.attempts}</td>'
f"<td>{star}</td>"
f"<td>{_status_badge(t.status)}</td>"
f"{deps_cell}"
f"</tr>"
)
def _render_html(profile: ProfileReport) -> str:
"""渲染完整 HTML 报告。"""
from datetime import datetime as _dt
bottlenecks = profile.top_bottlenecks(5)
bottleneck_rows = (
"\n".join(_render_task_row(t) for t in bottlenecks)
or ' <tr><td colspan="6" style="color:var(--muted);">(无)</td></tr>'
)
all_task_rows = (
"\n".join(_render_task_row(t, show_deps=True) for t in profile.tasks)
or ' <tr><td colspan="7" style="color:var(--muted);">(无)</td></tr>'
)
return _HTML_TEMPLATE.format(
generated_at=_dt.now().strftime("%Y-%m-%d %H:%M:%S"),
total_duration=profile.total_duration,
critical_duration=profile.critical_path_duration,
avg_par=profile.avg_parallelism,
peak_par=profile.peak_parallelism,
efficiency=profile.parallelism_efficiency * 100,
task_count=len(profile.tasks),
critical_chain=_format_critical_chain(profile.critical_path),
gantt_rows=_render_gantt(profile),
bottleneck_rows=bottleneck_rows,
all_task_rows=all_task_rows,
)
+16
View File
@@ -69,6 +69,22 @@ class RunReport:
"""以 FAILED 状态结束的任务名列表。"""
return [name for name, r in self.results.items() if r.status == TaskStatus.FAILED]
def succeeded_tasks(self) -> list[str]:
"""以 SUCCESS 状态结束的任务名列表。"""
return [name for name, r in self.results.items() if r.status == TaskStatus.SUCCESS]
def skipped_tasks(self) -> list[str]:
"""以 SKIPPED 状态结束的任务名列表。"""
return [name for name, r in self.results.items() if r.status == TaskStatus.SKIPPED]
def tasks_by_status(self, status: TaskStatus) -> list[str]:
"""返回指定状态的任务名列表。"""
return [name for name, r in self.results.items() if r.status == status]
def durations(self) -> dict[str, float]:
"""任务名 -> 执行时长(秒)。无时长记录的为 0.0。"""
return {name: (r.duration or 0.0) for name, r in self.results.items()}
def describe(self) -> str:
"""用于调试的人类可读多行报告。"""
lines: list[str] = [f"RunReport(success={self.success})"]
+95 -36
View File
@@ -72,67 +72,126 @@ def _apply_verbose_to_graph(graph: Graph, verbose: bool) -> Graph:
class CliRunner:
"""命令行运行器: 根据用户输入执行对应的任务流图.
将命令名映射到 Graph 实例.
通过 ``sys.argv`` 解析用户输入的命令, 执行对应的图.
将命令名映射到 Graph 实例. 通过 ``sys.argv`` 解析用户输入的命令,
执行对应的图.
Parameters
----------
aliases : dict[str, str | list[str] | Graph]
命令别名到任务引用的映射. 每个值可以是:
* ``str`` 单个任务名 (引用 ``tasks`` 中注册的任务),
生成单任务图.
* ``list[str]`` 任务名列表, 自动 :meth:`Graph.chain` 建立链式依赖,
即后一个任务依赖前一个.
* :class:`~pyflowx.graph.Graph` 直接使用该图 (用于复杂场景,
自定义 ``conditions``并行分支等).
tasks : list[TaskSpec]
扁平注册的任务列表. ``aliases`` 中的字符串引用这些任务名.
未被任何 alias 引用的任务不会被执行.
strategy : str | Strategy
默认执行策略 (``Strategy.SEQUENTIAL`` / ``Strategy.THREAD`` /
``Strategy.ASYNC`` 或对应字符串). 可被命令行 ``--strategy`` 覆盖.
默认执行策略. 可被命令行 ``--strategy`` 覆盖.
description : str
CLI 帮助文本.
verbose : bool
是否显示详细执行过程. ``True`` 时打印任务生命周期和 subprocess 输出.
默认 ``True``. 可被命令行 ``--quiet`` 关闭.
**graphs : Graph
命令名到图的映射. 每个 key 是一个命令名, value 是对应的
:class:`~pyflowx.graph.Graph`.
是否显示详细执行过程. 默认 ``True``, 可被命令行 ``--quiet`` 关闭.
Examples
--------
基本用法::
简单场景 (tasks + aliases)::
runner = px.CliRunner(
clean=px.Graph.from_specs(
[
px.TaskSpec("cargo_clean", cmd=["cargo", "clean"]),
]
),
build=px.Graph.from_specs(
[
px.TaskSpec("uv_build", cmd=["uv", "build"]),
]
),
tasks=[
px.cmd(["uv", "build"]), # name="uv_build"
px.cmd(["maturin", "build"], name="maturin_build"),
px.cmd(["ruff", "check", "--fix"], name="lint"),
],
aliases={
"b": "uv_build",
"ba": ["uv_build", "maturin_build"], # chain: maturin 依赖 uv
"lint": "lint",
},
)
runner.run() # 解析 sys.argv
runner.run()
指定策略与描述::
复杂场景 (直接用 Graph)::
runner = px.CliRunner(
strategy=px.Strategy.THREAD,
aliases={
"a": px.Graph.from_specs([
px.TaskSpec("add", cmd=["git", "add", "."], conditions=(...)),
px.TaskSpec("commit", cmd=["git", "commit"], depends_on=("add",)),
]),
},
)
runner.run(["test", "--strategy", "sequential"])
"""
graphs: dict[str, Graph] = field(default_factory=dict)
strategy: Strategy = field(default="sequential")
aliases: dict[str, str | list[str | TaskSpec[Any]] | TaskSpec[Any] | Graph] = field(default_factory=dict)
tasks: list[TaskSpec[Any]] = field(default_factory=list)
strategy: Strategy = field(default="dependency")
description: str = field(default_factory=str)
verbose: bool = field(default_factory=lambda: True)
# 解析后的命令→图映射,__post_init__ 填充
graphs: dict[str, Graph] = field(default_factory=dict, init=False)
def __post_init__(self) -> None:
if not self.graphs:
raise ValueError("CliRunner 至少需要一个命令 (通过关键字参数提供)")
if not self.aliases:
raise ValueError("CliRunner 至少需要一个别名 (通过 aliases= 提供)")
# 解析并展开字符串引用,委托给 GraphComposer。
# Graph 不再 frozen,可直接赋值,无需 object.__setattr__。
self.graphs = GraphComposer(self.graphs).resolve_all()
# 1. 把 tasks 注册为虚拟命令图(每个 task 一个图),加入 raw_graphs
# 使 GraphComposer 能解析对它们的字符串引用
raw_graphs: dict[str, Graph] = {}
for spec in self.tasks:
if spec.name in raw_graphs:
raise ValueError(f"任务名重复: {spec.name!r}")
raw_graphs[spec.name] = Graph.from_specs([spec])
# 2. 把每个 alias 转为 Graphalias 名可与 task 名相同,覆盖 task 注册)
for alias, value in self.aliases.items():
raw_graphs[alias] = self._alias_to_graph(alias, value)
# 3. 解析图间字符串引用(str / list[str] 引用其他 alias 或任务)
self.graphs = GraphComposer(raw_graphs).resolve_all()
@staticmethod
def _alias_to_graph(
alias: str,
value: str | list[str | TaskSpec[Any]] | TaskSpec[Any] | Graph,
) -> Graph:
"""把 alias 的值转换为 Graph.
* ``str`` 对其他 alias 或已注册任务名的引用, GraphComposer 展开.
* ``TaskSpec`` 单个内联任务, 生成单任务图.
* ``list[str | TaskSpec]`` 引用/任务混合列表, GraphComposer 展开时
自动让后续引用依赖前面 (chain 语义). 元素为 alias 任务名或
:class:`TaskSpec` 对象 (内联任务).
* ``Graph`` 原样返回 (用于复杂场景: conditions并行分支等).
"""
if isinstance(value, Graph):
return value
if isinstance(value, TaskSpec):
return Graph.from_specs([value])
if isinstance(value, str):
# 字符串引用,用 _pending_refs 占位,GraphComposer 后续展开
return Graph.from_specs([value]) # type: ignore[arg-type]
if isinstance(value, list):
if not value:
raise ValueError(f"别名 {alias!r} 的任务列表为空")
for item in value:
if not isinstance(item, (str, TaskSpec)):
raise TypeError(f"别名 {alias!r} 的列表元素类型无效: {type(item).__name__}, 预期 str 或 TaskSpec")
# str/TaskSpec 混合列表,由 GraphComposer 展开(自动建立 chain 依赖)
return Graph.from_specs(value)
raise TypeError(
f"别名 {alias!r} 的值类型无效: {type(value).__name__}, 预期 str/TaskSpec/list[str|TaskSpec]/Graph"
)
# ------------------------------------------------------------------ #
# 内省
# ------------------------------------------------------------------ #
@property
def commands(self) -> list[str]:
"""可用的命令列表 (按插入顺序)."""
return list(self.graphs.keys())
"""可用的命令列表 (按 aliases 定义顺序, 不含 tasks 中未引用的任务)."""
return list(self.aliases.keys())
# ------------------------------------------------------------------ #
# 参数解析
@@ -225,9 +284,9 @@ class CliRunner:
parser.print_help()
return CliExitCode.FAILURE.value
# 验证命令
if parsed.command not in self.graphs:
available = ", ".join(self.graphs.keys())
# 验证命令(必须是已注册的 alias,不接受裸任务名)
if parsed.command not in self.aliases:
available = ", ".join(self.commands)
print(
f"错误: 未知命令 {parsed.command!r} (可用命令: {available})",
file=sys.stderr,
+120
View File
@@ -254,6 +254,10 @@ class TaskSpec(Generic[T]):
存取状态后端使不同输入产生独立缓存条目``None`` 表示用任务名
hooks:
:class:`TaskHooks` 生命周期钩子
executor:
同步任务的执行器``"thread"``默认线程池/ ``"process"``
进程池绕过 GIL适合 CPU 密集型``fn`` 须可 pickle/
``"inline"``直接在事件循环线程调用最快但会阻塞循环
"""
name: str
@@ -279,6 +283,7 @@ class TaskSpec(Generic[T]):
continue_on_error: bool = False
cache_key: CacheKeyFn | None = None
hooks: TaskHooks = field(default_factory=TaskHooks)
executor: str = "thread" # "thread" | "process" | "inline"
def __post_init__(self) -> None:
if not self.name:
@@ -447,6 +452,121 @@ def _env_and_cwd(
# ---------------------------------------------------------------------- #
# 任务模板:批量生成相似 TaskSpec 的工厂
# ---------------------------------------------------------------------- #
def _task_noop() -> None:
"""task(cmd=...) 形式下的占位 fn(cmd 任务执行期不调用 fn)。"""
return None
def task(
fn: TaskFn[Any] | None = None,
*,
cmd: TaskCmd | None = None,
depends_on: tuple[str, ...] = (),
soft_depends_on: tuple[str, ...] = (),
defaults: Mapping[str, Any] | None = None,
args: tuple[Any, ...] = (),
kwargs: Mapping[str, Any] | None = None,
retry: RetryPolicy | None = None,
timeout: float | None = None,
tags: tuple[str, ...] = (),
conditions: tuple[Condition, ...] = (),
cwd: str | Path | None = None,
env: Mapping[str, str] | None = None,
verbose: bool = False,
skip_if_missing: bool = False,
allow_upstream_skip: bool = False,
strategy: str | None = None,
priority: int = 0,
concurrency_key: str | None = None,
continue_on_error: bool = False,
cache_key: CacheKeyFn | None = None,
hooks: TaskHooks | None = None,
name: str | None = None,
) -> Any:
"""装饰器:将函数转为 :class:`TaskSpec`。
``name`` 默认取 ``fn.__name__``可直接装饰函数或带参数使用
Examples
--------
>>> @px.task
... def extract(): return [1, 2, 3]
>>> @px.task(depends_on=("extract",))
... def double(extract): return [x * 2 for x in extract]
>>> graph = px.Graph.from_specs([extract, double])
"""
def _decorate(func: TaskFn[Any]) -> TaskSpec[Any]:
spec_name = name or func.__name__
return TaskSpec(
name=spec_name,
fn=func,
cmd=cmd,
depends_on=depends_on,
soft_depends_on=soft_depends_on,
defaults=dict(defaults) if defaults else {},
args=args,
kwargs=dict(kwargs) if kwargs else {},
retry=retry if retry is not None else RetryPolicy(),
timeout=timeout,
tags=tags,
conditions=conditions,
cwd=Path(cwd) if isinstance(cwd, str) else cwd,
env=dict(env) if env else None,
verbose=verbose,
skip_if_missing=skip_if_missing,
allow_upstream_skip=allow_upstream_skip,
strategy=strategy,
priority=priority,
concurrency_key=concurrency_key,
continue_on_error=continue_on_error,
cache_key=cache_key,
hooks=hooks if hooks is not None else TaskHooks(),
)
if fn is None and cmd is None:
# 带参数调用:@task(depends_on=...),等待被装饰函数
return _decorate
if fn is None:
# task(cmd=..., name=...) 直接构造,无被装饰函数
if name is None:
raise ValueError("task(cmd=...) 需要显式提供 name")
return _decorate(_task_noop)
return _decorate(fn)
def cmd(
command: list[str],
*,
name: str | None = None,
depends_on: tuple[str, ...] = (),
**kwargs: Any,
) -> TaskSpec[Any]:
"""从命令列表快速创建 :class:`TaskSpec`。
``name`` 默认为 ``"_".join(command[:2])`` ``["uv", "build"]`` ``"uv_build"``
若命令不足两个元素则用 ``"_".join(command)``
其余关键字参数透传给 :class:`TaskSpec` ``depends_on````tags``
Examples
--------
>>> uv_build = px.cmd(["uv", "build"])
>>> uv_build.name
'uv_build'
>>> lint = px.cmd(["ruff", "check", "--fix"], name="lint")
>>> lint.name
'lint'
"""
spec_name = name or "_".join(command[:2]) if len(command) >= 2 else "_".join(command)
return TaskSpec(
name=spec_name,
cmd=command,
depends_on=depends_on,
**kwargs,
)
def task_template(
fn: TaskFn[Any] | None = None,
cmd: TaskCmd | None = None,
+26
View File
@@ -0,0 +1,26 @@
"""进程池测试辅助:模块级函数(须可 pickle)。"""
from __future__ import annotations
import time
def cpu_heavy(n: int) -> int:
"""CPU 密集型计算(求平方和)。"""
return sum(i * i for i in range(n))
def add(a: int, b: int) -> int:
"""简单加法。"""
return a + b
def sub(a: int, b: int) -> int:
"""简单减法。"""
return a - b
def slow_sleep(seconds: float) -> int:
"""睡眠指定秒数,用于测试超时。"""
time.sleep(seconds)
return int(seconds)
+545
View File
@@ -0,0 +1,545 @@
"""pxp 性能分析器测试.
覆盖策略
* HTML 渲染to_html() 输出结构正确含关键章节
* pxp CLI参数解析脚本执行报告生成浏览器调用错误处理
* hook 注入捕获 px.run() 调用还原原始函数
"""
from __future__ import annotations
import sys
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any
import pytest
import pyflowx as px
from pyflowx.cli import profiler
from pyflowx.profiling import ProfileReport
from pyflowx.report import RunReport
from pyflowx.task import TaskResult, TaskSpec, TaskStatus
def _fn() -> int:
return 1
def _spec(name: str, deps: tuple[str, ...] = ()) -> TaskSpec[Any]:
return TaskSpec[Any](name, _fn, depends_on=deps)
def _result(
name: str,
start: datetime,
duration: float,
*,
status: TaskStatus = TaskStatus.SUCCESS,
attempts: int = 1,
) -> TaskResult[Any]:
"""构造带时间戳的 TaskResult."""
end = start + timedelta(seconds=duration) if duration > 0 else start
return TaskResult[Any](
spec=_spec(name),
status=status,
value=None,
attempts=attempts,
started_at=start if duration > 0 or status != TaskStatus.SKIPPED else None,
finished_at=end if duration > 0 or status != TaskStatus.SKIPPED else None,
)
def _build_simple_profile() -> ProfileReport:
"""构造一个简单的 ProfileReport 用于测试 HTML 输出."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
report.results["b"] = _result("b", start + timedelta(seconds=1), 2.0)
graph = px.Graph.from_specs([
_spec("a"),
_spec("b", deps=("a",)),
])
return ProfileReport.from_report(report, graph)
class TestToHtml:
"""测试 ProfileReport.to_html()."""
def test_to_html_contains_key_sections(self) -> None:
"""HTML 应包含所有关键章节标题。"""
profile = _build_simple_profile()
html = profile.to_html()
assert "<!DOCTYPE html>" in html
assert "PyFlowX 性能剖面报告" in html
assert "图级指标" in html
assert "关键路径" in html
assert "任务时间线" in html
assert "Top 瓶颈任务" in html
assert "全部任务" in html
def test_to_html_contains_metrics(self) -> None:
"""HTML 应包含图级指标数值。"""
profile = _build_simple_profile()
html = profile.to_html()
# 总耗时 3.0s (a=1 + b=2)
assert "3.000" in html
# 任务名
assert "a" in html
assert "b" in html
def test_to_html_contains_critical_path(self) -> None:
"""HTML 应包含关键路径任务链。"""
profile = _build_simple_profile()
html = profile.to_html()
# 关键路径是 a -> b
assert "<strong>a</strong>" in html
assert "<strong>b</strong>" in html
def test_to_html_contains_gantt_bars(self) -> None:
"""HTML 应包含甘特图条。"""
profile = _build_simple_profile()
html = profile.to_html()
assert "gantt-row" in html
assert "gantt-bar" in html
# 每个非 SKIPPED 任务一个条
assert html.count("gantt-bar") >= 2
def test_to_html_empty_profile(self) -> None:
"""空报告的 HTML 应不崩溃。"""
report = px.RunReport()
graph = px.Graph()
profile = ProfileReport.from_report(report, graph)
html = profile.to_html()
assert "PyFlowX 性能剖面报告" in html
assert "(无)" in html
def test_to_html_with_failed_task(self) -> None:
"""含 FAILED 任务的 HTML 应包含失败状态徽章。"""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0, status=TaskStatus.FAILED)
graph = px.Graph.from_specs([_spec("a")])
profile = ProfileReport.from_report(report, graph)
html = profile.to_html()
assert "failed" in html
assert "badge" in html
def test_to_html_with_skipped_task(self) -> None:
"""含 SKIPPED 任务的 HTML 不应在甘特图中显示该任务。"""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
report.results["b"] = TaskResult[Any](
spec=_spec("b"),
status=TaskStatus.SKIPPED,
reason="skip",
)
graph = px.Graph.from_specs([_spec("a"), _spec("b")])
profile = ProfileReport.from_report(report, graph)
html = profile.to_html()
# SKIPPED 任务的徽章应出现
assert "skipped" in html
def test_to_html_self_contained(self) -> None:
"""HTML 应自包含(无外部依赖)。"""
profile = _build_simple_profile()
html = profile.to_html()
# 不引用外部资源
assert "<link" not in html
assert "<script src" not in html
class TestProfilerArgumentParsing:
"""测试 pxp CLI 参数解析。"""
def test_default_export_is_html(self) -> None:
"""默认导出格式为 html。"""
parser = profiler._build_parser()
args, remaining = parser.parse_known_args(["pymake.py"])
assert args.export == "html"
assert args.no_browser is False
assert args.output is None
assert remaining == ["pymake.py"]
def test_export_text(self) -> None:
"""-E text 应设置导出格式为 text。"""
parser = profiler._build_parser()
args, _ = parser.parse_known_args(["-E", "text", "pymake.py"])
assert args.export == "text"
def test_no_browser_flag(self) -> None:
"""--no-browser 应设置标志。"""
parser = profiler._build_parser()
args, _ = parser.parse_known_args(["--no-browser", "pymake.py"])
assert args.no_browser is True
def test_output_option(self) -> None:
"""-o 应设置输出路径。"""
parser = profiler._build_parser()
args, _ = parser.parse_known_args(["-o", "report.html", "pymake.py"])
assert args.output == "report.html"
def test_script_args_separated(self) -> None:
"""脚本参数应通过 remaining 分离。"""
parser = profiler._build_parser()
_, remaining = parser.parse_known_args(["pymake.py", "t", "--quiet"])
assert remaining == ["pymake.py", "t", "--quiet"]
def test_no_args_prints_help(
self,
capsys: pytest.CaptureFixture[str],
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""无参数应打印帮助并以退出码 2 退出。"""
monkeypatch.setattr(sys, "argv", ["pxp"])
with pytest.raises(SystemExit) as exc_info:
profiler.main()
assert exc_info.value.code == 2
captured = capsys.readouterr()
assert "usage" in captured.out.lower() or "usage" in captured.err.lower()
class TestCapturePxRun:
"""测试 _capture_px_run hook 注入。"""
def test_capture_captures_run_call(self) -> None:
"""hook 应捕获 px.run() 调用的 graph 和 report。"""
captured = profiler._capture_px_run()
try:
graph = px.Graph.from_specs([px.TaskSpec("a", lambda: 1)])
px.run(graph, strategy="sequential")
assert "graph" in captured
assert "report" in captured
assert captured["graph"] is graph
finally:
captured["_restore"]()
def test_capture_restores_original(self) -> None:
"""还原后 px.run 和 RunReport.__init__ 应恢复为原函数。"""
original_run = px.run
original_init = RunReport.__init__
captured = profiler._capture_px_run()
# 注入期间 px.run 和 RunReport.__init__ 已被替换
assert px.run is not original_run
assert RunReport.__init__ is not original_init
captured["_restore"]()
# 还原后恢复
assert px.run is original_run
assert RunReport.__init__ is original_init
def test_capture_via_runner_run(self) -> None:
"""hook 应捕获通过 CliRunner 执行的 run() 调用。"""
from pyflowx import runner as runner_mod
captured = profiler._capture_px_run()
try:
# 验证 runner.run 也被 patch(指向 patched_run
assert runner_mod.run is px.executors.run
graph = px.Graph.from_specs([px.TaskSpec("a", lambda: 1)])
runner_mod.run(graph, strategy="sequential")
assert "report" in captured
finally:
captured["_restore"]()
def test_capture_captures_report_on_failure(self) -> None:
"""run() 抛出 TaskFailedError 时仍应捕获 report 实例。"""
from pyflowx.executors import TaskFailedError
def failing() -> None:
raise RuntimeError("boom")
graph = px.Graph.from_specs([px.TaskSpec("a", failing)])
captured = profiler._capture_px_run()
try:
with pytest.raises(TaskFailedError):
px.run(graph, strategy="sequential")
# 即使 run() 抛异常,report 也应被捕获(含已执行任务的结果)
assert "report" in captured
assert "graph" in captured
assert captured["graph"] is graph
finally:
captured["_restore"]()
class TestRunTargetScript:
"""测试 _run_target_script。"""
def test_run_simple_script(self, tmp_path: Path) -> None:
"""应能执行简单脚本并返回模块字典。"""
script = tmp_path / "simple.py"
script.write_text("x = 42\n", encoding="utf-8")
result = profiler._run_target_script(script, [])
assert result["x"] == 42
def test_run_script_with_sys_exit(self, tmp_path: Path) -> None:
"""脚本调用 sys.exit 应抛 SystemExit。"""
script = tmp_path / "exit.py"
script.write_text("import sys; sys.exit(0)\n", encoding="utf-8")
with pytest.raises(SystemExit):
profiler._run_target_script(script, [])
def test_run_script_sets_argv(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""应正确设置 sys.argv。"""
script = tmp_path / "argv.py"
script.write_text(
"import sys\nassert sys.argv[0] == __file__\nassert sys.argv[1:] == ['arg1', 'arg2']\n",
encoding="utf-8",
)
profiler._run_target_script(script, ["arg1", "arg2"])
def test_run_script_adds_dir_to_path(self, tmp_path: Path) -> None:
"""脚本所在目录应加入 sys.path。"""
script = tmp_path / "pathcheck.py"
script.write_text(
"import sys, os\nassert os.path.dirname(__file__) in sys.path\n",
encoding="utf-8",
)
profiler._run_target_script(script, [])
class TestOutputReport:
"""测试 _output_report。"""
def test_output_text_format(
self,
capsys: pytest.CaptureFixture[str],
) -> None:
"""text 格式应打印 describe() 到 stdout。"""
profile = _build_simple_profile()
profiler._output_report(profile, export="text", output=None, script_stem="test", no_browser=True)
captured = capsys.readouterr()
assert "PyFlowX 性能剖面报告" in captured.out
assert "图级指标" in captured.out
def test_output_html_default_filename(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""HTML 默认输出到 <script>_profile.html。"""
monkeypatch.chdir(tmp_path)
profile = _build_simple_profile()
profiler._output_report(profile, export="html", output=None, script_stem="mymake", no_browser=True)
out_file = tmp_path / "mymake_profile.html"
assert out_file.exists()
content = out_file.read_text(encoding="utf-8")
assert "PyFlowX 性能剖面报告" in content
def test_output_html_custom_path(self, tmp_path: Path) -> None:
"""HTML 应写入指定路径。"""
out_file = tmp_path / "custom.html"
profile = _build_simple_profile()
profiler._output_report(profile, export="html", output=str(out_file), script_stem="test", no_browser=True)
assert out_file.exists()
assert "PyFlowX" in out_file.read_text(encoding="utf-8")
def test_output_html_opens_browser(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""no_browser=False 应调用 webbrowser.open。"""
monkeypatch.chdir(tmp_path)
opened: list[str] = []
monkeypatch.setattr(profiler.webbrowser, "open", opened.append)
profile = _build_simple_profile()
profiler._output_report(profile, export="html", output=None, script_stem="test", no_browser=False)
assert len(opened) == 1
assert opened[0].startswith("file://")
def test_output_html_no_browser_flag(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""no_browser=True 不应调用 webbrowser.open。"""
monkeypatch.chdir(tmp_path)
opened: list[str] = []
monkeypatch.setattr(profiler.webbrowser, "open", opened.append)
profile = _build_simple_profile()
profiler._output_report(profile, export="html", output=None, script_stem="test", no_browser=True)
assert len(opened) == 0
class TestProfilerMainIntegration:
"""main() 集成测试。"""
def test_main_analyses_script_with_px_run(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""main() 应分析含 px.run() 的脚本并生成 HTML。"""
script = tmp_path / "mytool.py"
script.write_text(
"import pyflowx as px\n"
"graph = px.Graph.from_specs([\n"
" px.TaskSpec('a', lambda: 1),\n"
" px.TaskSpec('b', lambda: 2, depends_on=('a',)),\n"
"])\n"
"px.run(graph, strategy='sequential')\n",
encoding="utf-8",
)
out_file = tmp_path / "report.html"
monkeypatch.setattr(sys, "argv", ["pxp", "--no-browser", "-o", str(out_file), str(script)])
profiler.main()
assert out_file.exists()
content = out_file.read_text(encoding="utf-8")
assert "PyFlowX 性能剖面报告" in content
assert "任务时间线" in content
def test_main_analyses_script_with_clirunner(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""main() 应分析含 CliRunner 的脚本。"""
script = tmp_path / "clirunner_tool.py"
script.write_text(
"import pyflowx as px\n"
"runner = px.CliRunner(\n"
" aliases={'t': px.TaskSpec('t', lambda: 1)},\n"
")\n"
"runner.run_cli(['t'])\n",
encoding="utf-8",
)
out_file = tmp_path / "report.html"
monkeypatch.setattr(sys, "argv", ["pxp", "--no-browser", "-o", str(out_file), str(script)])
profiler.main()
assert out_file.exists()
content = out_file.read_text(encoding="utf-8")
assert "PyFlowX 性能剖面报告" in content
def test_main_text_export(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
"""main() -E text 应输出文本到 stdout。"""
script = tmp_path / "simple.py"
script.write_text(
"import pyflowx as px\n"
"graph = px.Graph.from_specs([px.TaskSpec('a', lambda: 1)])\n"
"px.run(graph, strategy='sequential')\n",
encoding="utf-8",
)
monkeypatch.setattr(sys, "argv", ["pxp", "-E", "text", "--no-browser", str(script)])
profiler.main()
captured = capsys.readouterr()
assert "PyFlowX 性能剖面报告" in captured.out
def test_main_script_not_exist(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
"""脚本不存在应以退出码 2 退出。"""
monkeypatch.setattr(sys, "argv", ["pxp", "--no-browser", str(tmp_path / "nonexistent.py")])
with pytest.raises(SystemExit) as exc_info:
profiler.main()
assert exc_info.value.code == 2
def test_main_no_px_run_captured(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""脚本未调用 px.run() 应以退出码 1 退出。"""
script = tmp_path / "no_run.py"
script.write_text("print('just printing')\n", encoding="utf-8")
monkeypatch.setattr(sys, "argv", ["pxp", "--no-browser", str(script)])
with pytest.raises(SystemExit) as exc_info:
profiler.main()
assert exc_info.value.code == 1
def test_main_passes_script_args(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""应将脚本参数传递给目标脚本。"""
script = tmp_path / "argcheck.py"
script.write_text(
"import sys\n"
"assert sys.argv[1:] == ['myarg'], f'got {sys.argv[1:]}'\n"
"import pyflowx as px\n"
"px.run(px.Graph.from_specs([px.TaskSpec('a', lambda: 1)]), strategy='sequential')\n",
encoding="utf-8",
)
out_file = tmp_path / "report.html"
monkeypatch.setattr(sys, "argv", ["pxp", "--no-browser", "-o", str(out_file), str(script), "myarg"])
profiler.main() # 不抛异常即成功
def test_main_handles_script_exception(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""脚本抛异常时应捕获并继续生成报告(如果有 report)。"""
script = tmp_path / "raise.py"
script.write_text(
"import pyflowx as px\n"
"px.run(px.Graph.from_specs([px.TaskSpec('a', lambda: 1)]), strategy='sequential')\n"
"raise RuntimeError('after run')\n",
encoding="utf-8",
)
out_file = tmp_path / "report.html"
monkeypatch.setattr(sys, "argv", ["pxp", "--no-browser", "-o", str(out_file), str(script)])
profiler.main() # 不抛异常即成功
assert out_file.exists()
def test_main_auto_calls_main_when_no_main_block(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""脚本无 __main__ 块但定义了 main() 时应自动调用。"""
script = tmp_path / "no_main_block.py"
script.write_text(
"import pyflowx as px\n"
"def main():\n"
" px.run(px.Graph.from_specs([px.TaskSpec('a', lambda: 1)]), strategy='sequential')\n"
"# 无 if __name__ == '__main__'\n",
encoding="utf-8",
)
out_file = tmp_path / "report.html"
monkeypatch.setattr(sys, "argv", ["pxp", "--no-browser", "-o", str(out_file), str(script)])
profiler.main()
assert out_file.exists()
def test_main_auto_calls_main_with_clirunner(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""脚本无 __main__ 块但定义了调用 CliRunner 的 main() 时应自动调用。"""
script = tmp_path / "cli_tool.py"
script.write_text(
"import pyflowx as px\n"
"def main():\n"
" runner = px.CliRunner(\n"
" aliases={'t': px.TaskSpec('t', lambda: 1)},\n"
" )\n"
" runner.run_cli(['t'])\n",
encoding="utf-8",
)
out_file = tmp_path / "report.html"
monkeypatch.setattr(sys, "argv", ["pxp", "--no-browser", "-o", str(out_file), str(script), "t"])
profiler.main()
assert out_file.exists()
content = out_file.read_text(encoding="utf-8")
assert "PyFlowX 性能剖面报告" in content
def test_main_no_main_function_exits_with_1(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""脚本无 main() 且未调用 px.run() 应以退出码 1 退出。"""
script = tmp_path / "no_main.py"
script.write_text("x = 1\n", encoding="utf-8")
monkeypatch.setattr(sys, "argv", ["pxp", "--no-browser", str(script)])
with pytest.raises(SystemExit) as exc_info:
profiler.main()
assert exc_info.value.code == 1
class TestTryCallMain:
"""测试 _try_call_main。"""
def test_calls_main_when_present(self) -> None:
"""模块字典含 main 可调用对象时应调用它。"""
called: list[bool] = []
def fake_main() -> None:
called.append(True)
profiler._try_call_main({"main": fake_main})
assert called == [True]
def test_no_main_does_nothing(self) -> None:
"""模块字典不含 main 时不应报错。"""
profiler._try_call_main({}) # 不抛异常即成功
def test_non_callable_main_does_nothing(self) -> None:
"""main 不是可调用对象时不应报错。"""
profiler._try_call_main({"main": "not a function"}) # 不抛异常即成功
+91 -108
View File
@@ -2,160 +2,143 @@
from __future__ import annotations
from pathlib import Path
from unittest.mock import patch
import pytest
from pyflowx.cli import pymake
from pyflowx.conditions import Constants
# ---------------------------------------------------------------------- #
# maturin_build_cmd
# ---------------------------------------------------------------------- #
class TestMaturinBuildCmd:
"""Test maturin_build_cmd function."""
def test_returns_list(self) -> None:
"""Should return a list."""
cmd = pymake.maturin_build_cmd()
assert isinstance(cmd, list)
def test_contains_maturin_build(self) -> None:
"""Should contain 'maturin' and 'build'."""
cmd = pymake.maturin_build_cmd()
assert "maturin" in cmd
assert "build" in cmd
def test_contains_release_flag(self) -> None:
"""Should contain release flag '-r'."""
cmd = pymake.maturin_build_cmd()
assert "-r" in cmd
def test_windows_includes_target(self) -> None:
"""On Windows, should include target-specific flags."""
cmd = pymake.maturin_build_cmd()
if Constants.IS_WINDOWS:
assert "--target" in cmd
assert "x86_64-win7-windows-msvc" in cmd
assert "-Zbuild-std" in cmd
assert "-i" in cmd
assert "python3.8" in cmd
else:
# On non-Windows, should not include Windows-specific flags
assert "--target" not in cmd
def test_does_not_mutate_on_multiple_calls(self) -> None:
"""Multiple calls should return independent lists."""
cmd1 = pymake.maturin_build_cmd()
cmd2 = pymake.maturin_build_cmd()
assert cmd1 == cmd2
# Mutating one should not affect the other
cmd1.append("extra")
assert "extra" not in cmd2
def test_non_windows_excludes_target_flags(self) -> None:
"""On non-Windows, should not include Windows-specific flags (覆盖 22->32 分支)."""
from unittest.mock import patch
with patch.object(pymake.Constants, "IS_WINDOWS", False):
cmd = pymake.maturin_build_cmd()
assert "maturin" in cmd
assert "build" in cmd
assert "-r" in cmd
assert "--target" not in cmd
assert "-Zbuild-std" not in cmd
# ---------------------------------------------------------------------- #
# TaskSpec definitions
# ---------------------------------------------------------------------- #
def _find_task(name: str) -> pymake.px.TaskSpec:
"""从 pymake.tasks 或 aliases 中查找指定名称的 TaskSpec."""
for spec in pymake.tasks:
if spec.name == name:
return spec
# 单任务别名(doc/lint/tox)内联在 aliases dict 中
value = pymake.aliases.get(name)
if isinstance(value, pymake.px.TaskSpec):
return value
raise KeyError(f"任务 {name!r} 未找到")
class TestTaskSpecDefinitions:
"""Test that all TaskSpec definitions are valid."""
def test_uv_build_spec(self) -> None:
"""uv_build spec should be properly defined."""
assert pymake.uv_build.name == "uv_build"
assert pymake.uv_build.cmd == ["uv", "build"]
assert pymake.uv_build.skip_if_missing is False
spec = _find_task("uv_build")
assert spec.name == "uv_build"
assert spec.cmd == ["uv", "build"]
assert spec.skip_if_missing is False
def test_maturin_build_spec(self) -> None:
"""maturin_build spec should be properly defined."""
assert pymake.maturin_build.name == "maturin_build"
assert isinstance(pymake.maturin_build.cmd, list)
assert pymake.maturin_build.skip_if_missing is False
spec = _find_task("maturin_build")
assert spec.name == "maturin_build"
assert isinstance(spec.cmd, list)
assert spec.skip_if_missing is False
def test_uv_sync_spec(self) -> None:
"""uv_sync spec should be properly defined."""
assert pymake.uv_sync.name == "uv_sync"
assert pymake.uv_sync.cmd == ["uv", "sync"]
assert pymake.uv_sync.skip_if_missing is False
spec = _find_task("uv_sync")
assert spec.name == "uv_sync"
assert spec.cmd == ["uv", "sync"]
assert spec.skip_if_missing is False
def test_git_clean_spec(self) -> None:
"""git_clean spec should be properly defined."""
assert pymake.git_clean.name == "git_clean"
assert pymake.git_clean.cmd == ["gitt", "c"]
assert pymake.git_clean.skip_if_missing is False
spec = _find_task("git_clean")
assert spec.name == "git_clean"
assert spec.cmd == ["gitt", "c"]
assert spec.skip_if_missing is False
def test_test_spec(self) -> None:
"""test spec should be properly defined."""
assert pymake.test.name == "test"
assert isinstance(pymake.test.cmd, list)
assert "pytest" in pymake.test.cmd
assert "-m" in pymake.test.cmd
assert "not slow" in pymake.test.cmd
assert pymake.test.skip_if_missing is False
spec = _find_task("test")
assert spec.name == "test"
assert isinstance(spec.cmd, list)
assert "pytest" in spec.cmd
assert "-m" in spec.cmd
assert "not slow" in spec.cmd
assert spec.skip_if_missing is False
def test_test_fast_spec(self) -> None:
"""test_fast spec should be properly defined."""
assert pymake.test_fast.name == "test_fast"
assert isinstance(pymake.test_fast.cmd, list)
assert "pytest" in pymake.test_fast.cmd
assert "-n" not in pymake.test_fast.cmd # test_fast doesn't use parallel
assert pymake.test_fast.skip_if_missing is False
spec = _find_task("test_fast")
assert spec.name == "test_fast"
assert isinstance(spec.cmd, list)
assert "pytest" in spec.cmd
assert "-n" not in spec.cmd # test_fast doesn't use parallel
assert spec.skip_if_missing is False
def test_test_coverage_spec(self) -> None:
"""test_coverage spec should be properly defined."""
assert pymake.test_coverage.name == "test_coverage"
assert isinstance(pymake.test_coverage.cmd, list)
assert "pytest" in pymake.test_coverage.cmd
assert "--cov" in pymake.test_coverage.cmd
assert pymake.test_coverage.skip_if_missing is False
spec = _find_task("test_coverage")
assert spec.name == "test_coverage"
assert isinstance(spec.cmd, list)
assert "pytest" in spec.cmd
assert "--cov" in spec.cmd
assert spec.skip_if_missing is False
def test_ruff_lint_spec(self) -> None:
"""ruff_lint spec should be properly defined."""
assert pymake.ruff_lint.name == "lint"
assert isinstance(pymake.ruff_lint.cmd, list)
assert "ruff" in pymake.ruff_lint.cmd
assert "check" in pymake.ruff_lint.cmd
assert pymake.ruff_lint.skip_if_missing is False
"""lint spec should be properly defined."""
spec = _find_task("lint")
assert spec.name == "lint"
assert isinstance(spec.cmd, list)
assert "ruff" in spec.cmd
assert "check" in spec.cmd
assert spec.skip_if_missing is False
def test_doc_spec(self) -> None:
"""doc spec should be properly defined."""
assert pymake.doc.name == "doc"
assert isinstance(pymake.doc.cmd, list)
assert "sphinx-build" in pymake.doc.cmd
assert pymake.doc.skip_if_missing is False
spec = _find_task("doc")
assert spec.name == "doc"
assert isinstance(spec.cmd, list)
assert "sphinx-build" in spec.cmd
assert spec.skip_if_missing is False
def test_hatch_publish_spec(self) -> None:
"""hatch_publish spec should be properly defined."""
assert pymake.hatch_publish.name == "publish_python"
assert pymake.hatch_publish.cmd == ["hatch", "publish"]
assert pymake.hatch_publish.skip_if_missing is False
"""publish_python spec should be properly defined."""
spec = _find_task("publish_python")
assert spec.name == "publish_python"
assert spec.cmd == ["hatch", "publish"]
assert spec.skip_if_missing is False
def test_twine_publish_spec(self) -> None:
"""twine_publish spec should be properly defined."""
assert pymake.twine_publish.name == "twine_publish"
assert isinstance(pymake.twine_publish.cmd, list)
assert "twine" in pymake.twine_publish.cmd
assert "upload" in pymake.twine_publish.cmd
assert pymake.twine_publish.skip_if_missing is False
spec = _find_task("twine_publish")
assert spec.name == "twine_publish"
assert isinstance(spec.cmd, list)
assert "twine" in spec.cmd
assert "upload" in spec.cmd
assert spec.skip_if_missing is False
def test_tox_spec(self) -> None:
"""tox spec should be properly defined."""
assert pymake.tox.name == "tox"
assert pymake.tox.cmd == ["tox", "-p", "auto"]
assert pymake.tox.skip_if_missing is False
spec = _find_task("tox")
assert spec.name == "tox"
assert spec.cmd == ["tox", "-p", "auto"]
assert spec.skip_if_missing is False
def test_all_tasks_have_correct_cwd(self) -> None:
"""所有任务应该有正确的 cwd 设置(指向项目根目录)."""
# 验证 ROOT_DIR 定义正确(向上三层到达项目根目录)
expected_root = Path(__file__).parent.parent.parent
assert expected_root == pymake.ROOT_DIR
# 验证 tasks 中的所有命令任务都有正确的 cwd
for spec in pymake.tasks:
if spec.cmd is not None:
assert spec.cwd == pymake.ROOT_DIR, f"任务 {spec.name} 的 cwd 应为 {pymake.ROOT_DIR}"
# 验证 aliases 中的内联任务(doc/lint/tox)也有正确的 cwd
for name in ("doc", "lint", "tox"):
spec = _find_task(name)
assert spec.cwd == pymake.ROOT_DIR, f"任务 {name} 的 cwd 应为 {pymake.ROOT_DIR}"
# ---------------------------------------------------------------------- #
+7
View File
@@ -1,9 +1,16 @@
from __future__ import annotations
import sys
from pathlib import Path
import pytest
# 将 tests 目录加入 sys.path,使进程池测试能 import _proc_helper 模块级辅助函数。
# 进程池 pickle 要求被调用函数为模块级,conftest.py 在 xdist worker 中也会执行。
_TESTS_DIR = str(Path(__file__).resolve().parent)
if _TESTS_DIR not in sys.path:
sys.path.insert(0, _TESTS_DIR)
@pytest.fixture(autouse=True)
def packtool_tmp_workdir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+101
View File
@@ -0,0 +1,101 @@
"""Tests for Graph.chain DSL."""
from __future__ import annotations
import pyflowx as px
from pyflowx.task import TaskSpec
def _fn() -> None:
return None
def test_chain_basic_linkage() -> None:
"""chain(a, b, c) 应建立 a->b->c 依赖."""
a = TaskSpec("a", _fn)
b = TaskSpec("b", _fn)
c = TaskSpec("c", _fn)
graph = px.Graph().chain(a, b, c)
assert graph.all_specs()["b"].depends_on == ("a",)
assert graph.all_specs()["c"].depends_on == ("b",)
assert graph.all_specs()["a"].depends_on == ()
def test_chain_single_spec() -> None:
"""chain(a) 应只注册 a,无依赖."""
a = TaskSpec("a", _fn)
graph = px.Graph().chain(a)
assert "a" in graph
assert graph.all_specs()["a"].depends_on == ()
def test_chain_preserves_existing_deps() -> None:
"""chain 应保留 spec 已有的 depends_on."""
a = TaskSpec("a", _fn)
b = TaskSpec("b", _fn)
c = TaskSpec("c", _fn, depends_on=("b",))
graph = px.Graph().chain(a, b, c)
# c 已有 depends_on=('b',),前驱是 b,已在依赖中,不重复添加
assert graph.all_specs()["c"].depends_on == ("b",)
def test_chain_merges_existing_deps() -> None:
"""chain 应将前驱追加到已有依赖前(若不存在)."""
a = TaskSpec("a", _fn)
x = TaskSpec("x", _fn)
c = TaskSpec("c", _fn, depends_on=("x",))
graph = px.Graph().chain(a, x, c)
# c 前驱是 x,但 c 已依赖 x,不重复
assert graph.all_specs()["c"].depends_on == ("x",)
def test_chain_returns_self() -> None:
"""chain 返回 self 支持链式调用."""
a = TaskSpec("a", _fn)
graph = px.Graph()
assert graph.chain(a) is graph
def test_chain_execution_order() -> None:
"""chain 应保证执行顺序."""
order: list[str] = []
def make(name: str):
def fn() -> str:
order.append(name)
return name
return fn
a = TaskSpec("a", make("a"))
b = TaskSpec("b", make("b"))
c = TaskSpec("c", make("c"))
graph = px.Graph().chain(a, b, c)
report = px.run(graph)
assert report.success
assert order == ["a", "b", "c"]
def test_chain_with_decorator_specs() -> None:
"""chain 应与 @task 装饰器配合."""
@px.task
def extract() -> int:
return 1
@px.task
def transform(extract: int) -> int:
return extract + 10
@px.task
def load(transform: int) -> int:
return transform + 100
graph = px.Graph().chain(extract, transform, load)
report = px.run(graph)
assert report.success
assert report["load"] == 111
+19 -19
View File
@@ -17,7 +17,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"build": px.Graph.from_specs([build_task]),
"test": px.Graph.from_specs([test_task]),
"all": px.Graph.from_specs([build_task, "test"]),
@@ -38,7 +38,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"cmd1": px.Graph.from_specs([task1]),
"cmd2": px.Graph.from_specs([task2]),
"cmd3": px.Graph.from_specs([task3]),
@@ -57,7 +57,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"lint": px.Graph.from_specs([lint_task, format_task]),
"quick": px.Graph.from_specs(["lint.lint"]),
},
@@ -75,7 +75,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"cmd1": px.Graph.from_specs([task1]),
"cmd2": px.Graph.from_specs(["cmd1", task2]),
"cmd3": px.Graph.from_specs(["cmd2", task3]),
@@ -93,7 +93,7 @@ class TestCommandReferences:
with pytest.raises(ValueError, match="循环引用"):
px.CliRunner(
strategy="sequential",
graphs={
aliases={
"cmd1": px.Graph.from_specs(["cmd1", task1]),
},
)
@@ -105,7 +105,7 @@ class TestCommandReferences:
with pytest.raises(ValueError, match="引用的命令 'invalid' 不存在"):
px.CliRunner(
strategy="sequential",
graphs={
aliases={
"cmd1": px.Graph.from_specs(["invalid", task1]),
},
)
@@ -117,7 +117,7 @@ class TestCommandReferences:
with pytest.raises(ValueError, match="任务 'invalid' 不存在于命令 'cmd1'"):
px.CliRunner(
strategy="sequential",
graphs={
aliases={
"cmd1": px.Graph.from_specs([task1]),
"cmd2": px.Graph.from_specs(["cmd1.invalid"]),
},
@@ -130,7 +130,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"cmd1": px.Graph.from_specs([task1, task2]),
"cmd2": px.Graph.from_specs(["cmd1"]),
},
@@ -148,7 +148,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"cmd1": px.Graph.from_specs([task1, task2]),
"cmd2": px.Graph.from_specs(["cmd1", task3]),
},
@@ -168,7 +168,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"cmd1": px.Graph.from_specs([task1]),
"cmd2": px.Graph.from_specs([task2, task3]),
"cmd3": px.Graph.from_specs([task4]),
@@ -205,7 +205,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"cmd1": px.Graph.from_specs([task1]),
"cmd2": px.Graph.from_specs([task2]),
"all": px.Graph.from_specs(["cmd1", "cmd2", task3, task4, task5]),
@@ -242,7 +242,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"cmd1": px.Graph.from_specs([task1, task2]),
"cmd2": px.Graph.from_specs([task3]),
"all": px.Graph.from_specs(["cmd1", "cmd2", task4]),
@@ -279,7 +279,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"c": px.Graph.from_specs([git_clean]),
"tc": px.Graph.from_specs([typecheck, "lint"]),
"lint": px.Graph.from_specs([lint, format_task]),
@@ -319,7 +319,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"cmd1": px.Graph.from_specs([task1]),
"cmd2": px.Graph.from_specs([task2]),
"cmd3": px.Graph.from_specs([task3]),
@@ -350,7 +350,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"all": px.Graph.from_specs([task1, task2, task3]),
},
)
@@ -373,7 +373,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"cmd1": px.Graph.from_specs([task1, task2]),
"all": px.Graph.from_specs(["cmd1"]),
},
@@ -399,7 +399,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"cmd1": px.Graph.from_specs([task1]),
"cmd2": px.Graph.from_specs(["cmd1", task2]),
"cmd3": px.Graph.from_specs(["cmd2", task3]),
@@ -430,7 +430,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"cmd1": px.Graph.from_specs([task1, task2]), # Parallel tasks
"cmd2": px.Graph.from_specs([task3, task4]), # Parallel tasks
"all": px.Graph.from_specs(["cmd1", "cmd2"]),
@@ -465,7 +465,7 @@ class TestCommandReferences:
runner = px.CliRunner(
strategy="sequential",
graphs={
aliases={
"clean": px.Graph.from_specs([clean]),
"build": px.Graph.from_specs([build1, build2]),
"test": px.Graph.from_specs([test1, test2]),
+62
View File
@@ -0,0 +1,62 @@
"""Tests for process executor (spec.executor='process')."""
from __future__ import annotations
import pytest
# pyrefly: ignore[missing-import]
from _proc_helper import add, cpu_heavy, slow_sleep, sub
import pyflowx as px
from pyflowx.errors import TaskFailedError
def test_process_executor_runs_cpu_task() -> None:
"""executor='process' 应在进程池中执行 CPU 密集型任务."""
spec = px.TaskSpec("cpu", fn=cpu_heavy, args=(1000,), executor="process")
graph = px.Graph.from_specs([spec])
report = px.run(graph)
assert report.success
assert report["cpu"] == sum(i * i for i in range(1000))
def test_process_executor_with_dependency() -> None:
"""进程池任务应支持依赖注入."""
spec1 = px.TaskSpec("a", fn=cpu_heavy, args=(100,), executor="process")
spec2 = px.TaskSpec("b", fn=add, args=(3, 4), executor="process", depends_on=("a",))
graph = px.Graph.from_specs([spec1, spec2])
report = px.run(graph)
assert report.success
assert report["b"] == 7
def test_process_executor_default_is_thread() -> None:
"""TaskSpec.executor 默认应为 'thread'."""
spec = px.TaskSpec("x", fn=lambda: None)
assert spec.executor == "thread"
def test_inline_executor_runs_in_event_loop() -> None:
"""executor='inline' 应直接在事件循环线程调用."""
spec = px.TaskSpec("inline", fn=add, args=(10, 20), executor="inline")
graph = px.Graph.from_specs([spec])
report = px.run(graph)
assert report.success
assert report["inline"] == 30
def test_process_executor_with_kwargs() -> None:
"""进程池任务应支持 kwargs 注入."""
spec = px.TaskSpec("kw", fn=sub, args=(10,), kwargs={"b": 3}, executor="process")
graph = px.Graph.from_specs([spec])
report = px.run(graph)
assert report.success
assert report["kw"] == 7
def test_process_executor_timeout() -> None:
"""进程池任务超时应抛 TaskFailedError."""
spec = px.TaskSpec("slow", fn=slow_sleep, args=(10.0,), executor="process", timeout=0.1)
graph = px.Graph.from_specs([spec])
with pytest.raises(TaskFailedError):
px.run(graph)
+152
View File
@@ -0,0 +1,152 @@
"""Tests for Graph namespace and add_subgraph."""
from __future__ import annotations
import pytest
import pyflowx as px
def _fn() -> None:
return None
def test_graph_namespace_field_default_none() -> None:
"""Graph 默认 namespace 为 None."""
graph = px.Graph()
assert graph.namespace is None
def test_graph_from_specs_with_namespace() -> None:
"""from_specs(namespace=...) 应设置 graph.namespace."""
graph = px.Graph.from_specs([px.TaskSpec("a", _fn)], namespace="ns1")
assert graph.namespace == "ns1"
def test_add_subgraph_prefixes_task_names() -> None:
"""add_subgraph 应给子图任务名加命名空间前缀."""
sub = px.Graph.from_specs(
[px.TaskSpec("extract", _fn), px.TaskSpec("build", _fn, depends_on=("extract",))],
namespace="build",
)
main = px.Graph.from_specs([px.TaskSpec("start", _fn)])
main.add_subgraph(sub)
assert "start" in main
assert "build:extract" in main
assert "build:build" in main
def test_add_subgraph_renames_internal_deps() -> None:
"""add_subgraph 应给子图内部依赖名加前缀."""
sub = px.Graph.from_specs(
[px.TaskSpec("a", _fn), px.TaskSpec("b", _fn, depends_on=("a",))],
namespace="ns",
)
main = px.Graph()
main.add_subgraph(sub)
b_spec = main.all_specs()["ns:b"]
assert b_spec.depends_on == ("ns:a",)
def test_add_subgraph_all_internal_deps_prefixed() -> None:
"""add_subgraph 子图内所有任务(含被依赖的)都加前缀."""
sub = px.Graph.from_specs(
[px.TaskSpec("ext", _fn), px.TaskSpec("b", _fn, depends_on=("ext",))],
namespace="ns",
)
main = px.Graph()
main.add_subgraph(sub)
b_spec = main.all_specs()["ns:b"]
assert b_spec.depends_on == ("ns:ext",)
assert "ns:ext" in main
def test_add_subgraph_requires_namespace() -> None:
"""add_subgraph 无 namespace 时应抛 ValueError."""
sub = px.Graph.from_specs([px.TaskSpec("a", _fn)]) # 无 namespace
main = px.Graph()
with pytest.raises(ValueError, match="namespace"):
main.add_subgraph(sub)
def test_add_subgraph_explicit_namespace_overrides() -> None:
"""add_subgraph(namespace=...) 应覆盖子图自带 namespace."""
sub = px.Graph.from_specs([px.TaskSpec("a", _fn)], namespace="original")
main = px.Graph()
main.add_subgraph(sub, namespace="override")
assert "override:a" in main
assert "original:a" not in main
def test_add_subgraph_internal_injection_works() -> None:
"""子图内部依赖注入应通过 wrapper 正常工作."""
sub = px.Graph.from_specs(
[
px.TaskSpec("extract", lambda: [1, 2, 3]),
px.TaskSpec("build", lambda extract: [x * 2 for x in extract], depends_on=("extract",)),
],
namespace="build",
)
main = px.Graph()
main.add_subgraph(sub)
report = px.run(main)
assert report.success
assert report["build:build"] == [2, 4, 6]
def test_add_subgraph_cross_namespace_ref_via_context() -> None:
"""跨命名空间引用应通过 Context 标注接收."""
def consumer(ctx: px.Context) -> str:
return f"got {ctx['ns:data']}"
sub = px.Graph.from_specs(
[px.TaskSpec("data", lambda: "data_value")],
namespace="ns",
)
main = px.Graph()
main.add_subgraph(sub)
main.add(px.TaskSpec("consumer", consumer, depends_on=("ns:data",)))
report = px.run(main)
assert report.success
assert report["consumer"] == "got data_value"
def test_add_subgraph_context_annotation_in_subgraph() -> None:
"""子图内部任务用 Context 标注时,wrapper 应正确传递."""
def sink(ctx: px.Context) -> int:
return ctx["src"]
sub = px.Graph.from_specs(
[
px.TaskSpec("src", lambda: 42),
px.TaskSpec("sink", sink, depends_on=("src",)),
],
namespace="ns",
)
main = px.Graph()
main.add_subgraph(sub)
report = px.run(main)
assert report.success
assert report["ns:sink"] == 42
def test_add_subgraph_chained() -> None:
"""多个子图可链式合并到主图."""
sub_a = px.Graph.from_specs([px.TaskSpec("a", _fn)], namespace="nsA")
sub_b = px.Graph.from_specs([px.TaskSpec("b", _fn)], namespace="nsB")
main = px.Graph()
main.add_subgraph(sub_a).add_subgraph(sub_b)
assert "nsA:a" in main
assert "nsB:b" in main
+574
View File
@@ -0,0 +1,574 @@
"""性能剖面(ProfileReport)测试.
覆盖策略
* 构造带时间戳的 RunReport + Graph验证关键路径并行度瓶颈排序
* 边界场景空报告单任务无时间戳SKIPPED 任务图校验失败
* 输出格式to_dict / describe / top_bottlenecks / critical_tasks
"""
from __future__ import annotations
from datetime import datetime, timedelta
from typing import Any
import pyflowx as px
from pyflowx.profiling import ProfileReport, TaskProfile
from pyflowx.task import TaskResult, TaskSpec, TaskStatus
def _fn() -> int:
return 1
def _spec(name: str, deps: tuple[str, ...] = ()) -> TaskSpec[Any]:
return TaskSpec[Any](name, _fn, depends_on=deps)
def _result(
name: str,
start: datetime,
duration: float,
*,
status: TaskStatus = TaskStatus.SUCCESS,
attempts: int = 1,
) -> TaskResult[Any]:
"""构造带时间戳的 TaskResult."""
end = start + timedelta(seconds=duration) if duration > 0 else start
return TaskResult[Any](
spec=_spec(name),
status=status,
value=None,
attempts=attempts,
started_at=start if duration > 0 or status != TaskStatus.SKIPPED else None,
finished_at=end if duration > 0 or status != TaskStatus.SKIPPED else None,
)
def _skipped_result(name: str, reason: str = "skip") -> TaskResult[Any]:
"""构造 SKIPPED 结果(无时间戳)."""
return TaskResult[Any](
spec=_spec(name),
status=TaskStatus.SKIPPED,
reason=reason,
)
class TestProfileReportConstruction:
"""测试 ProfileReport 构建."""
def test_empty_report(self) -> None:
"""空报告应产生空剖面."""
report = px.RunReport()
graph = px.Graph()
profile = ProfileReport.from_report(report, graph)
assert len(profile.tasks) == 0
assert profile.total_duration == 0.0
assert profile.critical_path == ()
assert profile.critical_path_duration == 0.0
assert profile.avg_parallelism == 0.0
assert profile.peak_parallelism == 0
def test_single_task(self) -> None:
"""单任务:关键路径就是它自己,并行度为 1."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.5)
graph = px.Graph.from_specs([_spec("a")])
profile = ProfileReport.from_report(report, graph)
assert len(profile.tasks) == 1
assert profile.tasks[0].name == "a"
assert profile.tasks[0].duration == 1.5
assert profile.tasks[0].is_on_critical_path
assert profile.total_duration == 1.5
assert profile.critical_path == ("a",)
assert profile.critical_path_duration == 1.5
assert profile.avg_parallelism == 1.0
assert profile.peak_parallelism == 1
assert profile.parallelism_efficiency == 1.0
def test_serial_chain(self) -> None:
"""串行链 a -> b -> c:关键路径为全部,效率 100%."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
report.results["b"] = _result("b", start + timedelta(seconds=1), 2.0)
report.results["c"] = _result("c", start + timedelta(seconds=3), 1.5)
graph = px.Graph.from_specs([
_spec("a"),
_spec("b", deps=("a",)),
_spec("c", deps=("b",)),
])
profile = ProfileReport.from_report(report, graph)
assert profile.total_duration == 4.5
assert profile.critical_path_duration == 4.5
assert profile.critical_path == ("a", "b", "c")
assert profile.parallelism_efficiency == 1.0
assert profile.peak_parallelism == 1
assert profile.avg_parallelism == 1.0
def test_parallel_tasks(self) -> None:
"""并行任务 a, b 同时执行:关键路径取较长者,效率 < 1."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
report.results["b"] = _result("b", start, 2.0)
graph = px.Graph.from_specs([_spec("a"), _spec("b")])
profile = ProfileReport.from_report(report, graph)
# wall-clock = 2.0, 关键路径 = 2.0 (b), 效率 = 1.0
# 因为关键路径定义就是最长路径,与 wall-clock 相同
assert profile.total_duration == 2.0
assert profile.critical_path_duration == 2.0
assert profile.critical_path == ("b",)
assert profile.peak_parallelism == 2
# 平均并行度 = (1.0 + 2.0) / 2.0 = 1.5
assert profile.avg_parallelism == 1.5
def test_parallel_with_join(self) -> None:
"""a, b 并行后 join 到 c:关键路径 a->c 或 b->c."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
report.results["b"] = _result("b", start, 3.0)
report.results["c"] = _result("c", start + timedelta(seconds=3), 1.0)
graph = px.Graph.from_specs([
_spec("a"),
_spec("b"),
_spec("c", deps=("a", "b")),
])
profile = ProfileReport.from_report(report, graph)
# 关键路径 = b -> c (3 + 1 = 4)
assert profile.critical_path_duration == 4.0
assert profile.critical_path == ("b", "c")
assert profile.tasks[0].is_on_critical_path is False # a 不在关键路径
# task("b") 在关键路径上
assert profile.task("b").is_on_critical_path
assert profile.task("c").is_on_critical_path
def test_skipped_task_no_timestamp(self) -> None:
"""SKIPPED 任务无时间戳:不影响并行度计算."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
report.results["b"] = _skipped_result("b")
graph = px.Graph.from_specs([_spec("a"), _spec("b")])
profile = ProfileReport.from_report(report, graph)
# b 是 SKIPPEDduration=0
assert profile.task("b").status == TaskStatus.SKIPPED
assert profile.task("b").duration == 0.0
assert profile.peak_parallelism == 1 # 只有 a 在跑
class TestWaitTime:
"""测试等待时间计算."""
def test_no_deps_zero_wait(self) -> None:
"""无依赖任务等待时间为 0."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
graph = px.Graph.from_specs([_spec("a")])
profile = ProfileReport.from_report(report, graph)
assert profile.task("a").wait_time == 0.0
def test_wait_after_dep_completes(self) -> None:
"""b 在 a 完成后等待 0.5s 才开始."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
report.results["b"] = _result("b", start + timedelta(seconds=1.5), 1.0)
graph = px.Graph.from_specs([
_spec("a"),
_spec("b", deps=("a",)),
])
profile = ProfileReport.from_report(report, graph)
assert profile.task("b").wait_time == 0.5
def test_wait_negative_clamped_to_zero(self) -> None:
"""b 在 a 完成前就开始(异常情况)应钳制为 0."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 2.0)
# b 在 a 还没完成时就开始(不应该但可能发生)
report.results["b"] = _result("b", start + timedelta(seconds=1), 1.0)
graph = px.Graph.from_specs([
_spec("a"),
_spec("b", deps=("a",)),
])
profile = ProfileReport.from_report(report, graph)
# a 在 t=2 结束,b 在 t=1 开始,delta = -1,钳制为 0
assert profile.task("b").wait_time == 0.0
def test_skipped_task_zero_wait(self) -> None:
"""SKIPPED 任务等待时间为 0."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
report.results["b"] = _skipped_result("b")
graph = px.Graph.from_specs([
_spec("a"),
_spec("b", deps=("a",)),
])
profile = ProfileReport.from_report(report, graph)
assert profile.task("b").wait_time == 0.0
class TestCriticalPath:
"""测试关键路径分析."""
def test_diamond_dependency(self) -> None:
"""菱形依赖:a -> b -> d, a -> c -> d,关键路径取较长分支."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
report.results["b"] = _result("b", start + timedelta(seconds=1), 3.0)
report.results["c"] = _result("c", start + timedelta(seconds=1), 1.0)
report.results["d"] = _result("d", start + timedelta(seconds=4), 1.0)
graph = px.Graph.from_specs([
_spec("a"),
_spec("b", deps=("a",)),
_spec("c", deps=("a",)),
_spec("d", deps=("b", "c")),
])
profile = ProfileReport.from_report(report, graph)
# 关键路径:a -> b -> d = 1 + 3 + 1 = 5
assert profile.critical_path_duration == 5.0
assert profile.critical_path == ("a", "b", "d")
def test_graph_validation_failure_returns_empty(self) -> None:
"""图校验失败(有环)应回退为空关键路径."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
# 手动构造带环的图(绕过校验)
graph = px.Graph()
graph.specs["a"] = _spec("a", deps=("b",))
graph.specs["b"] = _spec("b", deps=("a",))
graph.deps["a"] = ("b",)
graph.deps["b"] = ("a",)
profile = ProfileReport.from_report(report, graph)
# layers() 抛 CycleError,回退为空
assert profile.critical_path == ()
assert profile.critical_path_duration == 0.0
class TestParallelism:
"""测试并行度计算."""
def test_no_timestamps_zero_parallelism(self) -> None:
"""所有任务无时间戳:并行度为 0."""
report = px.RunReport()
report.results["a"] = TaskResult[Any](spec=_spec("a"), status=TaskStatus.SUCCESS)
graph = px.Graph.from_specs([_spec("a")])
profile = ProfileReport.from_report(report, graph)
assert profile.avg_parallelism == 0.0
assert profile.peak_parallelism == 0
def test_zero_duration_excluded(self) -> None:
"""零耗时任务(end <= start)不参与并行度计算."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 0.0) # 零耗时
report.results["b"] = _result("b", start, 1.0)
graph = px.Graph.from_specs([_spec("a"), _spec("b")])
profile = ProfileReport.from_report(report, graph)
# 只有 b 参与,峰值 = 1
assert profile.peak_parallelism == 1
def test_skipped_with_timestamps_excluded(self) -> None:
"""SKIPPED 任务即使带时间戳也不参与并行度计算."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
# SKIPPED 但带时间戳(异常但可能发生)
report.results["a"] = _result("a", start, 1.0, status=TaskStatus.SKIPPED)
report.results["b"] = _result("b", start, 1.0)
graph = px.Graph.from_specs([_spec("a"), _spec("b")])
profile = ProfileReport.from_report(report, graph)
# a 是 SKIPPED,被排除;只有 b 参与
assert profile.peak_parallelism == 1
def test_peak_parallelism_three_tasks(self) -> None:
"""三个任务完全重叠:峰值并行度 = 3."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 3.0)
report.results["b"] = _result("b", start, 3.0)
report.results["c"] = _result("c", start, 3.0)
graph = px.Graph.from_specs([_spec("a"), _spec("b"), _spec("c")])
profile = ProfileReport.from_report(report, graph)
assert profile.peak_parallelism == 3
assert profile.avg_parallelism == 3.0
class TestQueries:
"""测试查询方法."""
def test_task_lookup(self) -> None:
"""task(name) 应返回对应剖面."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
report.results["b"] = _result("b", start, 2.0)
graph = px.Graph.from_specs([_spec("a"), _spec("b")])
profile = ProfileReport.from_report(report, graph)
assert profile.task("a").name == "a"
assert profile.task("b").duration == 2.0
def test_task_lookup_not_found(self) -> None:
"""task(name) 不存在应抛 KeyError."""
report = px.RunReport()
graph = px.Graph()
profile = ProfileReport.from_report(report, graph)
try:
profile.task("missing")
except KeyError:
pass
else:
raise AssertionError("应抛出 KeyError")
def test_top_bottlenecks(self) -> None:
"""top_bottlenecks 应按耗时降序返回."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
report.results["b"] = _result("b", start, 3.0)
report.results["c"] = _result("c", start, 2.0)
graph = px.Graph.from_specs([_spec("a"), _spec("b"), _spec("c")])
profile = ProfileReport.from_report(report, graph)
top3 = profile.top_bottlenecks(3)
assert len(top3) == 3
assert top3[0].name == "b"
assert top3[1].name == "c"
assert top3[2].name == "a"
def test_top_bottlenecks_zero_or_negative(self) -> None:
"""n <= 0 应返回空元组."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
graph = px.Graph.from_specs([_spec("a")])
profile = ProfileReport.from_report(report, graph)
assert profile.top_bottlenecks(0) == ()
assert profile.top_bottlenecks(-1) == ()
def test_critical_tasks(self) -> None:
"""critical_tasks 应返回关键路径上的任务(按路径顺序)."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
report.results["b"] = _result("b", start + timedelta(seconds=1), 3.0)
report.results["c"] = _result("c", start + timedelta(seconds=1), 1.0)
report.results["d"] = _result("d", start + timedelta(seconds=4), 1.0)
graph = px.Graph.from_specs([
_spec("a"),
_spec("b", deps=("a",)),
_spec("c", deps=("a",)),
_spec("d", deps=("b", "c")),
])
profile = ProfileReport.from_report(report, graph)
# 关键路径 a -> b -> d
critical = profile.critical_tasks()
assert len(critical) == 3
assert [t.name for t in critical] == ["a", "b", "d"]
def test_failed_tasks(self) -> None:
"""failed_tasks 应返回 FAILED 状态的任务."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0, status=TaskStatus.FAILED)
report.results["b"] = _result("b", start, 1.0)
graph = px.Graph.from_specs([_spec("a"), _spec("b")])
profile = ProfileReport.from_report(report, graph)
failed = profile.failed_tasks()
assert len(failed) == 1
assert failed[0].name == "a"
def test_skipped_tasks(self) -> None:
"""skipped_tasks 应返回 SKIPPED 状态的任务."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
report.results["b"] = _skipped_result("b")
graph = px.Graph.from_specs([_spec("a"), _spec("b")])
profile = ProfileReport.from_report(report, graph)
skipped = profile.skipped_tasks()
assert len(skipped) == 1
assert skipped[0].name == "b"
class TestOutputFormats:
"""测试输出格式."""
def test_to_dict_structure(self) -> None:
"""to_dict 应返回包含所有字段的字典."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.5)
graph = px.Graph.from_specs([_spec("a")])
profile = ProfileReport.from_report(report, graph)
d = profile.to_dict()
assert "tasks" in d
assert "total_duration_seconds" in d
assert "critical_path_duration_seconds" in d
assert "critical_path" in d
assert "avg_parallelism" in d
assert "peak_parallelism" in d
assert "parallelism_efficiency" in d
assert "bottlenecks" in d
assert len(d["tasks"]) == 1
assert d["tasks"][0]["name"] == "a"
assert d["tasks"][0]["status"] == "success"
assert d["tasks"][0]["duration_seconds"] == 1.5
assert d["tasks"][0]["is_on_critical_path"] is True
def test_describe_contains_key_sections(self) -> None:
"""describe 应包含关键章节标题."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
graph = px.Graph.from_specs([_spec("a")])
profile = ProfileReport.from_report(report, graph)
text = profile.describe()
assert "PyFlowX 性能剖面报告" in text
assert "【图级指标】" in text
assert "【关键路径】" in text
assert "【Top" in text
assert "【全部任务】" in text
assert "a" in text
def test_describe_empty_report(self) -> None:
"""空报告的 describe 应不崩溃且包含章节标题."""
report = px.RunReport()
graph = px.Graph()
profile = ProfileReport.from_report(report, graph)
text = profile.describe()
assert "【图级指标】" in text
assert "(无)" in text
def test_repr(self) -> None:
"""__repr__ 应包含关键指标."""
start = datetime(2024, 1, 1, 0, 0, 0)
report = px.RunReport()
report.results["a"] = _result("a", start, 1.0)
graph = px.Graph.from_specs([_spec("a")])
profile = ProfileReport.from_report(report, graph)
r = repr(profile)
assert "ProfileReport" in r
assert "tasks=1" in r
assert "total=1.000s" in r
def test_task_profile_to_dict(self) -> None:
"""TaskProfile.to_dict 应返回正确字段."""
tp = TaskProfile(
name="x",
status=TaskStatus.SUCCESS,
duration=1.5,
attempts=2,
wait_time=0.3,
is_on_critical_path=True,
deps=("a", "b"),
)
d = tp.to_dict()
assert d["name"] == "x"
assert d["status"] == "success"
assert d["duration_seconds"] == 1.5
assert d["attempts"] == 2
assert d["wait_time_seconds"] == 0.3
assert d["is_on_critical_path"] is True
assert d["deps"] == ["a", "b"]
class TestIntegrationWithRun:
"""与真实 run() 集成测试."""
def test_profile_from_real_run(self) -> None:
"""从真实 run() 结果构建剖面."""
import time
def slow() -> int:
time.sleep(0.01) # 确保任务有实际耗时,避免 duration 极小导致并行度计算为 0
return 1
graph = px.Graph.from_specs([
px.TaskSpec("a", slow),
px.TaskSpec("b", slow, depends_on=("a",)),
px.TaskSpec("c", slow, depends_on=("a",)),
])
report = px.run(graph, strategy="sequential")
profile = ProfileReport.from_report(report, graph)
assert len(profile.tasks) == 3
# sequential 策略下应为串行,duration > 0
assert profile.critical_path_duration > 0
# sequential 策略下并行度应为 1
assert profile.peak_parallelism == 1
def test_profile_from_thread_run(self) -> None:
"""从 thread 策略 run() 结果构建剖面,验证并行度 > 1."""
import time
def slow() -> int:
time.sleep(0.05)
return 1
graph = px.Graph.from_specs([
px.TaskSpec("a", slow),
px.TaskSpec("b", slow),
px.TaskSpec("c", slow),
])
report = px.run(graph, strategy="thread", max_workers=3)
profile = ProfileReport.from_report(report, graph)
# 三个任务并行,峰值应 >= 2(可能因调度时机不到 3)
assert profile.peak_parallelism >= 2
assert profile.critical_path_duration > 0
+47
View File
@@ -126,3 +126,50 @@ class TestRunReportDescribe:
report.results["a"] = TaskResult[Any](spec=spec, status=TaskStatus.PENDING)
desc = report.describe()
assert "-" in desc # duration 显示为 "-"
class TestRunReportQueries:
"""测试 RunReport 的新查询 API."""
def test_succeeded_tasks(self) -> None:
"""succeeded_tasks 返回 SUCCESS 状态的任务名."""
report = px.RunReport()
report.results["a"] = _make_result("a", status=TaskStatus.SUCCESS)
report.results["b"] = _make_result("b", status=TaskStatus.FAILED)
report.results["c"] = _make_result("c", status=TaskStatus.SUCCESS)
assert report.succeeded_tasks() == ["a", "c"]
def test_skipped_tasks(self) -> None:
"""skipped_tasks 返回 SKIPPED 状态的任务名."""
report = px.RunReport()
report.results["a"] = _make_result("a", status=TaskStatus.SKIPPED)
report.results["b"] = _make_result("b", status=TaskStatus.SUCCESS)
assert report.skipped_tasks() == ["a"]
def test_tasks_by_status(self) -> None:
"""tasks_by_status 按指定状态过滤."""
report = px.RunReport()
report.results["a"] = _make_result("a", status=TaskStatus.FAILED)
report.results["b"] = _make_result("b", status=TaskStatus.FAILED)
report.results["c"] = _make_result("c", status=TaskStatus.SUCCESS)
assert report.tasks_by_status(TaskStatus.FAILED) == ["a", "b"]
assert report.tasks_by_status(TaskStatus.SUCCESS) == ["c"]
assert report.tasks_by_status(TaskStatus.SKIPPED) == []
def test_durations(self) -> None:
"""durations 返回任务名 -> 时长映射."""
report = px.RunReport()
report.results["a"] = _make_result("a", duration=1.5)
report.results["b"] = _make_result("b", duration=2.0)
durs = report.durations()
assert durs["a"] == 1.5
assert durs["b"] == 2.0
def test_durations_no_duration(self) -> None:
"""无时长的任务应返回 0.0."""
report = px.RunReport()
spec: TaskSpec[Any] = TaskSpec[Any]("a", _fn) # type: ignore[arg-type]
report.results["a"] = TaskResult[Any](spec=spec, status=TaskStatus.PENDING)
durs = report.durations()
assert durs["a"] == 0.0
+177 -65
View File
@@ -53,18 +53,18 @@ class TestCliRunnerConstruction:
def test_requires_at_least_one_command(self) -> None:
"""没有命令时应抛出 ValueError."""
with pytest.raises(ValueError, match="至少需要一个命令"):
with pytest.raises(ValueError, match="至少需要一个别名"):
_ = px.CliRunner()
def test_accepts_single_graph(self) -> None:
"""单个命令应正常构造."""
runner = px.CliRunner(graphs={"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
assert runner.commands == ["clean"]
def test_accepts_multiple_graphs(self) -> None:
"""多个命令应按插入顺序保留."""
runner = px.CliRunner(
graphs={
aliases={
"clean": _echo_graph("c", "clean"),
"build": _echo_graph("b", "build"),
"test": _echo_graph("t", "test"),
@@ -72,39 +72,39 @@ class TestCliRunnerConstruction:
)
assert runner.commands == ["clean", "build", "test"]
def test_default_strategy_is_sequential(self) -> None:
"""默认策略应为 Strategy.SEQUENTIAL."""
runner = px.CliRunner({"clean": _echo_graph()})
assert runner.strategy == "sequential"
def test_default_strategy_is_dependency(self) -> None:
"""默认策略应为 dependency(依赖驱动,最大并行度)."""
runner = px.CliRunner(aliases={"clean": _echo_graph()})
assert runner.strategy == "dependency"
def test_custom_strategy_string(self) -> None:
"""应支持通过字符串指定策略."""
runner = px.CliRunner({"clean": _echo_graph()}, strategy="thread")
runner = px.CliRunner(aliases={"clean": _echo_graph()}, strategy="thread")
assert runner.strategy == "thread"
def test_custom_strategy_enum(self) -> None:
"""应支持通过 Strategy 枚举指定策略."""
runner = px.CliRunner({"clean": _echo_graph()}, strategy="async")
runner = px.CliRunner(aliases={"clean": _echo_graph()}, strategy="async")
assert runner.strategy == "async"
def test_default_verbose_is_true(self) -> None:
"""默认 verbose 应为 True."""
runner = px.CliRunner({"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
assert runner.verbose is True
def test_custom_verbose_false(self) -> None:
"""应支持关闭 verbose."""
runner = px.CliRunner({"clean": _echo_graph()}, verbose=False)
runner = px.CliRunner(aliases={"clean": _echo_graph()}, verbose=False)
assert runner.verbose is False
def test_default_description_is_empty(self) -> None:
"""默认描述应为空字符串."""
runner = px.CliRunner({"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
assert runner.description == ""
def test_custom_description(self) -> None:
"""应支持自定义描述."""
runner = px.CliRunner({"clean": _echo_graph()}, description="My CLI")
runner = px.CliRunner(aliases={"clean": _echo_graph()}, description="My CLI")
assert runner.description == "My CLI"
@@ -116,13 +116,13 @@ class TestCliRunnerProperties:
def test_commands_returns_list(self) -> None:
"""commands 应返回列表."""
runner = px.CliRunner({"a": _echo_graph(), "b": _echo_graph()})
runner = px.CliRunner(aliases={"a": _echo_graph(), "b": _echo_graph()})
assert isinstance(runner.commands, list)
def test_graphs_contains_original_graphs(self) -> None:
"""graphs 应包含原始 Graph 实例."""
g = _echo_graph()
runner = px.CliRunner({"cmd": g})
runner = px.CliRunner(aliases={"cmd": g})
assert runner.graphs["cmd"] is g
@@ -136,69 +136,69 @@ class TestCliRunnerParser:
"""create_parser 应返回 ArgumentParser."""
from argparse import ArgumentParser
runner = px.CliRunner({"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
parser = runner.create_parser()
assert isinstance(parser, ArgumentParser)
def test_parser_has_command_argument(self) -> None:
"""解析器应有 command 位置参数."""
runner = px.CliRunner({"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
parser = runner.create_parser()
parsed = parser.parse_args(["clean"])
assert parsed.command == "clean"
def test_parser_command_is_optional(self) -> None:
"""command 应为可选参数."""
runner = px.CliRunner({"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
parser = runner.create_parser()
parsed = parser.parse_args([])
assert parsed.command is None
def test_parser_has_strategy_option(self) -> None:
"""解析器应有 --strategy 选项."""
runner = px.CliRunner({"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
parser = runner.create_parser()
parsed = parser.parse_args(["clean", "--strategy", "thread"])
assert parsed.strategy == "thread"
def test_parser_strategy_default(self) -> None:
"""--strategy 默认值应与构造时一致."""
runner = px.CliRunner({"clean": _echo_graph()}, strategy="async")
runner = px.CliRunner(aliases={"clean": _echo_graph()}, strategy="async")
parser = runner.create_parser()
parsed = parser.parse_args(["clean"])
assert parsed.strategy == "async"
def test_parser_has_dry_run_flag(self) -> None:
"""解析器应有 --dry-run 标志."""
runner = px.CliRunner({"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
parser = runner.create_parser()
parsed = parser.parse_args(["clean", "--dry-run"])
assert parsed.dry_run is True
def test_parser_dry_run_default_false(self) -> None:
"""--dry-run 默认为 False."""
runner = px.CliRunner({"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
parser = runner.create_parser()
parsed = parser.parse_args(["clean"])
assert parsed.dry_run is False
def test_parser_has_list_flag(self) -> None:
"""解析器应有 --list 标志."""
runner = px.CliRunner({"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
parser = runner.create_parser()
parsed = parser.parse_args(["--list"])
assert parsed.list is True
def test_parser_has_quiet_flag(self) -> None:
"""解析器应有 --quiet 标志."""
runner = px.CliRunner({"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
parser = runner.create_parser()
parsed = parser.parse_args(["clean", "--quiet"])
assert parsed.quiet is True
def test_parser_quiet_default_false(self) -> None:
"""--quiet 默认为 False."""
runner = px.CliRunner({"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
parser = runner.create_parser()
parsed = parser.parse_args(["clean"])
assert parsed.quiet is False
@@ -222,7 +222,7 @@ class TestCliRunnerRunSuccess:
def test_run_valid_command_returns_zero(self) -> None:
"""有效命令执行成功应返回 0."""
runner = px.CliRunner({"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
exit_code = runner.run(["clean"])
assert exit_code == CliExitCode.SUCCESS.value
@@ -236,28 +236,30 @@ class TestCliRunnerRunSuccess:
def track_b() -> None:
executed.append("b")
runner = px.CliRunner({
"a": px.Graph.from_specs([px.TaskSpec("a", track_a)]),
"b": px.Graph.from_specs([px.TaskSpec("b", track_b)]),
})
runner = px.CliRunner(
aliases={
"a": px.Graph.from_specs([px.TaskSpec("a", track_a)]),
"b": px.Graph.from_specs([px.TaskSpec("b", track_b)]),
}
)
_ = runner.run(["b"])
assert executed == ["b"]
def test_run_multi_task_graph(self) -> None:
"""应能执行带依赖的多任务图."""
runner = px.CliRunner({"multi": _multi_task_graph()})
runner = px.CliRunner(aliases={"multi": _multi_task_graph()})
exit_code = runner.run(["multi"])
assert exit_code == CliExitCode.SUCCESS.value
def test_run_with_strategy_override(self) -> None:
"""应支持通过 --strategy 覆盖默认策略."""
runner = px.CliRunner({"echo": _echo_graph()})
runner = px.CliRunner(aliases={"echo": _echo_graph()})
exit_code = runner.run(["echo", "--strategy", "thread"])
assert exit_code == CliExitCode.SUCCESS.value
def test_run_with_dry_run(self, capsys: pytest.CaptureFixture[str]) -> None:
"""--dry-run 应只打印计划不执行."""
runner = px.CliRunner({"echo": _echo_graph()})
runner = px.CliRunner(aliases={"echo": _echo_graph()})
exit_code = runner.run(["echo", "--dry-run"])
assert exit_code == CliExitCode.SUCCESS.value
captured = capsys.readouterr()
@@ -272,7 +274,7 @@ class TestCliRunnerVerbose:
def test_verbose_default_prints_lifecycle(self, capsys: pytest.CaptureFixture[str]) -> None:
"""默认 verbose=True 应打印任务生命周期."""
runner = px.CliRunner({"echo": _echo_graph()})
runner = px.CliRunner(aliases={"echo": _echo_graph()})
_ = runner.run(["echo"])
captured = capsys.readouterr()
# verbose 模式下应打印任务生命周期
@@ -280,7 +282,7 @@ class TestCliRunnerVerbose:
def test_quiet_flag_disables_verbose(self, capsys: pytest.CaptureFixture[str]) -> None:
"""--quiet 应关闭 verbose 输出."""
runner = px.CliRunner({"echo": _echo_graph()})
runner = px.CliRunner(aliases={"echo": _echo_graph()})
_ = runner.run(["echo", "--quiet"])
captured = capsys.readouterr()
# quiet 模式下不应有 [verbose] 前缀的输出
@@ -288,14 +290,14 @@ class TestCliRunnerVerbose:
def test_verbose_false_constructor_disables_verbose(self, capsys: pytest.CaptureFixture[str]) -> None:
"""构造时 verbose=False 应关闭 verbose 输出."""
runner = px.CliRunner({"echo": _echo_graph()}, verbose=False)
runner = px.CliRunner(aliases={"echo": _echo_graph()}, verbose=False)
_ = runner.run(["echo"])
captured = capsys.readouterr()
assert "[verbose]" not in captured.out
def test_verbose_prints_command_for_cmd_task(self, capsys: pytest.CaptureFixture[str]) -> None:
"""verbose 模式下 cmd 任务应打印执行的命令."""
runner = px.CliRunner({"echo": _echo_graph(msg="verbose-test")})
runner = px.CliRunner(aliases={"echo": _echo_graph(msg="verbose-test")})
_ = runner.run(["echo"])
captured = capsys.readouterr()
# 应打印执行的命令
@@ -305,7 +307,7 @@ class TestCliRunnerVerbose:
def test_verbose_prints_success_lifecycle(self, capsys: pytest.CaptureFixture[str]) -> None:
"""verbose 模式下成功任务应打印成功信息."""
runner = px.CliRunner({"echo": _echo_graph()})
runner = px.CliRunner(aliases={"echo": _echo_graph()})
_ = runner.run(["echo"])
captured = capsys.readouterr()
assert "成功" in captured.out
@@ -319,14 +321,14 @@ class TestCliRunnerVerbose:
conditions=(lambda _ctx: False,),
),
])
runner = px.CliRunner({"skip": graph})
runner = px.CliRunner(aliases={"skip": graph})
_ = runner.run(["skip"])
captured = capsys.readouterr()
assert "跳过" in captured.out
def test_verbose_prints_failure_lifecycle(self, capsys: pytest.CaptureFixture[str]) -> None:
"""verbose 模式下失败任务应打印失败信息."""
runner = px.CliRunner({"fail": _failing_graph()})
runner = px.CliRunner(aliases={"fail": _failing_graph()})
_ = runner.run(["fail"])
captured = capsys.readouterr()
# 失败信息可能出现在 stdout (verbose) 或 stderr (PyFlowXError)
@@ -342,7 +344,7 @@ class TestCliRunnerRunFailure:
def test_run_unknown_command_returns_failure(self, capsys: pytest.CaptureFixture[str]) -> None:
"""未知命令应返回 1 并打印错误."""
runner = px.CliRunner({"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
exit_code = runner.run(["unknown"])
assert exit_code == CliExitCode.FAILURE.value
captured = capsys.readouterr()
@@ -351,7 +353,7 @@ class TestCliRunnerRunFailure:
def test_run_no_command_returns_failure(self, capsys: pytest.CaptureFixture[str]) -> None:
"""无命令时应返回 1 并打印帮助."""
runner = px.CliRunner({"clean": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph()})
exit_code = runner.run([])
assert exit_code == CliExitCode.FAILURE.value
captured = capsys.readouterr()
@@ -359,13 +361,13 @@ class TestCliRunnerRunFailure:
def test_run_failing_task_returns_failure(self) -> None:
"""任务失败时应返回 1."""
runner = px.CliRunner({"fail": _failing_graph()})
runner = px.CliRunner(aliases={"fail": _failing_graph()})
exit_code = runner.run(["fail"])
assert exit_code == CliExitCode.FAILURE.value
def test_run_failing_task_prints_error(self, capsys: pytest.CaptureFixture[str]) -> None:
"""任务失败时应打印错误信息."""
runner = px.CliRunner({"fail": _failing_graph()})
runner = px.CliRunner(aliases={"fail": _failing_graph()})
_ = runner.run(["fail"])
captured = capsys.readouterr()
# PyFlowXError 信息应输出到 stderr
@@ -380,17 +382,19 @@ class TestCliRunnerList:
def test_list_returns_success(self) -> None:
"""--list 应返回 0."""
runner = px.CliRunner({"clean": _echo_graph(), "build": _echo_graph()})
runner = px.CliRunner(aliases={"clean": _echo_graph(), "build": _echo_graph()})
exit_code = runner.run(["--list"])
assert exit_code == CliExitCode.SUCCESS.value
def test_list_prints_all_commands(self, capsys: pytest.CaptureFixture[str]) -> None:
"""--list 应打印所有命令."""
runner = px.CliRunner({
"clean": _echo_graph("c", "clean"),
"build": _echo_graph("b", "build"),
"test": _echo_graph("t", "test"),
})
runner = px.CliRunner(
aliases={
"clean": _echo_graph("c", "clean"),
"build": _echo_graph("b", "build"),
"test": _echo_graph("t", "test"),
}
)
_ = runner.run(["--list"])
captured = capsys.readouterr()
assert "clean" in captured.out
@@ -404,7 +408,7 @@ class TestCliRunnerList:
def track() -> None:
executed.append("ran")
runner = px.CliRunner({"a": px.Graph.from_specs([px.TaskSpec("a", track)])})
runner = px.CliRunner(aliases={"a": px.Graph.from_specs([px.TaskSpec("a", track)])})
_ = runner.run(["--list"])
assert executed == []
@@ -417,7 +421,7 @@ class TestCliRunnerErrorHandling:
def test_keyboard_interrupt_returns_130(self, capsys: pytest.CaptureFixture[str]) -> None:
"""KeyboardInterrupt 应返回 130."""
runner = px.CliRunner({"echo": _echo_graph()})
runner = px.CliRunner(aliases={"echo": _echo_graph()})
def raise_interrupt(*_args: Any, **_kwargs: Any) -> None:
raise KeyboardInterrupt
@@ -430,7 +434,7 @@ class TestCliRunnerErrorHandling:
def test_pyflowx_error_returns_failure(self, capsys: pytest.CaptureFixture[str]) -> None:
"""PyFlowXError 应返回 1."""
runner = px.CliRunner({"echo": _echo_graph()})
runner = px.CliRunner(aliases={"echo": _echo_graph()})
def raise_error(*_args: Any, **_kwargs: Any) -> None:
raise TaskFailedError("echo", RuntimeError("boom"), 1)
@@ -447,7 +451,7 @@ class TestCliRunnerErrorHandling:
class CustomError(Exception):
pass
runner = px.CliRunner({"echo": _echo_graph()})
runner = px.CliRunner(aliases={"echo": _echo_graph()})
def raise_custom(*_args: Any, **_kwargs: Any) -> None:
raise CustomError("unexpected")
@@ -464,14 +468,14 @@ class TestCliRunnerRunCli:
def test_run_cli_calls_sys_exit(self) -> None:
"""run_cli 应调用 sys.exit."""
runner = px.CliRunner({"echo": _echo_graph()})
runner = px.CliRunner(aliases={"echo": _echo_graph()})
with pytest.raises(SystemExit) as exc_info:
runner.run_cli(["echo"])
assert exc_info.value.code == CliExitCode.SUCCESS.value
def test_run_cli_exit_code_on_failure(self) -> None:
"""run_cli 失败时应以非零码退出."""
runner = px.CliRunner({"fail": _failing_graph()})
runner = px.CliRunner(aliases={"fail": _failing_graph()})
with pytest.raises(SystemExit) as exc_info:
runner.run_cli(["fail"])
assert exc_info.value.code == CliExitCode.FAILURE.value
@@ -479,7 +483,7 @@ class TestCliRunnerRunCli:
def test_run_cli_no_args_uses_sys_argv(self, monkeypatch: pytest.MonkeyPatch) -> None:
"""run_cli 无参数时应使用 sys.argv."""
monkeypatch.setattr(sys, "argv", ["pymake", "echo"])
runner = px.CliRunner({"echo": _echo_graph()})
runner = px.CliRunner(aliases={"echo": _echo_graph()})
with pytest.raises(SystemExit) as exc_info:
runner.run_cli()
assert exc_info.value.code == CliExitCode.SUCCESS.value
@@ -520,7 +524,7 @@ class TestCliRunnerIntegration:
conditions=(lambda _ctx: False,),
),
])
runner = px.CliRunner({"skip": graph})
runner = px.CliRunner(aliases={"skip": graph})
exit_code = runner.run(["skip"])
assert exit_code == CliExitCode.SUCCESS.value
@@ -533,7 +537,7 @@ class TestCliRunnerIntegration:
conditions=(lambda _ctx: True,),
),
])
runner = px.CliRunner({"run": graph})
runner = px.CliRunner(aliases={"run": graph})
exit_code = runner.run(["run"])
assert exit_code == CliExitCode.SUCCESS.value
@@ -554,17 +558,19 @@ class TestCliRunnerIntegration:
px.TaskSpec("c", make("c"), depends_on=("a",)),
px.TaskSpec("d", make("d"), depends_on=("b", "c")),
])
runner = px.CliRunner({"diamond": graph})
runner = px.CliRunner(aliases={"diamond": graph})
exit_code = runner.run(["diamond"])
assert exit_code == CliExitCode.SUCCESS.value
assert order == ["a", "b", "c", "d"]
def test_mixed_fn_and_cmd_commands(self) -> None:
"""混合 fn 和 cmd 的命令应都能执行."""
runner = px.CliRunner({
"fn_cmd": px.Graph.from_specs([px.TaskSpec("fn", fn=lambda: "fn-result")]),
"cmd_cmd": px.Graph.from_specs([px.TaskSpec("cmd", cmd=[*ECHO_CMD, "cmd-result"])]),
})
runner = px.CliRunner(
aliases={
"fn_cmd": px.Graph.from_specs([px.TaskSpec("fn", fn=lambda: "fn-result")]),
"cmd_cmd": px.Graph.from_specs([px.TaskSpec("cmd", cmd=[*ECHO_CMD, "cmd-result"])]),
}
)
assert runner.run(["fn_cmd"]) == CliExitCode.SUCCESS.value
assert runner.run(["cmd_cmd"]) == CliExitCode.SUCCESS.value
@@ -580,7 +586,7 @@ class TestCliRunnerIntegration:
ls_cmd = ["ls"]
graph = px.Graph.from_specs([px.TaskSpec("ls", cmd=ls_cmd, cwd=Path(tmpdir))])
runner = px.CliRunner({"ls": graph})
runner = px.CliRunner(aliases={"ls": graph})
exit_code = runner.run(["ls"])
assert exit_code == CliExitCode.SUCCESS.value
@@ -612,3 +618,109 @@ class TestApplyVerboseToGraph:
new_graph = _apply_verbose_to_graph(graph, verbose=True)
new_spec = new_graph.spec("a")
assert new_spec.verbose is True
# ---------------------------------------------------------------------- #
# 新 API: tasks + aliases
# ---------------------------------------------------------------------- #
class TestCliRunnerNewApi:
"""测试 CliRunner 的 tasks + aliases 新 API."""
def test_tasks_plus_aliases_single_str(self) -> None:
"""tasks 注册 + aliases str 引用单任务."""
runner = px.CliRunner(
tasks=[px.cmd([*ECHO_CMD, "a"], name="task_a")],
aliases={"a": "task_a"},
)
assert runner.commands == ["a"]
assert runner.run(["a"]) == CliExitCode.SUCCESS.value
def test_aliases_list_str_builds_chain(self) -> None:
"""aliases list[str] 应建立 chain 依赖(后一个依赖前一个)."""
runner = px.CliRunner(
tasks=[
px.cmd([*ECHO_CMD, "a"], name="task_a"),
px.cmd([*ECHO_CMD, "b"], name="task_b"),
],
aliases={"ab": ["task_a", "task_b"]},
)
graph = runner.graphs["ab"]
specs = graph.all_specs()
assert specs["task_b"].depends_on == ("task_a",)
def test_aliases_taskspec_value(self) -> None:
"""aliases 值为 TaskSpec 时直接生成单任务图."""
spec = px.cmd([*ECHO_CMD, "x"], name="inline_x")
runner = px.CliRunner(aliases={"x": spec})
assert runner.run(["x"]) == CliExitCode.SUCCESS.value
def test_aliases_graph_value(self) -> None:
"""aliases 值为 Graph 时原样使用(复杂场景:conditions 等)."""
graph = px.Graph.from_specs([
px.TaskSpec("a", cmd=[*ECHO_CMD, "a"]),
px.TaskSpec("b", cmd=[*ECHO_CMD, "b"], depends_on=("a",)),
])
runner = px.CliRunner(aliases={"g": graph})
assert set(runner.graphs["g"].all_specs().keys()) == {"a", "b"}
def test_alias_name_same_as_task_name_via_taskspec(self) -> None:
"""alias 名与 task 名相同时,用 TaskSpec 避免自引用循环."""
spec = px.cmd([*ECHO_CMD, "same"], name="same")
runner = px.CliRunner(aliases={"same": spec})
assert runner.run(["same"]) == CliExitCode.SUCCESS.value
def test_alias_str_reference_to_other_alias(self) -> None:
"""alias 值为 str 引用其他 alias."""
runner = px.CliRunner(
aliases={
"base": px.cmd([*ECHO_CMD, "base"], name="base"),
"wrapper": "base",
},
)
assert runner.run(["wrapper"]) == CliExitCode.SUCCESS.value
def test_empty_aliases_raises(self) -> None:
"""空 aliases 应抛 ValueError."""
with pytest.raises(ValueError, match="至少需要一个别名"):
_ = px.CliRunner()
def test_empty_list_value_raises(self) -> None:
"""空 list 作为 alias 值应抛 ValueError."""
with pytest.raises(ValueError, match="任务列表为空"):
_ = px.CliRunner(aliases={"x": []})
def test_invalid_value_type_raises(self) -> None:
"""无效类型(int)作为 alias 值应抛 TypeError."""
with pytest.raises(TypeError, match="值类型无效"):
_ = px.CliRunner(aliases={"x": 123}) # type: ignore[dict-item]
def test_invalid_list_element_type_raises(self) -> None:
"""list 中非 str/TaskSpec 元素应抛 TypeError."""
with pytest.raises(TypeError, match="列表元素类型无效"):
_ = px.CliRunner(aliases={"x": [123]}) # type: ignore[list-item]
def test_duplicate_task_name_raises(self) -> None:
"""tasks 中重名任务应抛 ValueError."""
spec = px.cmd([*ECHO_CMD, "a"], name="dup")
with pytest.raises(ValueError, match="任务名重复"):
_ = px.CliRunner(tasks=[spec, spec], aliases={"a": "dup"})
def test_commands_excludes_unreferenced_tasks(self) -> None:
"""commands 只含 aliases,不含 tasks 中未引用的任务."""
runner = px.CliRunner(
tasks=[
px.cmd([*ECHO_CMD, "a"], name="used"),
px.cmd([*ECHO_CMD, "b"], name="unused"),
],
aliases={"a": "used"},
)
assert runner.commands == ["a"]
def test_unknown_command_rejected(self) -> None:
"""未注册的 alias 名应被拒绝(不接受裸 task 名)."""
runner = px.CliRunner(
tasks=[px.cmd([*ECHO_CMD, "a"], name="task_a")],
aliases={"a": "task_a"},
)
# task_a 是任务名,不是 alias,应被拒绝
assert runner.run(["task_a"]) == CliExitCode.FAILURE.value
+63
View File
@@ -0,0 +1,63 @@
"""Tests for streaming result passing (iterators between tasks)."""
from __future__ import annotations
from typing import Iterator
import pyflowx as px
def test_generator_passed_as_iterator() -> None:
"""上游返回生成器,下游应能惰性消费."""
@px.task
def source() -> Iterator[int]:
yield from range(5)
@px.task(depends_on=("source",))
def consume(source: Iterator[int]) -> int:
return sum(source)
graph = px.Graph.from_specs([source, consume])
report = px.run(graph)
assert report.success
assert report["consume"] == 10
def test_large_range_streaming() -> None:
"""大范围迭代器流式传递,避免中间列表."""
@px.task
def numbers() -> Iterator[int]:
yield from range(1000)
@px.task(depends_on=("numbers",))
def total(numbers: Iterator[int]) -> int:
return sum(numbers)
graph = px.Graph.from_specs([numbers, total])
report = px.run(graph)
assert report.success
assert report["total"] == sum(range(1000))
def test_chain_multiple_streams() -> None:
"""多个流式任务串联."""
@px.task
def gen() -> Iterator[int]:
yield from range(10)
@px.task(depends_on=("gen",))
def doubled(gen: Iterator[int]) -> Iterator[int]:
for x in gen:
yield x * 2
@px.task(depends_on=("doubled",))
def collect(doubled: Iterator[int]) -> list[int]:
return list(doubled)
graph = px.Graph.from_specs([gen, doubled, collect])
report = px.run(graph)
assert report.success
assert report["collect"] == [x * 2 for x in range(10)]
+38 -2
View File
@@ -14,6 +14,7 @@ from pyflowx.task import (
TaskSpec,
TaskStatus,
_env_and_cwd,
cmd,
task_template,
)
@@ -78,6 +79,41 @@ def test_retry_policy_negative_jitter_rejected() -> None:
RetryPolicy(jitter=-1)
# ---------------------------------------------------------------------- #
# cmd() 工厂
# ---------------------------------------------------------------------- #
def test_cmd_factory_default_name_from_two_elements() -> None:
"""cmd() 默认 name = '_'.join(command[:2])."""
spec = cmd(["uv", "build"])
assert spec.name == "uv_build"
assert spec.cmd == ["uv", "build"]
def test_cmd_factory_default_name_single_element() -> None:
"""cmd() 单元素命令 name = command[0]."""
spec = cmd(["ls"])
assert spec.name == "ls"
def test_cmd_factory_explicit_name() -> None:
"""cmd() 显式 name 覆盖默认推导."""
spec = cmd(["ruff", "check", "--fix"], name="lint")
assert spec.name == "lint"
def test_cmd_factory_passes_depends_on() -> None:
"""cmd() depends_on 透传给 TaskSpec."""
spec = cmd(["echo", "b"], name="b", depends_on=("a",))
assert spec.depends_on == ("a",)
def test_cmd_factory_passes_extra_kwargs() -> None:
"""cmd() 其余 kwargs 透传给 TaskSpec."""
spec = cmd(["echo", "x"], name="x", timeout=10.0, tags=("t1",))
assert spec.timeout == 10.0
assert spec.tags == ("t1",)
def test_retry_policy_retries_property() -> None:
policy = RetryPolicy(max_attempts=3)
assert policy.retries == 2
@@ -157,8 +193,8 @@ def test_should_execute_skip_if_missing_cmd_not_found() -> None:
def test_should_execute_skip_if_missing_cmd_found() -> None:
"""skip_if_missing 但命令存在时应执行."""
# 使用 Python 作为已安装的命令
spec = TaskSpec("a", cmd=["echo"], skip_if_missing=True) # echo 应存在
# 使用 Python 作为已安装的命令Windows 上 echo 是 shell 内置,shutil.which 找不到)
spec = TaskSpec("a", cmd=["python"], skip_if_missing=True) # python 应存在
should_run, reason = spec.should_execute({})
assert should_run is True
assert reason is None
+136
View File
@@ -0,0 +1,136 @@
"""Tests for the @task decorator API."""
from __future__ import annotations
from pathlib import Path
from typing import Any, Mapping
import pyflowx as px
from pyflowx.task import RetryPolicy, TaskHooks, TaskSpec
def test_task_decorator_plain() -> None:
"""@task 无参数装饰:name 取函数名,返回 TaskSpec."""
@px.task
def extract() -> list[int]:
return [1, 2, 3]
assert isinstance(extract, TaskSpec)
assert extract.name == "extract"
assert extract.fn is not None
assert extract.depends_on == ()
def test_task_decorator_with_params() -> None:
"""@task(...) 带参数装饰:传递依赖与重试."""
@px.task(depends_on=("extract",), retry=RetryPolicy(max_attempts=3))
def double(extract: list[int]) -> list[int]:
return [x * 2 for x in extract]
assert isinstance(double, TaskSpec)
assert double.name == "double"
assert double.depends_on == ("extract",)
assert double.retry.max_attempts == 3
def test_task_decorator_explicit_name() -> None:
"""@task(name=...) 应使用显式名称而非函数名."""
@px.task(name="custom_name")
def my_func() -> None:
return None
assert my_func.name == "custom_name"
def test_task_decorator_cmd_form() -> None:
"""@task(cmd=...) 应支持命令形式."""
spec = px.task(cmd=["ls", "-la"], name="list_files")
assert isinstance(spec, TaskSpec)
assert spec.name == "list_files"
assert spec.cmd == ["ls", "-la"]
def test_task_decorator_full_options() -> None:
"""@task 应支持全部 TaskSpec 字段."""
@px.task(
depends_on=("a",),
soft_depends_on=("b",),
defaults={"b": 0},
args=(1,),
kwargs={"x": 2},
retry=RetryPolicy(max_attempts=5),
timeout=10.0,
tags=("t1",),
conditions=(px.BuiltinConditions.IS_WINDOWS,), # type: ignore[arg-type]
cwd="/tmp",
env={"K": "v"},
verbose=True,
skip_if_missing=True,
allow_upstream_skip=True,
strategy="thread",
priority=3,
concurrency_key="db",
continue_on_error=True,
)
def f(a: int) -> int:
return a
assert f.depends_on == ("a",)
assert f.soft_depends_on == ("b",)
assert f.defaults == {"b": 0}
assert f.args == (1,)
assert f.kwargs == {"x": 2}
assert f.retry.max_attempts == 5
assert f.timeout == 10.0
assert f.tags == ("t1",)
assert len(f.conditions) == 1
assert isinstance(f.cwd, Path)
assert f.cwd == Path("/tmp")
assert f.env == {"K": "v"}
assert f.verbose is True
assert f.skip_if_missing is True
assert f.allow_upstream_skip is True
assert f.strategy == "thread"
assert f.priority == 3
assert f.concurrency_key == "db"
assert f.continue_on_error is True
def test_task_decorator_runs_in_graph() -> None:
"""装饰器生成的 TaskSpec 应能直接构建图并运行."""
@px.task
def extract() -> list[int]:
return [1, 2, 3]
@px.task(depends_on=("extract",))
def double(extract: list[int]) -> list[int]:
return [x * 2 for x in extract]
graph = px.Graph.from_specs([extract, double])
report = px.run(graph)
assert report.success
assert report["double"] == [2, 4, 6]
def test_task_decorator_hooks_passthrough() -> None:
"""@task(hooks=...) 应传递 TaskHooks 实例."""
hooks = TaskHooks(pre_run=lambda _spec: None)
spec = px.task(fn=lambda: None, hooks=hooks, name="h")
assert spec.hooks is hooks
def test_task_decorator_cache_key_passthrough() -> None:
"""@task(cache_key=...) 应传递缓存键函数."""
def ck(ctx: Mapping[str, Any]) -> str:
return "k"
spec = px.task(fn=lambda: None, cache_key=ck, name="c")
assert spec.cache_key is ck
Generated
+1 -1
View File
@@ -5603,7 +5603,7 @@ pycountry = [
[[package]]
name = "pyflowx"
version = "0.2.11"
version = "0.2.13"
source = { editable = "." }
dependencies = [
{ name = "graphlib-backport", marker = "python_full_version < '3.9'" },