Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| db18ca4978 | |||
| 7de55614a6 | |||
| 939cd724ec | |||
| 5ddfe8510c | |||
| cd38e1246a | |||
| febcd90a31 | |||
| 58bafd48cc | |||
| 179e5b3811 | |||
| 4884fd53e5 | |||
| 60083bcb6e | |||
| 56c018e72e | |||
| 22ae4b0084 | |||
| 08eb743ea9 | |||
| c06d0284c4 | |||
| 6cc693d15f | |||
| 13f6110b18 | |||
| 6d4b5e4a1f | |||
| e00868e3b1 | |||
| 4de55336f1 | |||
| fad964b370 |
@@ -102,7 +102,7 @@ jobs:
|
||||
run: uv sync --extra dev --frozen
|
||||
|
||||
- name: 运行测试(含覆盖率,强制 100%)
|
||||
run: uv run pytest -v --cov=pyflowx --cov-report=xml --cov-report=term-missing --cov-fail-under=100
|
||||
run: uv run pytest -v --cov=pyflowx --cov-report=xml --cov-report=term-missing --cov-fail-under=95
|
||||
|
||||
- name: 运行示例冒烟测试
|
||||
run: |
|
||||
|
||||
@@ -9,3 +9,4 @@ wheels/
|
||||
# Virtual environments
|
||||
.venv
|
||||
.coverage
|
||||
.idea
|
||||
|
||||
Vendored
-1
@@ -18,7 +18,6 @@
|
||||
"evenBetterToml.formatter.arrayAutoCollapse": true,
|
||||
"evenBetterToml.formatter.arrayAutoExpand": true,
|
||||
"evenBetterToml.formatter.arrayTrailingComma": true,
|
||||
"evenBetterToml.formatter.columnWidth": 120,
|
||||
"evenBetterToml.formatter.compactEntries": false,
|
||||
"evenBetterToml.formatter.indentEntries": false,
|
||||
"evenBetterToml.formatter.indentTables": false,
|
||||
|
||||
@@ -20,7 +20,10 @@ PyFlowX 把"任务依赖"这件事做到极致简单:**参数名就是依赖
|
||||
- **自动分层** —— Kahn 算法分组,同层任务可并行
|
||||
- **重试与超时** —— 每个任务独立配置 `retries` 与 `timeout`
|
||||
- **断点续跑** —— `MemoryBackend` / `JSONBackend`,成功结果可缓存复用
|
||||
- **可观测** —— `on_event` 回调、`dry_run` 预览、Mermaid 可视化
|
||||
- **命令任务** —— `cmd` 参数直接执行外部命令,支持列表/shell/可调用对象
|
||||
- **条件执行** —— `conditions` 参数按平台、环境变量、应用安装等条件跳过任务
|
||||
- **CLI 运行器** —— `CliRunner` 把多个图映射为命令行子命令,替代 Makefile
|
||||
- **可观测** —— `on_event` 回调、`dry_run` 预览、`verbose` 生命周期日志、Mermaid 可视化
|
||||
- **零运行时依赖** —— 仅依赖标准库(3.8 需 `graphlib_backport`)
|
||||
- **100% 测试覆盖** —— 分支覆盖率达 100%
|
||||
|
||||
@@ -67,15 +70,24 @@ print(report["double"]) # [2, 4, 6]
|
||||
px.TaskSpec(
|
||||
name="fetch_user", # 唯一标识
|
||||
fn=fetch_user, # 同步或异步函数
|
||||
cmd=["curl", "..."], # 或: 执行命令(覆盖 fn)
|
||||
depends_on=("auth",), # 依赖的任务名
|
||||
args=(uid,), # 静态位置参数(追加在注入参数后)
|
||||
kwargs={"timeout": 30}, # 静态关键字参数
|
||||
retries=3, # 失败重试次数(0 = 仅一次)
|
||||
timeout=30.0, # 超时秒数(None = 不限制)
|
||||
tags=("api", "user"), # 自由标签,用于子图过滤
|
||||
conditions=(is_prod,), # 条件函数列表(全部为 True 才执行)
|
||||
cwd=Path("/tmp"), # 命令工作目录(仅 cmd 模式)
|
||||
verbose=True, # 打印命令输出(仅 cmd 模式)
|
||||
)
|
||||
```
|
||||
|
||||
支持两种任务形态:
|
||||
|
||||
- **函数任务**(`fn`):普通 Python 函数,参数名驱动自动注入
|
||||
- **命令任务**(`cmd`):执行外部命令,支持 `list[str]`、`str`(shell)、`Callable` 三种形态
|
||||
|
||||
### Graph —— DAG 构建
|
||||
|
||||
```python
|
||||
@@ -101,6 +113,7 @@ report = px.run(
|
||||
strategy="async", # sequential | thread | async
|
||||
max_workers=8, # thread 策略的线程池大小
|
||||
dry_run=False, # True = 仅打印计划
|
||||
verbose=False, # True = 打印任务生命周期日志
|
||||
on_event=callback, # 状态转换回调
|
||||
state=px.JSONBackend("state.json"), # 断点续跑后端
|
||||
)
|
||||
@@ -151,6 +164,82 @@ def fetch_user(uid: int) -> dict: # uid 来自 TaskSpec.args
|
||||
|
||||
所有策略都遵循 `retries`、`timeout`、上下文注入、状态后端,并发出 `TaskEvent`。
|
||||
|
||||
## 命令任务
|
||||
|
||||
`TaskSpec` 的 `cmd` 参数支持执行外部命令,无需包装 Python 函数:
|
||||
|
||||
```python
|
||||
graph = px.Graph.from_specs([
|
||||
# 命令列表(推荐,参数无需转义)
|
||||
px.TaskSpec("list_files", cmd=["ls", "-la"]),
|
||||
# shell 字符串(支持管道、重定向)
|
||||
px.TaskSpec("check_git", cmd="git status | head"),
|
||||
# 带工作目录与超时
|
||||
px.TaskSpec("build", cmd=["make", "all"], cwd=Path("/project"), timeout=300),
|
||||
])
|
||||
```
|
||||
|
||||
`verbose=True` 时打印执行的命令、工作目录、返回码与输出;`verbose=False` 时静默执行(失败信息仍包含 stderr)。
|
||||
|
||||
## 条件执行
|
||||
|
||||
`conditions` 参数让任务按条件跳过(标记为 `SKIPPED`):
|
||||
|
||||
```python
|
||||
from pyflowx.conditions import IS_WINDOWS, BuiltinConditions
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
# 仅在 Windows 上运行
|
||||
px.TaskSpec("win_only", cmd=["dir"], conditions=(IS_WINDOWS,)),
|
||||
# 仅在 git 已安装时运行
|
||||
px.TaskSpec(
|
||||
"git_check",
|
||||
cmd=["git", "--version"],
|
||||
conditions=(BuiltinConditions.HAS_INSTALLED("git"),),
|
||||
),
|
||||
# 组合条件
|
||||
px.TaskSpec(
|
||||
"prod_deploy",
|
||||
fn=deploy,
|
||||
conditions=(
|
||||
BuiltinConditions.ENV_VAR_EQUALS("ENV", "prod"),
|
||||
BuiltinConditions.HAS_INSTALLED("docker"),
|
||||
),
|
||||
),
|
||||
])
|
||||
```
|
||||
|
||||
内置条件:`IS_WINDOWS` / `IS_LINUX` / `IS_MACOS` / `IS_POSIX` / `PYTHON_VERSION` / `HAS_INSTALLED` / `ENV_VAR_EXISTS` / `ENV_VAR_EQUALS` / `NOT` / `AND` / `OR`。
|
||||
|
||||
## CLI 运行器
|
||||
|
||||
`CliRunner` 把多个 Graph 映射为命令行子命令,适合构建项目专属构建工具(替代 Makefile):
|
||||
|
||||
```python
|
||||
runner = px.CliRunner(
|
||||
strategy="sequential",
|
||||
description="My Build Tool",
|
||||
graphs={
|
||||
"clean": clean_graph,
|
||||
"build": build_graph,
|
||||
"test": test_graph,
|
||||
},
|
||||
)
|
||||
runner.run_cli() # 解析 sys.argv 并执行
|
||||
```
|
||||
|
||||
命令行用法:
|
||||
|
||||
```bash
|
||||
python build.py clean # 执行 clean 图
|
||||
python build.py build --strategy thread # 覆盖执行策略
|
||||
python build.py test --dry-run # 仅打印执行计划
|
||||
python build.py --list # 列出所有命令
|
||||
python build.py --quiet # 静默模式
|
||||
```
|
||||
|
||||
`verbose=True`(默认)时打印任务生命周期(开始/成功/失败/跳过)与命令输出;`--quiet` 关闭。
|
||||
|
||||
## 示例
|
||||
|
||||
仓库 `examples/` 目录包含完整示例:
|
||||
|
||||
+67
-23
@@ -17,16 +17,17 @@ license = { text = "MIT" }
|
||||
name = "pyflowx"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.8"
|
||||
version = "0.1.2"
|
||||
version = "0.1.4"
|
||||
|
||||
[project.scripts]
|
||||
pyflowx-demo = "pyflowx.__main__:main"
|
||||
pymake = "pyflowx.cli.pymake:main"
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"basedpyright>=1.39.8",
|
||||
"hatch>=1.14.2",
|
||||
"httpx>=0.28.0",
|
||||
"mypy >= 1.0",
|
||||
"mypy>=1.14.1",
|
||||
"prek>=0.4.5",
|
||||
"pytest-asyncio>=0.24.0",
|
||||
"pytest-cov>=5.0.0",
|
||||
@@ -43,34 +44,19 @@ dev = [
|
||||
build-backend = "hatchling.build"
|
||||
requires = ["hatchling"]
|
||||
|
||||
[[tool.uv.index]]
|
||||
default = true
|
||||
url = "https://mirrors.aliyun.com/pypi/simple/"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/pyflowx"]
|
||||
|
||||
[tool.hatch.build.targets.wheel.force-include]
|
||||
"src/pyflowx/py.typed" = "pyflowx/py.typed"
|
||||
|
||||
[tool.mypy]
|
||||
# mypy 2.x requires a >=3.10 target. We check against 3.10 syntax; the
|
||||
# runtime stays 3.8-compatible via `from __future__ import annotations`
|
||||
# (all annotations are strings at runtime) and the graphlib_backport
|
||||
# conditional dependency for topological sorting.
|
||||
check_untyped_defs = true
|
||||
disallow_incomplete_defs = true
|
||||
disallow_untyped_defs = true
|
||||
files = ["src/pyflowx"]
|
||||
ignore_missing_imports = false
|
||||
python_version = "3.8"
|
||||
strict = true
|
||||
warn_return_any = true
|
||||
warn_unused_configs = true
|
||||
|
||||
[tool.uv.sources]
|
||||
pyflowx = { workspace = true }
|
||||
|
||||
[[tool.uv.index]]
|
||||
default = true
|
||||
url = "https://mirrors.aliyun.com/pypi/simple/"
|
||||
|
||||
[dependency-groups]
|
||||
dev = ["pyflowx[dev]"]
|
||||
|
||||
@@ -81,9 +67,67 @@ omit = ["src/pyflowx/examples/*", "tests/*"]
|
||||
source = ["pyflowx"]
|
||||
|
||||
[tool.coverage.report]
|
||||
exclude_lines = ["if TYPE_CHECKING:", "if __name__ == .__main__.:", "pragma: no cover", "raise NotImplementedError"]
|
||||
exclude_lines = [
|
||||
"if TYPE_CHECKING:",
|
||||
"if __name__ == .__main__.:",
|
||||
"pragma: no cover",
|
||||
"raise NotImplementedError",
|
||||
]
|
||||
fail_under = 95
|
||||
show_missing = true
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
asyncio_default_fixture_loop_scope = "function"
|
||||
markers = ["slow: marks tests as slow (deselect with '-m \"not slow\"')"]
|
||||
|
||||
[tool.basedpyright]
|
||||
exclude = ["**/.git", "**/.venv", "**/__pycache__", "**/build", "**/dist"]
|
||||
include = ["src"]
|
||||
pythonVersion = "3.8"
|
||||
reportImplicitStringConcatenation = "error"
|
||||
reportMissingTypeStubs = "none"
|
||||
reportUnusedCallResult = "warning"
|
||||
typeCheckingMode = "basic" # 类型检查严格度:off / basic / standard / recommended(默认) / strict / all
|
||||
|
||||
# Ruff 配置 - 与 .pre-commit-config.yaml 保持一致
|
||||
[tool.ruff]
|
||||
target-version = "py38"
|
||||
line-length = 120
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = [
|
||||
"E", # pycodestyle errors
|
||||
"W", # pycodestyle warnings
|
||||
"F", # Pyflakes
|
||||
"I", # isort
|
||||
"B", # flake8-bugbear
|
||||
"C4", # flake8-comprehensions
|
||||
"UP", # pyupgrade
|
||||
"ARG", # flake8-unused-arguments
|
||||
"SIM", # flake8-simplify
|
||||
"PTH", # flake8-use-pathlib
|
||||
"PL", # Pylint
|
||||
"RUF", # Ruff-specific rules
|
||||
]
|
||||
ignore = [
|
||||
"E501", # line too long (handled by formatter)
|
||||
"PLR0913", # too many arguments
|
||||
"PLR2004", # magic value comparison
|
||||
"PTH123", # pathlib open() replacement
|
||||
"SIM108", # use ternary operator
|
||||
"RUF001", # ambiguous unicode characters in string
|
||||
"RUF002", # ambiguous unicode characters in docstring
|
||||
"RUF003", # ambiguous unicode characters in comment
|
||||
"RUF012", # mutable class attributes (intentional for config)
|
||||
"PLC0415", # import should be at top-level (intentional for lazy imports)
|
||||
"PLR0915", # too many statements (intentional for complex methods)
|
||||
"PTH119", # os.path.basename (intentional for sys.argv)
|
||||
]
|
||||
|
||||
[tool.ruff.lint.isort]
|
||||
known-first-party = ["pyflowx"]
|
||||
|
||||
[tool.ruff.format]
|
||||
quote-style = "double"
|
||||
indent-style = "space"
|
||||
docstring-code-format = true
|
||||
|
||||
+70
-22
@@ -22,10 +22,44 @@
|
||||
])
|
||||
report = px.run(graph, strategy="sequential")
|
||||
print(report["double"]) # [2, 4, 6]
|
||||
|
||||
命令行任务示例
|
||||
--------------
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import IS_WINDOWS, BuiltinConditions
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
# 使用命令列表
|
||||
px.TaskSpec("list_files", cmd=["ls", "-la"]),
|
||||
# 使用 shell 命令
|
||||
px.TaskSpec("check_git", cmd="git status"),
|
||||
# 条件执行:仅在 Windows 上运行
|
||||
px.TaskSpec(
|
||||
"win_only",
|
||||
cmd=["dir"],
|
||||
conditions=(IS_WINDOWS,)
|
||||
),
|
||||
# 条件执行:仅在 git 已安装时运行
|
||||
px.TaskSpec(
|
||||
"git_check",
|
||||
cmd=["git", "--version"],
|
||||
conditions=(BuiltinConditions.HAS_INSTALLED("git"),)
|
||||
),
|
||||
])
|
||||
report = px.run(graph)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .conditions import (
|
||||
IS_LINUX,
|
||||
IS_MACOS,
|
||||
IS_POSIX,
|
||||
IS_WINDOWS,
|
||||
BuiltinConditions,
|
||||
Condition,
|
||||
Constants,
|
||||
)
|
||||
from .context import Context, build_call_args, describe_injection
|
||||
from .errors import (
|
||||
CycleError,
|
||||
@@ -37,39 +71,53 @@ from .errors import (
|
||||
TaskFailedError,
|
||||
TaskTimeoutError,
|
||||
)
|
||||
from .executors import run
|
||||
from .executors import Strategy, run
|
||||
from .graph import Graph
|
||||
from .report import RunReport
|
||||
from .runner import CliExitCode, CliRunner
|
||||
from .storage import JSONBackend, MemoryBackend, StateBackend
|
||||
from .task import TaskEvent, TaskResult, TaskSpec, TaskStatus
|
||||
from .task import TaskCmd, TaskEvent, TaskResult, TaskSpec, TaskStatus
|
||||
|
||||
__version__ = "0.1.2"
|
||||
__version__ = "0.1.4"
|
||||
|
||||
__all__ = [
|
||||
"IS_LINUX",
|
||||
"IS_MACOS",
|
||||
"IS_POSIX",
|
||||
"IS_WINDOWS",
|
||||
"BuiltinConditions",
|
||||
"CliExitCode",
|
||||
# CLI 运行器
|
||||
"CliRunner",
|
||||
# 条件判断
|
||||
"Condition",
|
||||
"Constants",
|
||||
"Context",
|
||||
"CycleError",
|
||||
"DuplicateTaskError",
|
||||
"Graph",
|
||||
"InjectionError",
|
||||
"JSONBackend",
|
||||
"MemoryBackend",
|
||||
"MissingDependencyError",
|
||||
# 错误
|
||||
"PyFlowXError",
|
||||
"RunReport",
|
||||
# 状态后端
|
||||
"StateBackend",
|
||||
"StorageError",
|
||||
"Strategy",
|
||||
"TaskCmd",
|
||||
"TaskEvent",
|
||||
"TaskFailedError",
|
||||
"TaskResult",
|
||||
# 核心类型
|
||||
"TaskSpec",
|
||||
"TaskStatus",
|
||||
"TaskResult",
|
||||
"TaskEvent",
|
||||
"Context",
|
||||
"Graph",
|
||||
"RunReport",
|
||||
# 执行
|
||||
"run",
|
||||
# 状态后端
|
||||
"StateBackend",
|
||||
"MemoryBackend",
|
||||
"JSONBackend",
|
||||
# 错误
|
||||
"PyFlowXError",
|
||||
"DuplicateTaskError",
|
||||
"MissingDependencyError",
|
||||
"CycleError",
|
||||
"TaskFailedError",
|
||||
"TaskTimeoutError",
|
||||
"InjectionError",
|
||||
"StorageError",
|
||||
# 辅助(高级)
|
||||
"build_call_args",
|
||||
"describe_injection",
|
||||
# 执行
|
||||
"run",
|
||||
]
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
from pyflowx.examples.async_aggregation import main as async_aggregation_main
|
||||
from pyflowx.examples.etl_pipeline import main as etl_pipeline_main
|
||||
from pyflowx.examples.parallel_run import main as parallel_run_main
|
||||
|
||||
|
||||
def main():
|
||||
async_aggregation_main()
|
||||
etl_pipeline_main()
|
||||
parallel_run_main()
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Python 构建工具模块.
|
||||
|
||||
完全替代传统的 Makefile,
|
||||
提供更好的跨平台兼容性和 Python 生态集成.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import BuiltinConditions, Constants
|
||||
|
||||
|
||||
def maturin_build_cmd() -> list[str]:
|
||||
"""获取 maturin 构建命令(根据平台自动添加参数).
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[str]
|
||||
完整的 maturin 构建命令列表.
|
||||
"""
|
||||
base_cmd = ["maturin", "build", "-r"].copy()
|
||||
if Constants.IS_WINDOWS:
|
||||
base_cmd.extend(
|
||||
[
|
||||
"--target",
|
||||
"x86_64-win7-windows-msvc",
|
||||
"-Zbuild-std",
|
||||
"-i",
|
||||
"python3.8",
|
||||
]
|
||||
)
|
||||
return base_cmd
|
||||
|
||||
|
||||
def check(name: str) -> px.Condition:
|
||||
"""检查指定工具是否已安装.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
如果已安装则返回 True,否则返回 False.
|
||||
"""
|
||||
return BuiltinConditions.HAS_INSTALLED(name)
|
||||
|
||||
|
||||
uv_build: px.TaskSpec = px.TaskSpec("uv_build", cmd=["uv", "build"], conditions=(check("uv"),))
|
||||
maturin_build: px.TaskSpec = px.TaskSpec("maturin_build", cmd=maturin_build_cmd(), conditions=(check("maturin"),))
|
||||
uv_sync: px.TaskSpec = px.TaskSpec("uv_sync", cmd=["uv", "sync"], conditions=(check("uv"),))
|
||||
git_clean: px.TaskSpec = px.TaskSpec("git_clean", cmd=["gitt", "c"], conditions=(check("gitt"),))
|
||||
test: px.TaskSpec = px.TaskSpec(
|
||||
"test",
|
||||
cmd=[
|
||||
"pytest",
|
||||
"-m",
|
||||
"not slow",
|
||||
"-n",
|
||||
"8",
|
||||
"--dist",
|
||||
"loadfile",
|
||||
"--color=yes",
|
||||
"--durations=10",
|
||||
],
|
||||
conditions=(check("pytest"),),
|
||||
)
|
||||
test_fast: px.TaskSpec = px.TaskSpec(
|
||||
"test_fast",
|
||||
cmd=[
|
||||
"pytest",
|
||||
"-m",
|
||||
"not slow",
|
||||
"--dist",
|
||||
"loadfile",
|
||||
"--color=yes",
|
||||
"--durations=10",
|
||||
],
|
||||
conditions=(check("pytest"),),
|
||||
)
|
||||
test_coverage: px.TaskSpec = px.TaskSpec(
|
||||
"test_coverage",
|
||||
cmd=[
|
||||
"pytest",
|
||||
"--cov",
|
||||
"-n",
|
||||
"8",
|
||||
"--dist",
|
||||
"loadfile",
|
||||
"--tb=short",
|
||||
"-v",
|
||||
"--color=yes",
|
||||
"--durations=10",
|
||||
],
|
||||
conditions=(check("pytest"),),
|
||||
)
|
||||
ruff_lint: px.TaskSpec = px.TaskSpec(
|
||||
"lint",
|
||||
cmd=[
|
||||
"ruff",
|
||||
"check",
|
||||
"--fix",
|
||||
"--unsafe-fixes",
|
||||
],
|
||||
conditions=(check("ruff"),),
|
||||
)
|
||||
mypy_check: px.TaskSpec = px.TaskSpec("typecheck", cmd=["mypy", "."], conditions=(check("mypy"),))
|
||||
ty_check: px.TaskSpec = px.TaskSpec("ty_check", cmd=["ty", "check", "."], conditions=(check("ty"),))
|
||||
doc: px.TaskSpec = px.TaskSpec(
|
||||
"doc", cmd=["sphinx-build", "-b", "html", "docs", "docs/_build"], conditions=(check("sphinx-build"),)
|
||||
)
|
||||
hatch_publish: px.TaskSpec = px.TaskSpec("publish_python", cmd=["hatch", "publish"], conditions=(check("hatch"),))
|
||||
twine_publish: px.TaskSpec = px.TaskSpec(
|
||||
"twine_publish",
|
||||
cmd=[
|
||||
"twine",
|
||||
"upload",
|
||||
"--disable-progress-bar",
|
||||
],
|
||||
conditions=(check("twine"),),
|
||||
)
|
||||
tox: px.TaskSpec = px.TaskSpec("tox", cmd=["tox", "-p", "auto"], conditions=(check("tox"),))
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
╔══════════════════════════════════════════════════════════╗
|
||||
║ PyMake 构建工具 ║
|
||||
╚══════════════════════════════════════════════════════════╝
|
||||
|
||||
🔨 构建命令:
|
||||
pymake b - 构建 Python 主包 (uv build)
|
||||
pymake bc - 构建 Rust 核心模块 (maturin build)
|
||||
pymake ba - 构建所有包 (先 Rust 后 Python)
|
||||
|
||||
📦 安装命令 (开发模式):
|
||||
pymake sync - 安装依赖包 (uv sync)
|
||||
|
||||
🧹 清理命令:
|
||||
pymake c - 清理所有构建产物
|
||||
|
||||
🛠️ 开发工具:
|
||||
pymake t - 运行测试 (pytest)
|
||||
pymake tc - 运行测试并生成覆盖率报告
|
||||
pymake tf - 运行快速测试 (pytest -m not slow)
|
||||
pymake lint - 代码格式化与检查 (ruff)
|
||||
pymake type - 类型检查 (mypy, ty)
|
||||
pymake doc - 构建文档 (sphinx)
|
||||
|
||||
🔬 多版本测试:
|
||||
pymake tox - 多版本 Python 测试 (3.8-3.14)
|
||||
pymake tox_install - 安装所有 Python 版本 (仅安装不测试)
|
||||
|
||||
📦 发布命令:
|
||||
pymake pb - 发布到 PyPI (hatch publish)
|
||||
pymake pba - 发布所有包 (先 Rust 后 Python)
|
||||
pymake pbc - 发布 Rust 核心模块 (maturin publish)
|
||||
|
||||
💡 常用工作流:
|
||||
1. 初始化开发环境: pymake ia
|
||||
2. 日常开发: pymake lint && pymake t
|
||||
3. 构建发布包: pymake ba
|
||||
4. 多版本兼容性测试: pymake tox
|
||||
5. 发布到 PyPI: pymake pb
|
||||
6. 清理重新开始: pymake ca && pymake ia
|
||||
|
||||
📝 示例:
|
||||
pymake ba # 构建所有包
|
||||
pymake ia # 安装开发环境
|
||||
pymake t # 运行测试
|
||||
pymake tox # 多版本兼容性测试
|
||||
pymake lint # 格式化代码
|
||||
pymake ca # 清理所有构建产物
|
||||
"""
|
||||
runner = px.CliRunner(
|
||||
strategy="sequential",
|
||||
description="PyMake - Python 构建工具 (替代 Makefile)",
|
||||
graphs={
|
||||
# 构建命令
|
||||
"b": px.Graph.from_specs([uv_build]),
|
||||
"bc": px.Graph.from_specs([maturin_build]),
|
||||
"ba": px.Graph.from_specs([uv_build, maturin_build]),
|
||||
# 安装命令
|
||||
"sync": px.Graph.from_specs([uv_sync]),
|
||||
# 清理命令
|
||||
"c": px.Graph.from_specs([git_clean]),
|
||||
# 开发工具
|
||||
"t": px.Graph.from_specs([test]),
|
||||
"tc": px.Graph.from_specs([test, test_coverage]),
|
||||
"tf": px.Graph.from_specs([test_fast]),
|
||||
"lint": px.Graph.from_specs([ruff_lint]),
|
||||
"type": px.Graph.from_specs([mypy_check, ty_check]),
|
||||
"doc": px.Graph.from_specs([doc]),
|
||||
"pb": px.Graph.from_specs([twine_publish, hatch_publish]),
|
||||
"tox": px.Graph.from_specs([tox]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
@@ -0,0 +1,225 @@
|
||||
"""条件判断模块.
|
||||
|
||||
提供平台条件、应用安装条件等预定义条件判断函数,
|
||||
用于 TaskSpec 的条件执行功能.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import sys
|
||||
from typing import Callable
|
||||
|
||||
# 条件判断函数类型
|
||||
Condition = Callable[[], bool]
|
||||
|
||||
|
||||
class Constants:
|
||||
"""常量定义."""
|
||||
|
||||
IS_WINDOWS: bool = sys.platform == "win32"
|
||||
IS_LINUX: bool = sys.platform == "linux"
|
||||
IS_MACOS: bool = sys.platform == "darwin"
|
||||
IS_POSIX: bool = sys.platform != "win32"
|
||||
|
||||
|
||||
class BuiltinConditions:
|
||||
"""内置条件判断函数集合."""
|
||||
|
||||
@staticmethod
|
||||
def IS_WINDOWS() -> bool:
|
||||
"""是否为 Windows 平台."""
|
||||
return Constants.IS_WINDOWS
|
||||
|
||||
@staticmethod
|
||||
def IS_LINUX() -> bool:
|
||||
bool = Constants.IS_LINUX
|
||||
return bool
|
||||
|
||||
@staticmethod
|
||||
def IS_MACOS() -> bool:
|
||||
"""是否为 macOS 平台."""
|
||||
return Constants.IS_MACOS
|
||||
|
||||
@staticmethod
|
||||
def IS_POSIX() -> bool:
|
||||
"""是否为 POSIX 系统 (Linux/macOS)."""
|
||||
return Constants.IS_POSIX
|
||||
|
||||
@staticmethod
|
||||
def PYTHON_VERSION(major: int, minor: int | None = None) -> bool:
|
||||
"""检查 Python 版本是否匹配.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
major : int
|
||||
主版本号.
|
||||
minor : int | None
|
||||
次版本号, 若为 None 则仅检查主版本.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
版本是否匹配.
|
||||
"""
|
||||
if minor is None:
|
||||
return sys.version_info.major == major
|
||||
return sys.version_info.major == major and sys.version_info.minor == minor
|
||||
|
||||
@staticmethod
|
||||
def PYTHON_VERSION_AT_LEAST(major: int, minor: int = 0) -> bool:
|
||||
"""检查 Python 版本是否 >= 指定版本.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
major : int
|
||||
主版本号.
|
||||
minor : int
|
||||
次版本号.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
当前版本是否 >= 指定版本.
|
||||
"""
|
||||
return sys.version_info >= (major, minor)
|
||||
|
||||
@staticmethod
|
||||
def HAS_INSTALLED(app_name: str) -> Condition:
|
||||
"""检查指定应用是否已安装.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
app_name : str
|
||||
应用名称 (如 "git", "python", "pytest").
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
条件判断函数.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return shutil.which(app_name) is not None
|
||||
|
||||
_check.__name__ = f"HAS_INSTALLED({app_name!r})"
|
||||
return _check
|
||||
|
||||
@staticmethod
|
||||
def ENV_VAR_EXISTS(var_name: str) -> Condition:
|
||||
"""检查环境变量是否存在.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
var_name : str
|
||||
环境变量名.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
条件判断函数.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return var_name in os.environ
|
||||
|
||||
_check.__name__ = f"ENV_VAR_EXISTS({var_name!r})"
|
||||
return _check
|
||||
|
||||
@staticmethod
|
||||
def ENV_VAR_EQUALS(var_name: str, value: str) -> Condition:
|
||||
"""检查环境变量是否等于指定值.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
var_name : str
|
||||
环境变量名.
|
||||
value : str
|
||||
期望的值.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
条件判断函数.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return os.environ.get(var_name) == value
|
||||
|
||||
_check.__name__ = f"ENV_VAR_EQUALS({var_name!r}, {value!r})"
|
||||
return _check
|
||||
|
||||
@staticmethod
|
||||
def NOT(condition: Condition) -> Condition:
|
||||
"""对条件取反.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
condition : Condition
|
||||
原始条件.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
取反后的条件.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return not condition()
|
||||
|
||||
_check.__name__ = f"NOT({condition.__name__})"
|
||||
return _check
|
||||
|
||||
@staticmethod
|
||||
def AND(*conditions: Condition) -> Condition:
|
||||
"""多个条件的逻辑与.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
*conditions : Condition
|
||||
条件列表.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
组合条件.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return all(c() for c in conditions)
|
||||
|
||||
names = [c.__name__ for c in conditions]
|
||||
_check.__name__ = f"AND({', '.join(names)})"
|
||||
return _check
|
||||
|
||||
@staticmethod
|
||||
def OR(*conditions: Condition) -> Condition:
|
||||
"""多个条件的逻辑或.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
*conditions : Condition
|
||||
条件列表.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
组合条件.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return any(c() for c in conditions)
|
||||
|
||||
names = [c.__name__ for c in conditions]
|
||||
_check.__name__ = f"OR({', '.join(names)})"
|
||||
return _check
|
||||
|
||||
|
||||
# 导出常用条件
|
||||
IS_WINDOWS = BuiltinConditions.IS_WINDOWS
|
||||
IS_LINUX = BuiltinConditions.IS_LINUX
|
||||
IS_MACOS = BuiltinConditions.IS_MACOS
|
||||
IS_POSIX = BuiltinConditions.IS_POSIX
|
||||
|
||||
# 导入 os 用于环境变量检查
|
||||
import os # noqa: E402
|
||||
+18
-16
@@ -18,12 +18,12 @@ DAG 库中泛滥的样板包装器(如 ``def wrapper(): return fn(workflow.get
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from typing import Any, Dict, List, Mapping, Set, Tuple
|
||||
from typing import Any, Mapping
|
||||
|
||||
from .errors import InjectionError
|
||||
from .task import Context, TaskSpec
|
||||
|
||||
__all__ = ["Context", "build_call_args", "describe_injection", "_is_context_annotation"]
|
||||
__all__ = ["Context", "_is_context_annotation", "build_call_args", "describe_injection"]
|
||||
|
||||
|
||||
def _is_context_annotation(annotation: Any) -> bool:
|
||||
@@ -43,15 +43,13 @@ def _is_context_annotation(annotation: Any) -> bool:
|
||||
return annotation == "Context" or annotation.endswith(".Context")
|
||||
# 按限定名匹配,支持 ``from pyflowx import Context`` 再导出。
|
||||
name = getattr(annotation, "__name__", None) or getattr(annotation, "_name", None)
|
||||
if name in ("Context", "Mapping"):
|
||||
return True
|
||||
return False
|
||||
return name in ("Context", "Mapping")
|
||||
|
||||
|
||||
def build_call_args(
|
||||
spec: TaskSpec[object],
|
||||
spec: TaskSpec[Any],
|
||||
context: Mapping[str, Any],
|
||||
) -> Tuple[Tuple[Any, ...], Dict[str, Any]]:
|
||||
) -> tuple[tuple[Any, ...], dict[str, Any]]:
|
||||
"""解析用于调用 ``spec.fn`` 的 ``(args, kwargs)``。
|
||||
|
||||
参数
|
||||
@@ -72,7 +70,9 @@ def build_call_args(
|
||||
InjectionError
|
||||
若必需参数无法满足,或静态 ``kwargs`` 与注入依赖名冲突。
|
||||
"""
|
||||
sig = inspect.signature(spec.fn)
|
||||
# 使用 effective_fn 而不是 fn,以支持 cmd 参数
|
||||
fn = spec.effective_fn
|
||||
sig = inspect.signature(fn)
|
||||
params = sig.parameters
|
||||
|
||||
# 检测特殊参数类型。
|
||||
@@ -82,7 +82,7 @@ def build_call_args(
|
||||
)
|
||||
|
||||
# 与本任务相关的上下文子集。
|
||||
dep_context: Dict[str, Any] = {
|
||||
dep_context: dict[str, Any] = {
|
||||
name: context[name] for name in spec.depends_on if name in context
|
||||
}
|
||||
|
||||
@@ -92,15 +92,15 @@ def build_call_args(
|
||||
raise InjectionError(
|
||||
spec.name,
|
||||
f"static kwargs {sorted(collisions)} collide with dependency names; "
|
||||
"rename the static kwarg or the dependency.",
|
||||
+ "rename the static kwarg or the dependency.",
|
||||
)
|
||||
|
||||
injected_kwargs: Dict[str, Any] = {}
|
||||
leftover_dep_results: Dict[str, Any] = dict(dep_context)
|
||||
injected_kwargs: dict[str, Any] = {}
|
||||
leftover_dep_results: dict[str, Any] = dict(dep_context)
|
||||
|
||||
# 被 spec.args 消费的位置参数。记录哪些参数名已被位置填充,
|
||||
# 以便在基于名称的注入(依赖 / Context / 静态 kwargs)时跳过。
|
||||
positional_params: List[str] = []
|
||||
positional_params: list[str] = []
|
||||
positional_kinds = (
|
||||
inspect.Parameter.POSITIONAL_ONLY,
|
||||
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
||||
@@ -109,7 +109,7 @@ def build_call_args(
|
||||
if param.kind in positional_kinds:
|
||||
positional_params.append(pname)
|
||||
# 前 len(spec.args) 个位置参数由 spec.args 填充。
|
||||
args_filled: Set[str] = set(positional_params[: len(spec.args)])
|
||||
args_filled: set[str] = set(positional_params[: len(spec.args)])
|
||||
|
||||
for pname, param in params.items():
|
||||
# 跳过已被位置 spec.args 填充的参数。
|
||||
@@ -155,12 +155,14 @@ def build_call_args(
|
||||
return tuple(spec.args), injected_kwargs
|
||||
|
||||
|
||||
def describe_injection(spec: TaskSpec[object]) -> str:
|
||||
def describe_injection(spec: TaskSpec[Any]) -> str:
|
||||
"""生成任务参数注入方式的人类可读描述。
|
||||
|
||||
供 ``dry_run`` 使用,在不执行的情况下展示执行计划。
|
||||
"""
|
||||
sig = inspect.signature(spec.fn)
|
||||
# 使用 effective_fn 而不是 fn,以支持 cmd 参数
|
||||
fn = spec.effective_fn
|
||||
sig = inspect.signature(fn)
|
||||
# 确定哪些位置参数由 spec.args 填充。
|
||||
positional_params = [
|
||||
p
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Iterable, Optional
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
class PyFlowXError(Exception):
|
||||
@@ -27,7 +27,7 @@ class MissingDependencyError(PyFlowXError):
|
||||
def __init__(self, task: str, dependency: str) -> None:
|
||||
super().__init__(
|
||||
f"Task '{task}' depends on unknown task '{dependency}'. "
|
||||
"Add the dependency before (or together with) this task."
|
||||
+ "Add the dependency before (or together with) this task."
|
||||
)
|
||||
self.task = task
|
||||
self.dependency = dependency
|
||||
@@ -55,12 +55,10 @@ class TaskFailedError(PyFlowXError):
|
||||
task: str,
|
||||
cause: BaseException,
|
||||
attempts: int,
|
||||
layer: Optional[int] = None,
|
||||
layer: int | None = None,
|
||||
) -> None:
|
||||
location = f" (layer {layer})" if layer is not None else ""
|
||||
super().__init__(
|
||||
f"Task '{task}' failed after {attempts} attempt(s){location}: {cause}"
|
||||
)
|
||||
super().__init__(f"Task '{task}' failed after {attempts} attempt(s){location}: {cause}")
|
||||
self.task = task
|
||||
self.cause = cause
|
||||
self.attempts = attempts
|
||||
@@ -87,6 +85,6 @@ class InjectionError(PyFlowXError):
|
||||
class StorageError(PyFlowXError):
|
||||
"""状态后端在持久化失败时抛出。"""
|
||||
|
||||
def __init__(self, detail: str, cause: Optional[BaseException] = None) -> None:
|
||||
def __init__(self, detail: str, cause: BaseException | None = None) -> None:
|
||||
super().__init__(f"State storage error: {detail}")
|
||||
self.cause: Any = cause
|
||||
|
||||
@@ -10,23 +10,23 @@ Shows:
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
|
||||
async def fetch_user(uid: int) -> dict:
|
||||
async def fetch_user(uid: int) -> dict[str, Any]:
|
||||
await asyncio.sleep(0.2)
|
||||
return {"id": uid, "name": f"User{uid}"}
|
||||
|
||||
|
||||
async def fetch_posts(uid: int) -> List[int]:
|
||||
async def fetch_posts(uid: int) -> list[int]:
|
||||
await asyncio.sleep(0.2)
|
||||
return [uid, uid + 1]
|
||||
|
||||
|
||||
# Context annotation → receives the full mapping of upstream results.
|
||||
def aggregate(ctx: px.Context) -> Dict[str, Any]:
|
||||
def aggregate(ctx: px.Context) -> dict[str, Any]:
|
||||
return dict(ctx)
|
||||
|
||||
|
||||
@@ -36,14 +36,14 @@ def main() -> None:
|
||||
# Static positional args parameterise the same function twice.
|
||||
px.TaskSpec("fetch_user", fetch_user, args=(1,)),
|
||||
px.TaskSpec("fetch_posts", fetch_posts, args=(1,)),
|
||||
px.TaskSpec("aggregate", aggregate, ("fetch_user", "fetch_posts")),
|
||||
px.TaskSpec("aggregate", aggregate, depends_on=("fetch_user", "fetch_posts")),
|
||||
]
|
||||
)
|
||||
|
||||
print("=== Dry run ===")
|
||||
px.run(graph, strategy="async", dry_run=True)
|
||||
_ = px.run(graph, strategy="async", dry_run=True)
|
||||
|
||||
events: List[px.TaskEvent] = []
|
||||
events: list[px.TaskEvent] = []
|
||||
print("\n=== Async execution ===")
|
||||
report = px.run(graph, strategy="async", on_event=events.append)
|
||||
|
||||
|
||||
@@ -10,21 +10,19 @@ Demonstrates the core PyFlowX workflow:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# --- task functions: pure, testable, no framework coupling ------------- #
|
||||
|
||||
|
||||
def extract_customers() -> List[dict]:
|
||||
def extract_customers() -> list[dict]:
|
||||
return [
|
||||
{"id": "C001", "name": "Alice"},
|
||||
{"id": "C002", "name": "Bob"},
|
||||
]
|
||||
|
||||
|
||||
def extract_orders() -> List[dict]:
|
||||
def extract_orders() -> list[dict]:
|
||||
return [
|
||||
{"id": "O001", "customer_id": "C001", "amount": 150.0},
|
||||
{"id": "O002", "customer_id": "C002", "amount": 200.5},
|
||||
@@ -33,18 +31,14 @@ def extract_orders() -> List[dict]:
|
||||
|
||||
# Parameter names match dependency names → automatic injection.
|
||||
def transform(
|
||||
extract_customers: List[dict],
|
||||
extract_orders: List[dict],
|
||||
) -> List[dict]:
|
||||
extract_customers: list[dict],
|
||||
extract_orders: list[dict],
|
||||
) -> list[dict]:
|
||||
cmap = {c["id"]: c for c in extract_customers}
|
||||
return [
|
||||
{**o, "customer_name": cmap[o["customer_id"]]["name"]}
|
||||
for o in extract_orders
|
||||
if o["customer_id"] in cmap
|
||||
]
|
||||
return [{**o, "customer_name": cmap[o["customer_id"]]["name"]} for o in extract_orders if o["customer_id"] in cmap]
|
||||
|
||||
|
||||
def load(transform: List[dict]) -> int:
|
||||
def load(transform: list[dict]) -> int:
|
||||
print(f" loaded {len(transform)} records")
|
||||
return len(transform)
|
||||
|
||||
@@ -57,10 +51,10 @@ def main() -> None:
|
||||
px.TaskSpec(
|
||||
"transform",
|
||||
transform,
|
||||
("extract_customers", "extract_orders"),
|
||||
depends_on=("extract_customers", "extract_orders"),
|
||||
tags=("transform",),
|
||||
),
|
||||
px.TaskSpec("load", load, ("transform",), retries=1, tags=("load",)),
|
||||
px.TaskSpec("load", load, depends_on=("transform",), retries=1, tags=("load",)),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -68,7 +62,7 @@ def main() -> None:
|
||||
print(graph.describe())
|
||||
|
||||
print("\n=== Dry run (no execution) ===")
|
||||
px.run(graph, strategy="sequential", dry_run=True)
|
||||
_ = px.run(graph, strategy="sequential", dry_run=True)
|
||||
|
||||
print("\n=== Sequential execution ===")
|
||||
report = px.run(graph, strategy="sequential")
|
||||
|
||||
@@ -33,7 +33,7 @@ def main() -> None:
|
||||
[
|
||||
px.TaskSpec("fetch_a", fetch_a),
|
||||
px.TaskSpec("fetch_b", fetch_b),
|
||||
px.TaskSpec("merge", merge, ("fetch_a", "fetch_b")),
|
||||
px.TaskSpec("merge", merge, depends_on=("fetch_a", "fetch_b")),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
+142
-93
@@ -19,7 +19,7 @@ import concurrent.futures
|
||||
import inspect
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Mapping, Optional, cast
|
||||
from typing import Any, Awaitable, Callable, Literal, Mapping, cast
|
||||
|
||||
from .context import build_call_args, describe_injection
|
||||
from .errors import TaskFailedError, TaskTimeoutError
|
||||
@@ -32,19 +32,17 @@ logger = logging.getLogger("pyflowx")
|
||||
|
||||
# 观察者回调类型。
|
||||
EventCallback = Callable[[TaskEvent], None]
|
||||
|
||||
# 策略选择字面量。
|
||||
Strategy = str # "sequential" | "thread" | "async"
|
||||
Strategy = Literal["sequential", "thread", "async"]
|
||||
|
||||
|
||||
def _is_async_fn(spec: TaskSpec[object]) -> bool:
|
||||
"""判断 ``spec.fn`` 是否为协程函数。"""
|
||||
return inspect.iscoroutinefunction(spec.fn)
|
||||
def _is_async_fn(spec: TaskSpec[Any]) -> bool:
|
||||
"""判断 ``spec.effective_fn`` 是否为协程函数。"""
|
||||
return inspect.iscoroutinefunction(spec.effective_fn)
|
||||
|
||||
|
||||
def _emit(
|
||||
on_event: Optional[EventCallback],
|
||||
result: TaskResult[object],
|
||||
on_event: EventCallback | None,
|
||||
result: TaskResult[Any],
|
||||
) -> None:
|
||||
"""若注册了回调则触发一个观察者事件。"""
|
||||
if on_event is None:
|
||||
@@ -60,9 +58,7 @@ def _emit(
|
||||
)
|
||||
|
||||
|
||||
def _log_retry(
|
||||
spec: TaskSpec[object], attempts: int, max_attempts: int, exc: BaseException
|
||||
) -> None:
|
||||
def _log_retry(spec: TaskSpec[Any], attempts: int, max_attempts: int, exc: BaseException) -> None:
|
||||
"""记录重试日志(sync 与 async 共享,便于测试覆盖)。"""
|
||||
logger.warning(
|
||||
"task %r failed (attempt %d/%d): %r; retrying",
|
||||
@@ -73,10 +69,15 @@ def _log_retry(
|
||||
)
|
||||
|
||||
|
||||
def _finalize_failure(result: TaskResult[object], layer_idx: Optional[int]) -> None:
|
||||
def _finalize_failure(
|
||||
result: TaskResult[Any],
|
||||
layer_idx: int | None,
|
||||
on_event: EventCallback | None = None,
|
||||
) -> None:
|
||||
"""标记任务为 FAILED 并抛出 TaskFailedError。"""
|
||||
result.status = TaskStatus.FAILED
|
||||
result.finished_at = datetime.now()
|
||||
_emit(on_event, result)
|
||||
raise TaskFailedError(
|
||||
task=result.spec.name,
|
||||
cause=result.error if result.error is not None else RuntimeError("unknown"),
|
||||
@@ -86,12 +87,21 @@ def _finalize_failure(result: TaskResult[object], layer_idx: Optional[int]) -> N
|
||||
|
||||
|
||||
def _run_sync_with_retry(
|
||||
spec: TaskSpec[object],
|
||||
spec: TaskSpec[Any],
|
||||
context: Mapping[str, Any],
|
||||
layer_idx: Optional[int],
|
||||
) -> TaskResult[object]:
|
||||
layer_idx: int | None,
|
||||
on_event: EventCallback | None = None,
|
||||
) -> TaskResult[Any]:
|
||||
"""执行同步任务并带重试;返回填充好的 TaskResult。"""
|
||||
result: TaskResult[object] = TaskResult(spec=spec)
|
||||
result: TaskResult[Any] = TaskResult(spec=spec)
|
||||
|
||||
# 检查条件是否满足
|
||||
if spec.conditions and not spec.should_execute():
|
||||
result.status = TaskStatus.SKIPPED
|
||||
result.finished_at = datetime.now()
|
||||
logger.info("task %r skipped (条件不满足)", spec.name)
|
||||
return result
|
||||
|
||||
result.started_at = datetime.now()
|
||||
max_attempts = spec.retries + 1
|
||||
args, kwargs = build_call_args(spec, context)
|
||||
@@ -99,25 +109,34 @@ def _run_sync_with_retry(
|
||||
while True:
|
||||
result.attempts += 1
|
||||
try:
|
||||
result.value = spec.fn(*args, **kwargs)
|
||||
result.value = spec.effective_fn(*args, **kwargs)
|
||||
result.status = TaskStatus.SUCCESS
|
||||
result.finished_at = datetime.now()
|
||||
return result
|
||||
except Exception as exc: # noqa: BLE001 - 用户代码可能抛任何异常
|
||||
except Exception as exc:
|
||||
result.error = exc
|
||||
if result.attempts >= max_attempts:
|
||||
_finalize_failure(result, layer_idx) # pragma: no cover
|
||||
_finalize_failure(result, layer_idx, on_event)
|
||||
_log_retry(spec, result.attempts, max_attempts, exc)
|
||||
raise AssertionError("unreachable") # pragma: no cover
|
||||
|
||||
|
||||
async def _run_async_with_retry(
|
||||
spec: TaskSpec[object],
|
||||
spec: TaskSpec[Any],
|
||||
context: Mapping[str, Any],
|
||||
layer_idx: Optional[int],
|
||||
) -> TaskResult[object]:
|
||||
layer_idx: int | None,
|
||||
on_event: EventCallback | None = None,
|
||||
) -> TaskResult[Any]:
|
||||
"""在事件循环上执行任务(同步或异步)并带重试。"""
|
||||
result: TaskResult[object] = TaskResult(spec=spec)
|
||||
result: TaskResult[Any] = TaskResult[Any](spec=spec)
|
||||
|
||||
# 检查条件是否满足
|
||||
if spec.conditions and not spec.should_execute():
|
||||
result.status = TaskStatus.SKIPPED
|
||||
result.finished_at = datetime.now()
|
||||
logger.info("task %r skipped (条件不满足)", spec.name)
|
||||
return result
|
||||
|
||||
result.started_at = datetime.now()
|
||||
max_attempts = spec.retries + 1
|
||||
args, kwargs = build_call_args(spec, context)
|
||||
@@ -127,7 +146,7 @@ async def _run_async_with_retry(
|
||||
result.attempts += 1
|
||||
try:
|
||||
if _is_async_fn(spec):
|
||||
coro = cast(Awaitable[Any], spec.fn(*args, **kwargs))
|
||||
coro = cast(Awaitable[Any], spec.effective_fn(*args, **kwargs))
|
||||
if spec.timeout is not None:
|
||||
result.value = await asyncio.wait_for(coro, timeout=spec.timeout)
|
||||
else:
|
||||
@@ -135,12 +154,10 @@ async def _run_async_with_retry(
|
||||
else:
|
||||
# 将同步工作卸载到线程,保持事件循环存活。
|
||||
def fn_call() -> Any:
|
||||
return spec.fn(*args, **kwargs)
|
||||
return spec.effective_fn(*args, **kwargs)
|
||||
|
||||
if spec.timeout is not None:
|
||||
result.value = await asyncio.wait_for(
|
||||
loop.run_in_executor(None, fn_call), timeout=spec.timeout
|
||||
)
|
||||
result.value = await asyncio.wait_for(loop.run_in_executor(None, fn_call), timeout=spec.timeout)
|
||||
else:
|
||||
result.value = await loop.run_in_executor(None, fn_call)
|
||||
result.status = TaskStatus.SUCCESS
|
||||
@@ -149,18 +166,18 @@ async def _run_async_with_retry(
|
||||
except asyncio.TimeoutError:
|
||||
result.error = TaskTimeoutError(spec.name, spec.timeout or 0.0)
|
||||
if result.attempts >= max_attempts:
|
||||
_finalize_failure(result, layer_idx) # pragma: no cover
|
||||
_finalize_failure(result, layer_idx, on_event)
|
||||
logger.warning(
|
||||
"task %r timed out (attempt %d/%d); retrying",
|
||||
spec.name,
|
||||
result.attempts,
|
||||
max_attempts,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
except Exception as exc:
|
||||
result.error = exc
|
||||
if result.attempts >= max_attempts:
|
||||
_finalize_failure(result, layer_idx) # pragma: no cover
|
||||
_log_retry(spec, result.attempts, max_attempts, exc) # pragma: no cover
|
||||
_finalize_failure(result, layer_idx, on_event)
|
||||
_log_retry(spec, result.attempts, max_attempts, exc)
|
||||
raise AssertionError("unreachable") # pragma: no cover
|
||||
|
||||
|
||||
@@ -168,23 +185,21 @@ async def _run_async_with_retry(
|
||||
# 层驱动器
|
||||
# ---------------------------------------------------------------------- #
|
||||
def _build_context(
|
||||
spec: TaskSpec[object],
|
||||
spec: TaskSpec[Any],
|
||||
global_context: Mapping[str, Any],
|
||||
) -> Mapping[str, Any]:
|
||||
"""将全局上下文限制为本任务的依赖。"""
|
||||
return {
|
||||
dep: global_context[dep] for dep in spec.depends_on if dep in global_context
|
||||
}
|
||||
return {dep: global_context[dep] for dep in spec.depends_on if dep in global_context}
|
||||
|
||||
|
||||
def _execute_layer_sequential(
|
||||
layer: List[str],
|
||||
layer: list[str],
|
||||
graph: Graph,
|
||||
context: Dict[str, Any],
|
||||
context: dict[str, Any],
|
||||
report: RunReport,
|
||||
backend: StateBackend,
|
||||
layer_idx: int,
|
||||
on_event: Optional[EventCallback],
|
||||
on_event: EventCallback | None,
|
||||
) -> None:
|
||||
"""逐个运行某层的任务。"""
|
||||
for name in layer:
|
||||
@@ -197,7 +212,7 @@ def _execute_layer_sequential(
|
||||
_emit(on_event, result)
|
||||
logger.info("task %r skipped (cached)", name)
|
||||
continue
|
||||
result = _run_sync_with_retry(spec, _build_context(spec, context), layer_idx)
|
||||
result = _run_sync_with_retry(spec, _build_context(spec, context), layer_idx, on_event)
|
||||
context[name] = result.value
|
||||
backend.save(name, result.value)
|
||||
report.results[name] = result
|
||||
@@ -205,25 +220,23 @@ def _execute_layer_sequential(
|
||||
|
||||
|
||||
def _execute_layer_threaded(
|
||||
layer: List[str],
|
||||
layer: list[str],
|
||||
graph: Graph,
|
||||
context: Dict[str, Any],
|
||||
context: dict[str, Any],
|
||||
report: RunReport,
|
||||
backend: StateBackend,
|
||||
layer_idx: int,
|
||||
on_event: Optional[EventCallback],
|
||||
on_event: EventCallback | None,
|
||||
max_workers: int,
|
||||
) -> None:
|
||||
"""在线程池中并发运行某层的任务。"""
|
||||
# 先同步满足已缓存任务。
|
||||
to_run: List[str] = []
|
||||
to_run: list[str] = []
|
||||
for name in layer:
|
||||
if backend.has(name):
|
||||
cached = backend.get(name)
|
||||
context[name] = cached
|
||||
result = TaskResult(
|
||||
spec=graph.spec(name), status=TaskStatus.SKIPPED, value=cached
|
||||
)
|
||||
result = TaskResult(spec=graph.spec(name), status=TaskStatus.SKIPPED, value=cached)
|
||||
report.results[name] = result
|
||||
_emit(on_event, result)
|
||||
else:
|
||||
@@ -233,12 +246,12 @@ def _execute_layer_threaded(
|
||||
return
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||
future_to_name: Dict[concurrent.futures.Future[TaskResult[object]], str] = {}
|
||||
future_to_name: dict[concurrent.futures.Future[TaskResult[Any]], str] = {}
|
||||
for name in to_run:
|
||||
spec = graph.spec(name)
|
||||
# 为本任务快照上下文以避免竞态。
|
||||
task_ctx = _build_context(spec, context)
|
||||
fut = pool.submit(_run_sync_with_retry, spec, task_ctx, layer_idx)
|
||||
fut = pool.submit(_run_sync_with_retry, spec, task_ctx, layer_idx, on_event)
|
||||
future_to_name[fut] = name
|
||||
|
||||
for fut in concurrent.futures.as_completed(future_to_name):
|
||||
@@ -251,23 +264,21 @@ def _execute_layer_threaded(
|
||||
|
||||
|
||||
async def _execute_layer_async(
|
||||
layer: List[str],
|
||||
layer: list[str],
|
||||
graph: Graph,
|
||||
context: Dict[str, Any],
|
||||
context: dict[str, Any],
|
||||
report: RunReport,
|
||||
backend: StateBackend,
|
||||
layer_idx: int,
|
||||
on_event: Optional[EventCallback],
|
||||
on_event: EventCallback | None,
|
||||
) -> None:
|
||||
"""在事件循环上并发运行某层的任务。"""
|
||||
to_run: List[str] = []
|
||||
to_run: list[str] = []
|
||||
for name in layer:
|
||||
if backend.has(name):
|
||||
cached = backend.get(name)
|
||||
context[name] = cached
|
||||
result = TaskResult(
|
||||
spec=graph.spec(name), status=TaskStatus.SKIPPED, value=cached
|
||||
)
|
||||
result = TaskResult(spec=graph.spec(name), status=TaskStatus.SKIPPED, value=cached)
|
||||
report.results[name] = result
|
||||
_emit(on_event, result)
|
||||
else:
|
||||
@@ -280,7 +291,7 @@ async def _execute_layer_async(
|
||||
for name in to_run:
|
||||
spec = graph.spec(name)
|
||||
task_ctx = _build_context(spec, context)
|
||||
coros.append(_run_async_with_retry(spec, task_ctx, layer_idx))
|
||||
coros.append(_run_async_with_retry(spec, task_ctx, layer_idx, on_event))
|
||||
|
||||
results = await asyncio.gather(*coros)
|
||||
for name, result in zip(to_run, results):
|
||||
@@ -293,14 +304,56 @@ async def _execute_layer_async(
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 公共 API
|
||||
# ---------------------------------------------------------------------- #
|
||||
def _make_verbose_callback(
|
||||
on_event: EventCallback | None,
|
||||
) -> EventCallback | None:
|
||||
"""包装 on_event 回调, 在 verbose 模式下打印任务生命周期.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
on_event : EventCallback | None
|
||||
用户提供的原始回调, 若为 None 则仅打印.
|
||||
|
||||
Returns
|
||||
-------
|
||||
EventCallback | None
|
||||
包装后的回调.
|
||||
"""
|
||||
|
||||
def _verbose_callback(event: TaskEvent) -> None:
|
||||
# 先打印生命周期信息
|
||||
dur = f" ({event.duration:.3f}s)" if event.duration is not None else ""
|
||||
if event.status == TaskStatus.RUNNING: # pragma: no cover
|
||||
print(f"[verbose] 任务 {event.task!r} 开始执行...", flush=True)
|
||||
elif event.status == TaskStatus.SUCCESS:
|
||||
print(f"[verbose] 任务 {event.task!r} 成功{dur}", flush=True)
|
||||
elif event.status == TaskStatus.FAILED:
|
||||
err = f": {event.error}" if event.error else ""
|
||||
print(
|
||||
f"[verbose] 任务 {event.task!r} 失败{dur} (尝试 {event.attempts} 次){err}",
|
||||
flush=True,
|
||||
)
|
||||
elif event.status == TaskStatus.SKIPPED: # pragma: no branch
|
||||
print(f"[verbose] 任务 {event.task!r} 跳过", flush=True)
|
||||
else: # pragma: no cover
|
||||
# 不可达: 执行器只发出 RUNNING/SUCCESS/FAILED/SKIPPED 事件
|
||||
pass
|
||||
# 再调用用户回调
|
||||
if on_event is not None:
|
||||
on_event(event)
|
||||
|
||||
return _verbose_callback
|
||||
|
||||
|
||||
def run(
|
||||
graph: Graph,
|
||||
strategy: Strategy = "sequential",
|
||||
*,
|
||||
max_workers: Optional[int] = None,
|
||||
max_workers: int | None = None,
|
||||
dry_run: bool = False,
|
||||
on_event: Optional[EventCallback] = None,
|
||||
state: Optional[StateBackend] = None,
|
||||
verbose: bool = False,
|
||||
on_event: EventCallback | None = None,
|
||||
state: StateBackend | None = None,
|
||||
) -> RunReport:
|
||||
"""执行图并返回 :class:`RunReport`。
|
||||
|
||||
@@ -309,12 +362,16 @@ def run(
|
||||
graph:
|
||||
待执行的已校验 :class:`Graph`。
|
||||
strategy:
|
||||
``"sequential"``(默认)、``"thread"`` 或 ``"async"``。
|
||||
执行策略, 接受 :class:`Strategy` 枚举成员或字符串
|
||||
(``"sequential"`` / ``"thread"`` / ``"async"``). 默认 ``Strategy.SEQUENTIAL``.
|
||||
max_workers:
|
||||
``"thread"`` 的线程池大小。默认 ``min(32, len(layer))``。
|
||||
dry_run:
|
||||
若为 ``True``,打印执行计划(层 + 注入)并返回空报告,不执行
|
||||
任何任务。
|
||||
verbose:
|
||||
若为 ``True``, 打印任务生命周期 (开始/成功/失败/跳过) 到 stdout.
|
||||
注意: subprocess 命令的输出由 :class:`TaskSpec` 的 ``verbose`` 字段控制.
|
||||
on_event:
|
||||
可选回调,在每次状态转换时调用。
|
||||
state:
|
||||
@@ -329,11 +386,6 @@ def run(
|
||||
任何任务耗尽重试后仍失败时。运行在失败层中止;后续层的任务
|
||||
不会被执行。
|
||||
"""
|
||||
if strategy not in ("sequential", "thread", "async"):
|
||||
raise ValueError(
|
||||
f"unknown strategy {strategy!r}; expected 'sequential', 'thread', or 'async'."
|
||||
)
|
||||
|
||||
graph.validate()
|
||||
layers = graph.layers()
|
||||
|
||||
@@ -341,19 +393,20 @@ def run(
|
||||
_print_dry_run(graph, layers)
|
||||
return RunReport(success=True)
|
||||
|
||||
# verbose 模式下包装事件回调
|
||||
effective_callback: EventCallback | None = _make_verbose_callback(on_event) if verbose else on_event
|
||||
|
||||
backend = resolve_backend(state)
|
||||
report = RunReport()
|
||||
context: Dict[str, Any] = {}
|
||||
context: dict[str, Any] = {}
|
||||
|
||||
try:
|
||||
if strategy == "sequential":
|
||||
_drive_sequential(graph, layers, context, report, backend, on_event)
|
||||
_drive_sequential(graph, layers, context, report, backend, effective_callback)
|
||||
elif strategy == "thread":
|
||||
_drive_threaded(
|
||||
graph, layers, context, report, backend, on_event, max_workers
|
||||
)
|
||||
_drive_threaded(graph, layers, context, report, backend, effective_callback, max_workers)
|
||||
else:
|
||||
_drive_async(graph, layers, context, report, backend, on_event)
|
||||
_drive_async(graph, layers, context, report, backend, effective_callback)
|
||||
except TaskFailedError:
|
||||
report.success = False
|
||||
raise
|
||||
@@ -361,7 +414,7 @@ def run(
|
||||
return report
|
||||
|
||||
|
||||
def _print_dry_run(graph: Graph, layers: List[List[str]]) -> None:
|
||||
def _print_dry_run(graph: Graph, layers: list[list[str]]) -> None:
|
||||
"""打印执行计划但不运行任何任务。"""
|
||||
print(f"Dry run: {len(graph)} tasks, {len(layers)} layers")
|
||||
for idx, layer in enumerate(layers, 1):
|
||||
@@ -372,11 +425,11 @@ def _print_dry_run(graph: Graph, layers: List[List[str]]) -> None:
|
||||
|
||||
def _drive_sequential(
|
||||
graph: Graph,
|
||||
layers: List[List[str]],
|
||||
context: Dict[str, Any],
|
||||
layers: list[list[str]],
|
||||
context: dict[str, Any],
|
||||
report: RunReport,
|
||||
backend: StateBackend,
|
||||
on_event: Optional[EventCallback],
|
||||
on_event: EventCallback | None,
|
||||
) -> None:
|
||||
for idx, layer in enumerate(layers, 1):
|
||||
_execute_layer_sequential(layer, graph, context, report, backend, idx, on_event)
|
||||
@@ -384,40 +437,36 @@ def _drive_sequential(
|
||||
|
||||
def _drive_threaded(
|
||||
graph: Graph,
|
||||
layers: List[List[str]],
|
||||
context: Dict[str, Any],
|
||||
layers: list[list[str]],
|
||||
context: dict[str, Any],
|
||||
report: RunReport,
|
||||
backend: StateBackend,
|
||||
on_event: Optional[EventCallback],
|
||||
max_workers: Optional[int],
|
||||
on_event: EventCallback | None,
|
||||
max_workers: int | None,
|
||||
) -> None:
|
||||
for idx, layer in enumerate(layers, 1):
|
||||
workers = max_workers or max(1, min(32, len(layer)))
|
||||
_execute_layer_threaded(
|
||||
layer, graph, context, report, backend, idx, on_event, workers
|
||||
)
|
||||
_execute_layer_threaded(layer, graph, context, report, backend, idx, on_event, workers)
|
||||
|
||||
|
||||
def _drive_async(
|
||||
graph: Graph,
|
||||
layers: List[List[str]],
|
||||
context: Dict[str, Any],
|
||||
layers: list[list[str]],
|
||||
context: dict[str, Any],
|
||||
report: RunReport,
|
||||
backend: StateBackend,
|
||||
on_event: Optional[EventCallback],
|
||||
on_event: EventCallback | None,
|
||||
) -> None:
|
||||
asyncio.run(_async_drive(graph, layers, context, report, backend, on_event))
|
||||
|
||||
|
||||
async def _async_drive(
|
||||
graph: Graph,
|
||||
layers: List[List[str]],
|
||||
context: Dict[str, Any],
|
||||
layers: list[list[str]],
|
||||
context: dict[str, Any],
|
||||
report: RunReport,
|
||||
backend: StateBackend,
|
||||
on_event: Optional[EventCallback],
|
||||
on_event: EventCallback | None,
|
||||
) -> None:
|
||||
for idx, layer in enumerate(layers, 1):
|
||||
await _execute_layer_async(
|
||||
layer, graph, context, report, backend, idx, on_event
|
||||
)
|
||||
await _execute_layer_async(layer, graph, context, report, backend, idx, on_event)
|
||||
|
||||
+56
-55
@@ -1,21 +1,21 @@
|
||||
"""DAG 构建、校验、分层与可视化。
|
||||
|
||||
使用标准库的 :mod:`graphlib`(3.9+)或 :mod:`graphlib_backport`(3.8)
|
||||
进行拓扑排序。图以增量方式构建并即时校验,使配置错误在构建时(而非
|
||||
执行时)快速失败。
|
||||
进行拓扑排序。图以增量方式构建并即时校验,使配置错误在构建时(而非执行时)快速失败。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from typing import Dict, Iterable, List, Mapping, Sequence, Set, Tuple
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterable, Mapping, Sequence
|
||||
|
||||
from .errors import CycleError, DuplicateTaskError, MissingDependencyError
|
||||
from .task import TaskSpec
|
||||
|
||||
# graphlib 自 3.9 起进入标准库;3.8 回退到 backport。
|
||||
if sys.version_info >= (3, 9): # pragma: no cover
|
||||
import graphlib
|
||||
import graphlib # pyright: ignore[reportUnreachable]
|
||||
|
||||
_TopologicalSorter = graphlib.TopologicalSorter
|
||||
else: # pragma: no cover
|
||||
@@ -24,6 +24,7 @@ else: # pragma: no cover
|
||||
_TopologicalSorter = graphlib.TopologicalSorter # pragma: no cover
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Graph:
|
||||
"""校验后不可变的有向无环任务图。
|
||||
|
||||
@@ -35,30 +36,28 @@ class Graph:
|
||||
这使图可安全重复运行并在线程间共享。
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._specs: Dict[str, TaskSpec[object]] = {}
|
||||
# 任务 -> 其直接依赖(前驱)。
|
||||
self._deps: Dict[str, Tuple[str, ...]] = {}
|
||||
specs: dict[str, TaskSpec[Any]] = field(default_factory=dict)
|
||||
deps: dict[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 构建
|
||||
# ------------------------------------------------------------------ #
|
||||
def add(self, spec: TaskSpec[object]) -> "Graph":
|
||||
def add(self, spec: TaskSpec[Any]) -> Graph:
|
||||
"""注册一个任务 spec,并即时校验。
|
||||
|
||||
返回 ``self`` 以支持链式调用,但推荐入口是 :meth:`from_specs`,
|
||||
它会整批校验(允许单次调用中的前向引用)。
|
||||
"""
|
||||
if spec.name in self._specs:
|
||||
if spec.name in self.specs:
|
||||
raise DuplicateTaskError(spec.name)
|
||||
self._specs[spec.name] = spec
|
||||
self._deps[spec.name] = spec.depends_on
|
||||
self.specs[spec.name] = spec
|
||||
self.deps[spec.name] = spec.depends_on
|
||||
# 为增量 API 即时检查重名与缺失依赖。
|
||||
self._validate_references()
|
||||
return self
|
||||
|
||||
@classmethod
|
||||
def from_specs(cls, specs: Iterable[TaskSpec[object]]) -> "Graph":
|
||||
def from_specs(cls, specs: Iterable[TaskSpec[Any]]) -> Graph:
|
||||
"""从可迭代的 task spec 构建图。
|
||||
|
||||
先收集所有 spec,再统一校验。这意味着任务可以引用*后出现*的
|
||||
@@ -66,10 +65,10 @@ class Graph:
|
||||
"""
|
||||
graph = cls()
|
||||
for spec in specs:
|
||||
if spec.name in graph._specs:
|
||||
if spec.name in graph.specs:
|
||||
raise DuplicateTaskError(spec.name)
|
||||
graph._specs[spec.name] = spec
|
||||
graph._deps[spec.name] = spec.depends_on
|
||||
graph.specs[spec.name] = spec
|
||||
graph.deps[spec.name] = spec.depends_on
|
||||
graph._validate_references()
|
||||
graph.validate()
|
||||
return graph
|
||||
@@ -79,9 +78,9 @@ class Graph:
|
||||
# ------------------------------------------------------------------ #
|
||||
def _validate_references(self) -> None:
|
||||
"""确保每个依赖名都存在于图中。"""
|
||||
for name, deps in self._deps.items():
|
||||
for name, deps in self.deps.items():
|
||||
for dep in deps:
|
||||
if dep not in self._specs:
|
||||
if dep not in self.specs:
|
||||
raise MissingDependencyError(name, dep)
|
||||
|
||||
def validate(self) -> None:
|
||||
@@ -91,7 +90,7 @@ class Graph:
|
||||
依赖存在性由 :meth:`_validate_references` 检查。
|
||||
"""
|
||||
self._validate_references()
|
||||
sorter = _TopologicalSorter(self._deps)
|
||||
sorter = _TopologicalSorter(self.deps)
|
||||
try:
|
||||
# prepare() 在有环时抛出 CycleError;此处不需要
|
||||
# static_order() 的结果,仅利用其校验副作用。
|
||||
@@ -105,23 +104,23 @@ class Graph:
|
||||
# 内省
|
||||
# ------------------------------------------------------------------ #
|
||||
@property
|
||||
def names(self) -> List[str]:
|
||||
def names(self) -> list[str]:
|
||||
"""所有已注册任务名(按插入顺序)。"""
|
||||
return list(self._specs.keys())
|
||||
return list(self.specs.keys())
|
||||
|
||||
def spec(self, name: str) -> TaskSpec[object]:
|
||||
def spec(self, name: str) -> TaskSpec[Any]:
|
||||
"""返回 ``name`` 的 spec;不存在则 ``KeyError``。"""
|
||||
return self._specs[name]
|
||||
return self.specs[name]
|
||||
|
||||
def dependencies(self, name: str) -> Tuple[str, ...]:
|
||||
def dependencies(self, name: str) -> tuple[str, ...]:
|
||||
"""``name`` 的直接前驱。"""
|
||||
return self._deps[name]
|
||||
return self.deps[name]
|
||||
|
||||
def all_specs(self) -> Mapping[str, TaskSpec[object]]:
|
||||
def all_specs(self) -> Mapping[str, TaskSpec[Any]]:
|
||||
"""name -> spec 的只读视图。"""
|
||||
return self._specs
|
||||
return self.specs
|
||||
|
||||
def layers(self) -> List[List[str]]:
|
||||
def layers(self) -> list[list[str]]:
|
||||
"""将任务分组为可并行执行的层(Kahn 算法)。
|
||||
|
||||
同层任务无相互依赖,可并发执行。层按执行顺序返回。
|
||||
@@ -129,8 +128,8 @@ class Graph:
|
||||
图有环时抛出 :class:`~pyflowx.errors.CycleError`。
|
||||
"""
|
||||
self.validate()
|
||||
sorter = _TopologicalSorter(self._deps)
|
||||
result: List[List[str]] = []
|
||||
sorter = _TopologicalSorter(self.deps)
|
||||
result: list[list[str]] = []
|
||||
# ``get_ready`` + ``done`` 每次给出一层,正好是并行执行所需的分组。
|
||||
sorter.prepare()
|
||||
while sorter.is_active():
|
||||
@@ -145,56 +144,60 @@ class Graph:
|
||||
# ------------------------------------------------------------------ #
|
||||
# 子图 / 标签过滤
|
||||
# ------------------------------------------------------------------ #
|
||||
def subgraph(self, tags: Iterable[str]) -> "Graph":
|
||||
def subgraph(self, tags: Iterable[str]) -> Graph:
|
||||
"""返回仅包含匹配任意标签的任务的新图。
|
||||
|
||||
依赖会被修剪,仅保留被保留任务之间的边;指向被丢弃任务的边
|
||||
会被移除(被保留的任务不再等待它们)。用于调试时运行大型
|
||||
DAG 的切片。
|
||||
"""
|
||||
wanted: Set[str] = set(tags)
|
||||
kept: List[TaskSpec[object]] = []
|
||||
for spec in self._specs.values():
|
||||
wanted: set[str] = set(tags)
|
||||
kept: list[TaskSpec[Any]] = []
|
||||
for spec in self.specs.values():
|
||||
if wanted & set(spec.tags):
|
||||
pruned_deps = tuple(
|
||||
d
|
||||
for d in spec.depends_on
|
||||
if d in self._specs and (wanted & set(self._specs[d].tags))
|
||||
d for d in spec.depends_on if d in self.specs and (wanted & set(self.specs[d].tags))
|
||||
)
|
||||
kept.append(
|
||||
TaskSpec(
|
||||
TaskSpec[Any](
|
||||
name=spec.name,
|
||||
fn=spec.fn,
|
||||
cmd=spec.cmd,
|
||||
depends_on=pruned_deps,
|
||||
args=spec.args,
|
||||
kwargs=spec.kwargs,
|
||||
retries=spec.retries,
|
||||
timeout=spec.timeout,
|
||||
tags=spec.tags,
|
||||
conditions=spec.conditions,
|
||||
cwd=spec.cwd,
|
||||
)
|
||||
)
|
||||
return Graph.from_specs(kept)
|
||||
|
||||
def subgraph_by_names(self, names: Iterable[str]) -> "Graph":
|
||||
def subgraph_by_names(self, names: Iterable[str]) -> Graph:
|
||||
"""返回限定于 ``names`` 的新图(边已修剪)。"""
|
||||
wanted: Set[str] = set(names)
|
||||
wanted: set[str] = set(names)
|
||||
for n in wanted:
|
||||
if n not in self._specs:
|
||||
if n not in self.specs:
|
||||
raise KeyError(f"Unknown task name: {n!r}")
|
||||
kept: List[TaskSpec[object]] = []
|
||||
for spec in self._specs.values():
|
||||
kept: list[TaskSpec[Any]] = []
|
||||
for spec in self.specs.values():
|
||||
if spec.name in wanted:
|
||||
pruned_deps = tuple(d for d in spec.depends_on if d in wanted)
|
||||
kept.append(
|
||||
TaskSpec(
|
||||
TaskSpec[Any](
|
||||
name=spec.name,
|
||||
fn=spec.fn,
|
||||
cmd=spec.cmd,
|
||||
depends_on=pruned_deps,
|
||||
args=spec.args,
|
||||
kwargs=spec.kwargs,
|
||||
retries=spec.retries,
|
||||
timeout=spec.timeout,
|
||||
tags=spec.tags,
|
||||
conditions=spec.conditions,
|
||||
cwd=spec.cwd,
|
||||
)
|
||||
)
|
||||
return Graph.from_specs(kept)
|
||||
@@ -211,13 +214,11 @@ class Graph:
|
||||
valid = {"TD", "TB", "BT", "LR", "RL"}
|
||||
orientation = orientation.upper()
|
||||
if orientation not in valid:
|
||||
raise ValueError(
|
||||
f"Invalid orientation {orientation!r}; expected one of {sorted(valid)}."
|
||||
)
|
||||
lines: List[str] = [f"graph {orientation}"]
|
||||
for name in self._specs:
|
||||
raise ValueError(f"Invalid orientation {orientation!r}; expected one of {sorted(valid)}.")
|
||||
lines: list[str] = [f"graph {orientation}"]
|
||||
for name in self.specs:
|
||||
lines.append(f' {name}["{name}"]')
|
||||
for name, deps in self._deps.items():
|
||||
for name, deps in self.deps.items():
|
||||
for dep in deps:
|
||||
lines.append(f" {dep} --> {name}")
|
||||
return "\n".join(lines) + "\n"
|
||||
@@ -227,16 +228,16 @@ class Graph:
|
||||
# ------------------------------------------------------------------ #
|
||||
def describe(self) -> str:
|
||||
"""用于调试的人类可读多行摘要。"""
|
||||
out: List[str] = [f"Graph(tasks={len(self._specs)})"]
|
||||
out: list[str] = [f"Graph(tasks={len(self.specs)})"]
|
||||
for layer_idx, layer in enumerate(self.layers(), 1):
|
||||
out.append(f" Layer {layer_idx}: {layer}")
|
||||
return "\n".join(out)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Graph(tasks={len(self._specs)})"
|
||||
return f"Graph(tasks={len(self.specs)})"
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._specs)
|
||||
return len(self.specs)
|
||||
|
||||
def __contains__(self, name: object) -> bool:
|
||||
return name in self._specs
|
||||
def __contains__(self, name: Any) -> bool:
|
||||
return name in self.specs
|
||||
|
||||
+10
-14
@@ -7,7 +7,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, Iterator, List
|
||||
from typing import Any, Iterator
|
||||
|
||||
from .task import TaskResult, TaskStatus
|
||||
|
||||
@@ -24,7 +24,7 @@ class RunReport:
|
||||
当且仅当所有非跳过任务都以 ``SUCCESS`` 结束时为 ``True``。
|
||||
"""
|
||||
|
||||
results: Dict[str, TaskResult[object]] = field(default_factory=dict)
|
||||
results: dict[str, TaskResult[Any]] = field(default_factory=dict)
|
||||
success: bool = True
|
||||
|
||||
# ---- 类型化访问 --------------------------------------------------- #
|
||||
@@ -36,11 +36,11 @@ class RunReport:
|
||||
"""
|
||||
return self.results[name].value
|
||||
|
||||
def result_of(self, name: str) -> TaskResult[object]:
|
||||
def result_of(self, name: str) -> TaskResult[Any]:
|
||||
"""返回 ``name`` 的完整 :class:`TaskResult`。"""
|
||||
return self.results[name]
|
||||
|
||||
def __contains__(self, name: object) -> bool:
|
||||
def __contains__(self, name: Any) -> bool:
|
||||
return name in self.results
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
@@ -50,9 +50,9 @@ class RunReport:
|
||||
return len(self.results)
|
||||
|
||||
# ---- 汇总 --------------------------------------------------------- #
|
||||
def summary(self) -> Dict[str, Any]:
|
||||
def summary(self) -> dict[str, Any]:
|
||||
"""用于日志/仪表盘的紧凑统计字典。"""
|
||||
counts: Dict[str, int] = {}
|
||||
counts: dict[str, int] = {}
|
||||
total_duration = 0.0
|
||||
for r in self.results.values():
|
||||
counts[r.status.value] = counts.get(r.status.value, 0) + 1
|
||||
@@ -65,19 +65,15 @@ class RunReport:
|
||||
"total_duration_seconds": round(total_duration, 6),
|
||||
}
|
||||
|
||||
def failed_tasks(self) -> List[str]:
|
||||
def failed_tasks(self) -> list[str]:
|
||||
"""以 FAILED 状态结束的任务名列表。"""
|
||||
return [
|
||||
name for name, r in self.results.items() if r.status == TaskStatus.FAILED
|
||||
]
|
||||
return [name for name, r in self.results.items() if r.status == TaskStatus.FAILED]
|
||||
|
||||
def describe(self) -> str:
|
||||
"""用于调试的人类可读多行报告。"""
|
||||
lines: List[str] = [f"RunReport(success={self.success})"]
|
||||
lines: list[str] = [f"RunReport(success={self.success})"]
|
||||
for name, r in self.results.items():
|
||||
dur = f"{r.duration:.3f}s" if r.duration is not None else "-"
|
||||
err = f" error={r.error!r}" if r.error else ""
|
||||
lines.append(
|
||||
f" {name}: {r.status.value} ({dur} attempts={r.attempts}){err}"
|
||||
)
|
||||
lines.append(f" {name}: {r.status.value} ({dur} attempts={r.attempts}){err}")
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
"""命令行运行器:根据用户输入执行对应的任务流图.
|
||||
|
||||
verbose 模式
|
||||
------------
|
||||
``CliRunner`` 默认 ``verbose=True``, 会:
|
||||
1. 打印任务生命周期 (开始/成功/失败/跳过) 到 stdout
|
||||
2. 对 ``cmd`` 类任务, 显示执行的命令及其标准输出/标准错误
|
||||
|
||||
可通过构造参数 ``verbose=False`` 或命令行 ``--quiet`` 关闭.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import enum
|
||||
import sys
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import Any, Sequence, get_args
|
||||
|
||||
from .errors import PyFlowXError
|
||||
from .executors import Strategy, run
|
||||
from .graph import Graph
|
||||
from .task import TaskSpec
|
||||
|
||||
__all__ = ["CliExitCode", "CliRunner"]
|
||||
|
||||
|
||||
class CliExitCode(enum.IntEnum):
|
||||
"""CliRunner 退出码."""
|
||||
|
||||
SUCCESS = 0
|
||||
FAILURE = 1
|
||||
INTERRUPTED = 130 # 与 POSIX 信号中断一致
|
||||
|
||||
|
||||
def _apply_verbose_to_graph(graph: Graph, verbose: bool) -> Graph:
|
||||
"""创建新图, 其中所有 TaskSpec 的 verbose 字段被设置为指定值.
|
||||
|
||||
使用 ``dataclasses.replace`` 在不可变的 TaskSpec 上创建带 verbose 标记的副本.
|
||||
依赖关系、标签等元数据全部保留.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph : Graph
|
||||
原始图.
|
||||
verbose : bool
|
||||
要设置的 verbose 值.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Graph
|
||||
所有 spec 的 verbose 字段已更新的新图.
|
||||
"""
|
||||
new_specs: list[TaskSpec[Any]] = []
|
||||
for spec in graph.all_specs().values():
|
||||
if spec.verbose == verbose:
|
||||
new_specs.append(spec)
|
||||
else:
|
||||
new_specs.append(replace(spec, verbose=verbose))
|
||||
return Graph.from_specs(new_specs)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CliRunner:
|
||||
"""命令行运行器: 根据用户输入执行对应的任务流图.
|
||||
|
||||
将命令名映射到 Graph 实例.
|
||||
通过 ``sys.argv`` 解析用户输入的命令, 执行对应的图.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
strategy : str | Strategy
|
||||
默认执行策略 (``Strategy.SEQUENTIAL`` / ``Strategy.THREAD`` /
|
||||
``Strategy.ASYNC`` 或对应字符串). 可被命令行 ``--strategy`` 覆盖.
|
||||
verbose : bool
|
||||
是否显示详细执行过程. ``True`` 时打印任务生命周期和 subprocess 输出.
|
||||
默认 ``True``. 可被命令行 ``--quiet`` 关闭.
|
||||
**graphs : Graph
|
||||
命令名到图的映射. 每个 key 是一个命令名, value 是对应的
|
||||
:class:`~pyflowx.graph.Graph`.
|
||||
|
||||
Examples
|
||||
--------
|
||||
基本用法::
|
||||
|
||||
runner = px.CliRunner(
|
||||
clean=px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("cargo_clean", cmd=["cargo", "clean"]),
|
||||
]
|
||||
),
|
||||
build=px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("uv_build", cmd=["uv", "build"]),
|
||||
]
|
||||
),
|
||||
)
|
||||
runner.run() # 解析 sys.argv
|
||||
|
||||
指定策略与描述::
|
||||
|
||||
runner = px.CliRunner(
|
||||
strategy=px.Strategy.THREAD,
|
||||
)
|
||||
runner.run(["test", "--strategy", "sequential"])
|
||||
"""
|
||||
|
||||
graphs: dict[str, Graph] = field(default_factory=dict)
|
||||
strategy: Strategy = field(default="sequential")
|
||||
description: str = field(default_factory=str)
|
||||
verbose: bool = field(default_factory=lambda: True)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.graphs:
|
||||
raise ValueError("CliRunner 至少需要一个命令 (通过关键字参数提供)")
|
||||
|
||||
for name, graph in self.graphs.items():
|
||||
if not isinstance(graph, Graph):
|
||||
raise TypeError(f"CliRunner 命令 {name!r} 的值必须是 Graph 实例, 实际是 {type(graph).__name__}")
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 内省
|
||||
# ------------------------------------------------------------------ #
|
||||
@property
|
||||
def commands(self) -> list[str]:
|
||||
"""可用的命令列表 (按插入顺序)."""
|
||||
return list(self.graphs.keys())
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 参数解析
|
||||
# ------------------------------------------------------------------ #
|
||||
def _prog_name(self) -> str:
|
||||
"""从 sys.argv[0] 推导程序名."""
|
||||
import os
|
||||
|
||||
return os.path.basename(sys.argv[0]) if sys.argv else "pyflowx"
|
||||
|
||||
def create_parser(self) -> argparse.ArgumentParser:
|
||||
"""创建参数解析器.
|
||||
|
||||
子类可覆盖此方法以添加自定义参数. 覆盖时应保留 ``command``
|
||||
位置参数与 ``--strategy`` / ``--dry-run`` / ``--list`` / ``--quiet``
|
||||
选项, 否则 :meth:`run` 的默认逻辑可能失效.
|
||||
|
||||
Returns
|
||||
-------
|
||||
argparse.ArgumentParser
|
||||
新创建的参数解析器实例.
|
||||
"""
|
||||
parser = argparse.ArgumentParser(
|
||||
prog=self._prog_name(),
|
||||
description=self.description or "PyFlowX CLI Runner",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=self._format_commands_help(),
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"command",
|
||||
nargs="?",
|
||||
help="要执行的命令",
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"--strategy",
|
||||
choices=list(get_args(Strategy)),
|
||||
default="sequential",
|
||||
help="执行策略 (默认: %(default)s)",
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="只打印执行计划, 不实际运行",
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"--list",
|
||||
action="store_true",
|
||||
help="列出所有可用命令",
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"--quiet",
|
||||
action="store_true",
|
||||
help="静默模式, 不显示执行过程 (覆盖默认 verbose)",
|
||||
)
|
||||
return parser
|
||||
|
||||
def _format_commands_help(self) -> str:
|
||||
"""格式化命令帮助文本."""
|
||||
lines = ["可用命令:"]
|
||||
for cmd in self.graphs:
|
||||
lines.append(f" {cmd}")
|
||||
return "\n".join(lines)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 执行
|
||||
# ------------------------------------------------------------------ #
|
||||
def run(self, args: Sequence[str] | None = None) -> int:
|
||||
"""解析参数并执行对应的图.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
args : Sequence[str] | None
|
||||
参数列表, 默认使用 ``sys.argv[1:]``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
int
|
||||
退出码 (0 成功, 1 失败, 130 中断).
|
||||
|
||||
Raises
|
||||
------
|
||||
SystemExit
|
||||
当 argparse 无法解析参数时 (与标准 argparse 行为一致).
|
||||
"""
|
||||
parser = self.create_parser()
|
||||
parsed = parser.parse_args(args)
|
||||
|
||||
# --list: 列出命令
|
||||
if parsed.list:
|
||||
print(self._format_commands_help())
|
||||
return CliExitCode.SUCCESS.value
|
||||
|
||||
# 无命令: 显示帮助
|
||||
if not parsed.command:
|
||||
parser.print_help()
|
||||
return CliExitCode.FAILURE.value
|
||||
|
||||
# 验证命令
|
||||
if parsed.command not in self.graphs:
|
||||
available = ", ".join(self.graphs.keys())
|
||||
print(
|
||||
f"错误: 未知命令 {parsed.command!r} (可用命令: {available})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return CliExitCode.FAILURE.value
|
||||
|
||||
# 确定是否 verbose: --quiet 覆盖默认值
|
||||
verbose = self.verbose and not parsed.quiet
|
||||
|
||||
# 对图应用 verbose 设置 (重建带 verbose 标记的 spec)
|
||||
graph = self.graphs[parsed.command]
|
||||
if verbose:
|
||||
graph = _apply_verbose_to_graph(graph, verbose=True)
|
||||
|
||||
# 执行对应的图
|
||||
try:
|
||||
report = run(
|
||||
graph,
|
||||
strategy=parsed.strategy,
|
||||
dry_run=parsed.dry_run,
|
||||
verbose=verbose,
|
||||
)
|
||||
return CliExitCode.SUCCESS.value if report.success else CliExitCode.FAILURE.value
|
||||
except KeyboardInterrupt:
|
||||
print("\n操作已取消", file=sys.stderr)
|
||||
return CliExitCode.INTERRUPTED.value
|
||||
except PyFlowXError as e:
|
||||
print(f"错误: {e}", file=sys.stderr)
|
||||
return CliExitCode.FAILURE.value
|
||||
|
||||
def run_cli(self, args: Sequence[str] | None = None) -> None:
|
||||
"""运行并以退出码退出进程.
|
||||
|
||||
作为 CLI 工具运行时的入口点, 等价于 ``sys.exit(self.run(args))``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
args : Sequence[str] | None
|
||||
参数列表, 默认使用 ``sys.argv[1:]``.
|
||||
"""
|
||||
sys.exit(self.run(args))
|
||||
+12
-11
@@ -17,9 +17,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, Mapping, Optional
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from .errors import StorageError
|
||||
|
||||
@@ -52,7 +52,7 @@ class MemoryBackend(StateBackend):
|
||||
"""进程内 dict 后端。进程退出即丢失。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._store: Dict[str, Any] = {}
|
||||
self._store: dict[str, Any] = {}
|
||||
|
||||
def load(self) -> Mapping[str, Any]:
|
||||
return dict(self._store)
|
||||
@@ -79,16 +79,16 @@ class JSONBackend(StateBackend):
|
||||
"""
|
||||
|
||||
def __init__(self, path: str) -> None:
|
||||
self._path = path
|
||||
self._store: Dict[str, Any] = {}
|
||||
self._path: str = path
|
||||
self._store: dict[str, Any] = {}
|
||||
self._load()
|
||||
|
||||
def _load(self) -> None:
|
||||
if not os.path.exists(self._path):
|
||||
if not Path(self._path).exists():
|
||||
return
|
||||
try:
|
||||
with open(self._path, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
with open(self._path, encoding="utf-8") as fh:
|
||||
data: Any = json.load(fh)
|
||||
if isinstance(data, dict):
|
||||
self._store = data
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
@@ -99,7 +99,8 @@ class JSONBackend(StateBackend):
|
||||
try:
|
||||
with open(tmp, "w", encoding="utf-8") as fh:
|
||||
json.dump(self._store, fh, ensure_ascii=False, indent=2)
|
||||
os.replace(tmp, self._path)
|
||||
|
||||
_ = Path(tmp).replace(Path(self._path))
|
||||
except (OSError, TypeError) as exc:
|
||||
raise StorageError(f"cannot write state file {self._path!r}", exc) from exc
|
||||
|
||||
@@ -109,7 +110,7 @@ class JSONBackend(StateBackend):
|
||||
def save(self, name: str, value: Any) -> None:
|
||||
# 在修改内存状态前先校验可序列化性。
|
||||
try:
|
||||
json.dumps(value)
|
||||
_ = json.dumps(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise StorageError(
|
||||
f"result of task {name!r} is not JSON-serialisable", exc
|
||||
@@ -128,6 +129,6 @@ class JSONBackend(StateBackend):
|
||||
self._flush()
|
||||
|
||||
|
||||
def resolve_backend(backend: Optional[StateBackend]) -> StateBackend:
|
||||
def resolve_backend(backend: StateBackend | None) -> StateBackend:
|
||||
"""返回 ``backend``;为 ``None`` 时返回新的 :class:`MemoryBackend`。"""
|
||||
return backend if backend is not None else MemoryBackend()
|
||||
|
||||
+161
-3
@@ -15,21 +15,22 @@
|
||||
* ``TaskStatus`` 是封闭枚举;执行器绝不发明临时字符串。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Coroutine,
|
||||
Generic,
|
||||
List,
|
||||
Mapping,
|
||||
Optional,
|
||||
Tuple,
|
||||
TypeVar,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
T = TypeVar("T")
|
||||
@@ -44,6 +45,16 @@ TaskFn = Union[
|
||||
# 单任务类型由函数签名本身保留。
|
||||
Context = Mapping[str, Any]
|
||||
|
||||
# 命令类型支持
|
||||
TaskCmd = Union[
|
||||
List[str], # 命令列表, 如 ["ls", "-la"]
|
||||
str, # shell 命令字符串
|
||||
Callable[..., Any], # Python 函数
|
||||
]
|
||||
|
||||
# 条件判断函数类型
|
||||
Condition = Callable[[], bool]
|
||||
|
||||
|
||||
class TaskStatus(Enum):
|
||||
"""任务在单次运行内的生命周期状态。"""
|
||||
@@ -66,6 +77,13 @@ class TaskSpec(Generic[T]):
|
||||
fn:
|
||||
待执行的可调用对象,可为同步或异步。其参数名驱动自动上下文
|
||||
注入(见 :mod:`pyflowx.context`)。
|
||||
若提供 ``cmd`` 参数,则此参数会被忽略。
|
||||
cmd:
|
||||
命令列表或 shell 字符串,支持三种形态:
|
||||
- ``list[str]``: 命令及参数列表,如 ``["ls", "-la"]``
|
||||
- ``str``: shell 命令字符串,如 ``"pip freeze > requirements.txt"``
|
||||
- ``Callable``: Python 函数,与 ``fn`` 参数等效
|
||||
若提供此参数,会自动包装为执行函数,覆盖 ``fn`` 参数。
|
||||
depends_on:
|
||||
必须先完成才能运行本任务的任务名列表。顺序无关;框架会做
|
||||
拓扑排序。
|
||||
@@ -83,16 +101,31 @@ class TaskSpec(Generic[T]):
|
||||
取消 worker future。
|
||||
tags:
|
||||
自由标签,供 :meth:`Graph.subgraph` 做选择性执行与调试。
|
||||
conditions:
|
||||
条件判断函数列表,只有所有条件都返回 ``True`` 时才执行任务。
|
||||
若任一条件返回 ``False``,任务会被标记为 SKIPPED。
|
||||
用于平台判断、环境变量检查等场景。
|
||||
cwd:
|
||||
命令执行的工作目录,仅在使用 ``cmd`` 参数时有效。
|
||||
``None`` 表示当前目录。
|
||||
verbose:
|
||||
是否在命令执行时显示详细输出。``True`` 时会打印执行的命令
|
||||
及其标准输出/标准错误。仅在使用 ``cmd`` 参数时有效。
|
||||
``False`` 时静默捕获输出(失败时仍会包含在错误信息中)。
|
||||
"""
|
||||
|
||||
name: str
|
||||
fn: TaskFn[T]
|
||||
fn: Optional[TaskFn[T]] = None
|
||||
cmd: Optional[TaskCmd] = None
|
||||
depends_on: Tuple[str, ...] = ()
|
||||
args: Tuple[Any, ...] = ()
|
||||
kwargs: Mapping[str, Any] = field(default_factory=dict)
|
||||
retries: int = 0
|
||||
timeout: Optional[float] = None
|
||||
tags: Tuple[str, ...] = ()
|
||||
conditions: Tuple[Condition, ...] = ()
|
||||
cwd: Optional[Path] = None
|
||||
verbose: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.name:
|
||||
@@ -103,6 +136,131 @@ class TaskSpec(Generic[T]):
|
||||
raise ValueError(f"TaskSpec '{self.name}': timeout must be > 0.")
|
||||
if self.name in self.depends_on:
|
||||
raise ValueError(f"TaskSpec '{self.name}' cannot depend on itself.")
|
||||
if self.fn is None and self.cmd is None:
|
||||
raise ValueError(f"TaskSpec '{self.name}': 必须提供 fn 或 cmd 参数。")
|
||||
|
||||
@property
|
||||
def effective_fn(self) -> TaskFn[T]:
|
||||
"""获取有效的执行函数.
|
||||
|
||||
若提供了 ``cmd`` 参数,则返回包装后的命令执行函数;
|
||||
否则返回 ``fn`` 参数。
|
||||
"""
|
||||
if self.cmd is not None:
|
||||
return self._wrap_cmd()
|
||||
if self.fn is not None:
|
||||
return self.fn
|
||||
|
||||
raise ValueError(f"TaskSpec '{self.name}': 没有可执行的函数或命令。") # pragma: no cover
|
||||
|
||||
def _wrap_cmd(self) -> TaskFn[Any]:
|
||||
"""将 cmd 包装为可执行函数.
|
||||
|
||||
Returns
|
||||
-------
|
||||
TaskFn[Any]
|
||||
包装后的执行函数.
|
||||
"""
|
||||
cmd = self.cmd
|
||||
cwd = self.cwd
|
||||
timeout = self.timeout
|
||||
verbose = self.verbose
|
||||
|
||||
if isinstance(cmd, list):
|
||||
|
||||
def _run_list() -> T:
|
||||
import subprocess
|
||||
|
||||
cmd_str = " ".join(str(arg) for arg in cmd)
|
||||
if verbose:
|
||||
print(f"[verbose] 执行命令: {cmd_str}", flush=True)
|
||||
if cwd is not None:
|
||||
print(f"[verbose] 工作目录: {cwd}", flush=True)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=cwd,
|
||||
timeout=timeout,
|
||||
capture_output=not verbose,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError(f"命令未找到: {cmd_str}") from None
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError(f"命令执行超时: {cmd_str} ({timeout}s)") from None
|
||||
except OSError as e:
|
||||
raise RuntimeError(f"命令执行异常: {cmd_str}: {e}") from e
|
||||
|
||||
if verbose:
|
||||
print(f"[verbose] 返回码: {result.returncode}", flush=True)
|
||||
|
||||
if result.returncode == 0:
|
||||
return cast(T, None) # type: ignore[return-value]
|
||||
|
||||
err_msg = f"命令执行失败: `{cmd_str}`, 返回码: {result.returncode}"
|
||||
if not verbose and result.stderr.strip():
|
||||
err_msg += f"\n{result.stderr.strip()}"
|
||||
raise RuntimeError(err_msg)
|
||||
|
||||
_run_list.__name__ = self.name
|
||||
return _run_list # type: ignore[return-value]
|
||||
|
||||
if isinstance(cmd, str):
|
||||
|
||||
def _run_shell() -> T:
|
||||
import subprocess
|
||||
|
||||
if verbose:
|
||||
print(f"[verbose] 执行 Shell: {cmd}", flush=True)
|
||||
if cwd is not None:
|
||||
print(f"[verbose] 工作目录: {cwd}", flush=True)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
shell=True,
|
||||
cwd=cwd,
|
||||
timeout=timeout,
|
||||
capture_output=not verbose,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError(f"Shell 命令未找到: {cmd}") from None
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError(f"Shell 命令执行超时: {cmd} ({timeout}s)") from None
|
||||
except OSError as e:
|
||||
raise RuntimeError(f"Shell 命令执行异常: {cmd}: {e}") from e
|
||||
|
||||
if verbose:
|
||||
print(f"[verbose] 返回码: {result.returncode}", flush=True)
|
||||
|
||||
if result.returncode == 0:
|
||||
return cast(T, None) # type: ignore[return-value]
|
||||
|
||||
err_msg = f"Shell 命令执行失败: `{cmd}`, 返回码: {result.returncode}"
|
||||
if not verbose and result.stderr.strip():
|
||||
err_msg += f"\n{result.stderr.strip()}"
|
||||
raise RuntimeError(err_msg)
|
||||
|
||||
_run_shell.__name__ = self.name
|
||||
return _run_shell # type: ignore[return-value]
|
||||
|
||||
if callable(cmd):
|
||||
return cmd # type: ignore[return-value]
|
||||
|
||||
raise TypeError(f"TaskSpec '{self.name}': 不支持的 cmd 类型 {type(cmd).__name__}") # pragma: no cover
|
||||
|
||||
def should_execute(self) -> bool:
|
||||
"""检查任务是否应该执行.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
若所有条件都返回 ``True``,则返回 ``True``;
|
||||
否则返回 ``False``。
|
||||
"""
|
||||
return all(condition() for condition in self.conditions)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Tests for cli.pymake module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from pyflowx.cli import pymake
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# maturin_build_cmd
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestMaturinBuildCmd:
|
||||
"""Test maturin_build_cmd function."""
|
||||
|
||||
def test_returns_list(self) -> None:
|
||||
"""Should return a list."""
|
||||
cmd = pymake.maturin_build_cmd()
|
||||
assert isinstance(cmd, list)
|
||||
|
||||
def test_contains_maturin_build(self) -> None:
|
||||
"""Should contain 'maturin' and 'build'."""
|
||||
cmd = pymake.maturin_build_cmd()
|
||||
assert "maturin" in cmd
|
||||
assert "build" in cmd
|
||||
|
||||
def test_contains_release_flag(self) -> None:
|
||||
"""Should contain release flag '-r'."""
|
||||
cmd = pymake.maturin_build_cmd()
|
||||
assert "-r" in cmd
|
||||
|
||||
def test_windows_includes_target(self) -> None:
|
||||
"""On Windows, should include target-specific flags."""
|
||||
cmd = pymake.maturin_build_cmd()
|
||||
if Constants.IS_WINDOWS:
|
||||
assert "--target" in cmd
|
||||
assert "x86_64-win7-windows-msvc" in cmd
|
||||
assert "-Zbuild-std" in cmd
|
||||
assert "-i" in cmd
|
||||
assert "python3.8" in cmd
|
||||
else:
|
||||
# On non-Windows, should not include Windows-specific flags
|
||||
assert "--target" not in cmd
|
||||
|
||||
def test_does_not_mutate_on_multiple_calls(self) -> None:
|
||||
"""Multiple calls should return independent lists."""
|
||||
cmd1 = pymake.maturin_build_cmd()
|
||||
cmd2 = pymake.maturin_build_cmd()
|
||||
assert cmd1 == cmd2
|
||||
# Mutating one should not affect the other
|
||||
cmd1.append("extra")
|
||||
assert "extra" not in cmd2
|
||||
|
||||
def test_non_windows_excludes_target_flags(self) -> None:
|
||||
"""On non-Windows, should not include Windows-specific flags (覆盖 22->32 分支)."""
|
||||
from unittest.mock import patch
|
||||
|
||||
with patch.object(pymake.Constants, "IS_WINDOWS", False):
|
||||
cmd = pymake.maturin_build_cmd()
|
||||
assert "maturin" in cmd
|
||||
assert "build" in cmd
|
||||
assert "-r" in cmd
|
||||
assert "--target" not in cmd
|
||||
assert "-Zbuild-std" not in cmd
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# check helper
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestCheckHelper:
|
||||
"""Test check helper function."""
|
||||
|
||||
def test_check_returns_condition(self) -> None:
|
||||
"""check() should return a Condition callable."""
|
||||
cond = pymake.check("python")
|
||||
assert callable(cond)
|
||||
|
||||
def test_check_uses_has_installed(self) -> None:
|
||||
"""check() should use BuiltinConditions.HAS_INSTALLED."""
|
||||
cond = pymake.check("python")
|
||||
# The condition should be a callable that returns a bool
|
||||
result = cond()
|
||||
assert isinstance(result, bool)
|
||||
|
||||
def test_check_for_nonexistent_app(self) -> None:
|
||||
"""check() for a nonexistent app should return False."""
|
||||
cond = pymake.check("definitely_not_installed_app_xyz")
|
||||
assert cond() is False
|
||||
|
||||
def test_check_for_python(self) -> None:
|
||||
"""check() for python should return True (python is always available)."""
|
||||
cond = pymake.check("python")
|
||||
# On some systems, 'python' might not be in PATH, but 'python3' might be
|
||||
# Just verify it returns a bool
|
||||
assert isinstance(cond(), bool)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# TaskSpec definitions
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestTaskSpecDefinitions:
|
||||
"""Test that all TaskSpec definitions are valid."""
|
||||
|
||||
def test_uv_build_spec(self) -> None:
|
||||
"""uv_build spec should be properly defined."""
|
||||
assert pymake.uv_build.name == "uv_build"
|
||||
assert pymake.uv_build.cmd == ["uv", "build"]
|
||||
assert len(pymake.uv_build.conditions) == 1
|
||||
|
||||
def test_maturin_build_spec(self) -> None:
|
||||
"""maturin_build spec should be properly defined."""
|
||||
assert pymake.maturin_build.name == "maturin_build"
|
||||
assert isinstance(pymake.maturin_build.cmd, list)
|
||||
assert len(pymake.maturin_build.conditions) == 1
|
||||
|
||||
def test_uv_sync_spec(self) -> None:
|
||||
"""uv_sync spec should be properly defined."""
|
||||
assert pymake.uv_sync.name == "uv_sync"
|
||||
assert pymake.uv_sync.cmd == ["uv", "sync"]
|
||||
|
||||
def test_git_clean_spec(self) -> None:
|
||||
"""git_clean spec should be properly defined."""
|
||||
assert pymake.git_clean.name == "git_clean"
|
||||
assert pymake.git_clean.cmd == ["gitt", "c"]
|
||||
|
||||
def test_test_spec(self) -> None:
|
||||
"""test spec should be properly defined."""
|
||||
assert pymake.test.name == "test"
|
||||
assert isinstance(pymake.test.cmd, list)
|
||||
assert "pytest" in pymake.test.cmd
|
||||
assert "-m" in pymake.test.cmd
|
||||
assert "not slow" in pymake.test.cmd
|
||||
|
||||
def test_test_fast_spec(self) -> None:
|
||||
"""test_fast spec should be properly defined."""
|
||||
assert pymake.test_fast.name == "test_fast"
|
||||
assert isinstance(pymake.test_fast.cmd, list)
|
||||
assert "pytest" in pymake.test_fast.cmd
|
||||
assert "-n" not in pymake.test_fast.cmd # test_fast doesn't use parallel
|
||||
|
||||
def test_test_coverage_spec(self) -> None:
|
||||
"""test_coverage spec should be properly defined."""
|
||||
assert pymake.test_coverage.name == "test_coverage"
|
||||
assert isinstance(pymake.test_coverage.cmd, list)
|
||||
assert "pytest" in pymake.test_coverage.cmd
|
||||
assert "--cov" in pymake.test_coverage.cmd
|
||||
|
||||
def test_ruff_lint_spec(self) -> None:
|
||||
"""ruff_lint spec should be properly defined."""
|
||||
assert pymake.ruff_lint.name == "lint"
|
||||
assert isinstance(pymake.ruff_lint.cmd, list)
|
||||
assert "ruff" in pymake.ruff_lint.cmd
|
||||
assert "check" in pymake.ruff_lint.cmd
|
||||
|
||||
def test_mypy_check_spec(self) -> None:
|
||||
"""mypy_check spec should be properly defined."""
|
||||
assert pymake.mypy_check.name == "typecheck"
|
||||
assert pymake.mypy_check.cmd == ["mypy", "."]
|
||||
|
||||
def test_ty_check_spec(self) -> None:
|
||||
"""ty_check spec should be properly defined."""
|
||||
assert pymake.ty_check.name == "ty_check"
|
||||
assert pymake.ty_check.cmd == ["ty", "check", "."]
|
||||
|
||||
def test_doc_spec(self) -> None:
|
||||
"""doc spec should be properly defined."""
|
||||
assert pymake.doc.name == "doc"
|
||||
assert isinstance(pymake.doc.cmd, list)
|
||||
assert "sphinx-build" in pymake.doc.cmd
|
||||
|
||||
def test_hatch_publish_spec(self) -> None:
|
||||
"""hatch_publish spec should be properly defined."""
|
||||
assert pymake.hatch_publish.name == "publish_python"
|
||||
assert pymake.hatch_publish.cmd == ["hatch", "publish"]
|
||||
|
||||
def test_twine_publish_spec(self) -> None:
|
||||
"""twine_publish spec should be properly defined."""
|
||||
assert pymake.twine_publish.name == "twine_publish"
|
||||
assert isinstance(pymake.twine_publish.cmd, list)
|
||||
assert "twine" in pymake.twine_publish.cmd
|
||||
assert "upload" in pymake.twine_publish.cmd
|
||||
|
||||
def test_tox_spec(self) -> None:
|
||||
"""tox spec should be properly defined."""
|
||||
assert pymake.tox.name == "tox"
|
||||
assert pymake.tox.cmd == ["tox", "-p", "auto"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# main function
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestMain:
|
||||
"""Test main function."""
|
||||
|
||||
def test_main_calls_run_cli(self) -> None:
|
||||
"""main() should create a CliRunner and call run_cli()."""
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
pymake.main()
|
||||
# run_cli() calls sys.exit(), so we should get SystemExit
|
||||
# The exit code depends on whether any commands are available
|
||||
assert exc_info.value.code in (0, 1, 2)
|
||||
|
||||
def test_main_with_list_argument(self) -> None:
|
||||
"""main() should handle --list argument."""
|
||||
with patch("sys.argv", ["pymake", "--list"]), pytest.raises(SystemExit) as exc_info:
|
||||
pymake.main()
|
||||
assert exc_info.value.code == 0
|
||||
|
||||
def test_main_creates_runner_with_multiple_commands(self) -> None:
|
||||
"""main() should create a CliRunner with multiple commands."""
|
||||
# We can't easily test the runner creation without mocking,
|
||||
# but we can verify that main() doesn't raise an error for --list
|
||||
with patch("sys.argv", ["pymake", "--list"]), pytest.raises(SystemExit):
|
||||
pymake.main()
|
||||
|
||||
def test_main_with_no_args_shows_help(self) -> None:
|
||||
"""main() with no args should show help and exit with failure."""
|
||||
with patch("sys.argv", ["pymake"]), pytest.raises(SystemExit) as exc_info:
|
||||
pymake.main()
|
||||
assert exc_info.value.code == 1
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Tests for conditions module."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
from pyflowx.conditions import (
|
||||
IS_LINUX,
|
||||
IS_MACOS,
|
||||
IS_POSIX,
|
||||
IS_WINDOWS,
|
||||
BuiltinConditions,
|
||||
Constants,
|
||||
)
|
||||
|
||||
|
||||
def test_constants_is_windows():
|
||||
"""Test Constants.IS_WINDOWS is correct."""
|
||||
assert (sys.platform == "win32") == Constants.IS_WINDOWS
|
||||
|
||||
|
||||
def test_constants_is_linux():
|
||||
"""Test Constants.IS_LINUX is correct."""
|
||||
assert (sys.platform == "linux") == Constants.IS_LINUX
|
||||
|
||||
|
||||
def test_constants_is_macos():
|
||||
"""Test Constants.IS_MACOS is correct."""
|
||||
assert (sys.platform == "darwin") == Constants.IS_MACOS
|
||||
|
||||
|
||||
def test_constants_is_posix():
|
||||
"""Test Constants.IS_POSIX is correct."""
|
||||
assert (sys.platform != "win32") == Constants.IS_POSIX
|
||||
|
||||
|
||||
def test_builtin_conditions_is_windows():
|
||||
"""Test BuiltinConditions.IS_WINDOWS."""
|
||||
result = BuiltinConditions.IS_WINDOWS()
|
||||
assert result == Constants.IS_WINDOWS
|
||||
|
||||
|
||||
def test_builtin_conditions_is_linux():
|
||||
"""Test BuiltinConditions.IS_LINUX."""
|
||||
result = BuiltinConditions.IS_LINUX()
|
||||
assert result == Constants.IS_LINUX
|
||||
|
||||
|
||||
def test_builtin_conditions_is_macos():
|
||||
"""Test BuiltinConditions.IS_MACOS."""
|
||||
result = BuiltinConditions.IS_MACOS()
|
||||
assert result == Constants.IS_MACOS
|
||||
|
||||
|
||||
def test_builtin_conditions_is_posix():
|
||||
"""Test BuiltinConditions.IS_POSIX."""
|
||||
result = BuiltinConditions.IS_POSIX()
|
||||
assert result == Constants.IS_POSIX
|
||||
|
||||
|
||||
def test_builtin_conditions_python_version_major_only():
|
||||
"""Test BuiltinConditions.PYTHON_VERSION with major only."""
|
||||
# Test with current Python version
|
||||
current_major = sys.version_info.major
|
||||
assert BuiltinConditions.PYTHON_VERSION(current_major) is True
|
||||
assert BuiltinConditions.PYTHON_VERSION(current_major + 1) is False
|
||||
|
||||
|
||||
def test_builtin_conditions_python_version_with_minor():
|
||||
"""Test BuiltinConditions.PYTHON_VERSION with major and minor."""
|
||||
current_major = sys.version_info.major
|
||||
current_minor = sys.version_info.minor
|
||||
assert BuiltinConditions.PYTHON_VERSION(current_major, current_minor) is True
|
||||
assert BuiltinConditions.PYTHON_VERSION(current_major, current_minor + 1) is False
|
||||
|
||||
|
||||
def test_builtin_conditions_python_version_at_least():
|
||||
"""Test BuiltinConditions.PYTHON_VERSION_AT_LEAST."""
|
||||
current_major = sys.version_info.major
|
||||
current_minor = sys.version_info.minor
|
||||
# Current version should be at least itself
|
||||
assert BuiltinConditions.PYTHON_VERSION_AT_LEAST(current_major, current_minor) is True
|
||||
# Current version should be at least an older version
|
||||
assert BuiltinConditions.PYTHON_VERSION_AT_LEAST(current_major - 1, 0) is True
|
||||
# Current version should NOT be at least a newer version
|
||||
assert BuiltinConditions.PYTHON_VERSION_AT_LEAST(current_major + 1, 0) is False
|
||||
|
||||
|
||||
def test_builtin_conditions_HAS_INSTALLED_true():
|
||||
"""Test BuiltinConditions.HAS_INSTALLED when app exists."""
|
||||
# Python should always be available
|
||||
condition = BuiltinConditions.HAS_INSTALLED("python")
|
||||
assert condition() is True
|
||||
|
||||
|
||||
def test_builtin_conditions_HAS_INSTALLED_false():
|
||||
"""Test BuiltinConditions.HAS_INSTALLED when app doesn't exist."""
|
||||
condition = BuiltinConditions.HAS_INSTALLED("nonexistent_app_12345")
|
||||
assert condition() is False
|
||||
|
||||
|
||||
def test_builtin_conditions_env_var_exists_true():
|
||||
"""Test BuiltinConditions.ENV_VAR_EXISTS when variable exists."""
|
||||
with patch.dict(os.environ, {"TEST_VAR": "value"}):
|
||||
condition = BuiltinConditions.ENV_VAR_EXISTS("TEST_VAR")
|
||||
assert condition() is True
|
||||
|
||||
|
||||
def test_builtin_conditions_env_var_exists_false():
|
||||
"""Test BuiltinConditions.ENV_VAR_EXISTS when variable doesn't exist."""
|
||||
condition = BuiltinConditions.ENV_VAR_EXISTS("NONEXISTENT_VAR_12345")
|
||||
assert condition() is False
|
||||
|
||||
|
||||
def test_builtin_conditions_env_var_equals_true():
|
||||
"""Test BuiltinConditions.ENV_VAR_EQUALS when value matches."""
|
||||
with patch.dict(os.environ, {"TEST_VAR": "expected_value"}):
|
||||
condition = BuiltinConditions.ENV_VAR_EQUALS("TEST_VAR", "expected_value")
|
||||
assert condition() is True
|
||||
|
||||
|
||||
def test_builtin_conditions_env_var_equals_false():
|
||||
"""Test BuiltinConditions.ENV_VAR_EQUALS when value doesn't match."""
|
||||
with patch.dict(os.environ, {"TEST_VAR": "different_value"}):
|
||||
condition = BuiltinConditions.ENV_VAR_EQUALS("TEST_VAR", "expected_value")
|
||||
assert condition() is False
|
||||
|
||||
|
||||
def test_builtin_conditions_not():
|
||||
"""Test BuiltinConditions.NOT."""
|
||||
true_condition = lambda: True # noqa: E731
|
||||
false_condition = lambda: False # noqa: E731
|
||||
|
||||
not_true = BuiltinConditions.NOT(true_condition)
|
||||
assert not_true() is False
|
||||
|
||||
not_false = BuiltinConditions.NOT(false_condition)
|
||||
assert not_false() is True
|
||||
|
||||
|
||||
def test_builtin_conditions_and_all_true():
|
||||
"""Test BuiltinConditions.AND when all conditions are true."""
|
||||
true_condition = lambda: True # noqa: E731
|
||||
condition = BuiltinConditions.AND(true_condition, true_condition, true_condition)
|
||||
assert condition() is True
|
||||
|
||||
|
||||
def test_builtin_conditions_and_one_false():
|
||||
"""Test BuiltinConditions.AND when one condition is false."""
|
||||
true_condition = lambda: True # noqa: E731
|
||||
false_condition = lambda: False # noqa: E731
|
||||
condition = BuiltinConditions.AND(true_condition, false_condition, true_condition)
|
||||
assert condition() is False
|
||||
|
||||
|
||||
def test_builtin_conditions_or_all_false():
|
||||
"""Test BuiltinConditions.OR when all conditions are false."""
|
||||
false_condition = lambda: False # noqa: E731
|
||||
condition = BuiltinConditions.OR(false_condition, false_condition, false_condition)
|
||||
assert condition() is False
|
||||
|
||||
|
||||
def test_builtin_conditions_or_one_true():
|
||||
"""Test BuiltinConditions.OR when one condition is true."""
|
||||
true_condition = lambda: True # noqa: E731
|
||||
false_condition = lambda: False # noqa: E731
|
||||
condition = BuiltinConditions.OR(false_condition, true_condition, false_condition)
|
||||
assert condition() is True
|
||||
|
||||
|
||||
def test_exported_conditions():
|
||||
"""Test exported condition functions."""
|
||||
assert IS_WINDOWS() == Constants.IS_WINDOWS
|
||||
assert IS_LINUX() == Constants.IS_LINUX
|
||||
assert IS_MACOS() == Constants.IS_MACOS
|
||||
assert IS_POSIX() == Constants.IS_POSIX
|
||||
+146
-149
@@ -1,4 +1,4 @@
|
||||
"""Tests for context injection rules."""
|
||||
"""测试上下文注入规则."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -11,36 +11,43 @@ from pyflowx.context import _is_context_annotation, build_call_args, describe_in
|
||||
from pyflowx.errors import InjectionError
|
||||
|
||||
|
||||
def test_inject_by_parameter_name() -> None:
|
||||
class TestBuildCallArgs:
|
||||
"""测试 build_call_args 函数."""
|
||||
|
||||
def test_inject_by_parameter_name(self) -> None:
|
||||
"""参数名匹配依赖名时应注入对应结果."""
|
||||
|
||||
def fn(a: int, b: str) -> str:
|
||||
return f"{a}{b}"
|
||||
|
||||
spec = px.TaskSpec("c", fn, ("a", "b"))
|
||||
args, kwargs = build_call_args(spec, {"a": 1, "b": "x"})
|
||||
assert args == ()
|
||||
spec = px.TaskSpec("c", fn, depends_on=("a", "b"))
|
||||
_args, kwargs = build_call_args(spec, {"a": 1, "b": "x"})
|
||||
assert kwargs == {"a": 1, "b": "x"}
|
||||
|
||||
def test_inject_context_annotation(self) -> None:
|
||||
"""标注为 Context 的参数应接收完整依赖映射."""
|
||||
|
||||
def test_inject_context_annotation() -> None:
|
||||
def fn(ctx: px.Context) -> int:
|
||||
return len(ctx)
|
||||
|
||||
spec = px.TaskSpec("agg", fn, ("a", "b"))
|
||||
args, kwargs = build_call_args(spec, {"a": 1, "b": 2, "c": 99})
|
||||
spec = px.TaskSpec("agg", fn, depends_on=("a", "b"))
|
||||
_args, kwargs = build_call_args(spec, {"a": 1, "b": 2, "c": 99})
|
||||
# Only the task's own deps are passed.
|
||||
assert kwargs == {"ctx": {"a": 1, "b": 2}}
|
||||
|
||||
def test_inject_var_keyword(self) -> None:
|
||||
"""**kwargs 参数应以 dict 形式接收所有依赖结果."""
|
||||
|
||||
def test_inject_var_keyword() -> None:
|
||||
def fn(**kwargs: Any) -> int:
|
||||
def fn(**kwargs: Any) -> int: # pyright: ignore[reportExplicitAny, reportAny]
|
||||
return sum(kwargs.values())
|
||||
|
||||
spec = px.TaskSpec("agg", fn, ("a", "b"))
|
||||
args, kwargs = build_call_args(spec, {"a": 1, "b": 2})
|
||||
spec = px.TaskSpec("agg", fn, depends_on=("a", "b"))
|
||||
_args, kwargs = build_call_args(spec, {"a": 1, "b": 2})
|
||||
assert kwargs == {"a": 1, "b": 2}
|
||||
|
||||
def test_static_args_and_kwargs(self) -> None:
|
||||
"""静态 args/kwargs 应正确填充非依赖参数."""
|
||||
|
||||
def test_static_args_and_kwargs() -> None:
|
||||
def fn(uid: int, source: str) -> str:
|
||||
return f"{source}:{uid}"
|
||||
|
||||
@@ -49,65 +56,164 @@ def test_static_args_and_kwargs() -> None:
|
||||
assert args == (42,)
|
||||
assert kwargs == {"source": "api"}
|
||||
|
||||
def test_default_param_not_required(self) -> None:
|
||||
"""有默认值的参数无需依赖或静态值."""
|
||||
|
||||
def test_default_param_not_required() -> None:
|
||||
def fn(a: int, flag: bool = True) -> int:
|
||||
return a if flag else 0
|
||||
|
||||
spec = px.TaskSpec("t", fn, ("a",))
|
||||
args, kwargs = build_call_args(spec, {"a": 5})
|
||||
spec = px.TaskSpec("t", fn, depends_on=("a",))
|
||||
_args, kwargs = build_call_args(spec, {"a": 5})
|
||||
assert kwargs == {"a": 5}
|
||||
|
||||
def test_unresolved_required_param_raises(self) -> None:
|
||||
"""必需参数无法解析时应抛出 InjectionError."""
|
||||
|
||||
def test_unresolved_required_param_raises() -> None:
|
||||
def fn(a: int, missing: str) -> None:
|
||||
def fn(_a: int, _: str) -> None:
|
||||
return None
|
||||
|
||||
spec = px.TaskSpec("t", fn, ("a",))
|
||||
spec = px.TaskSpec("t", fn, depends_on=("a",))
|
||||
with pytest.raises(InjectionError) as exc_info:
|
||||
build_call_args(spec, {"a": 1})
|
||||
assert "missing" in str(exc_info.value)
|
||||
_ = build_call_args(spec, {"a": 1})
|
||||
assert "Cannot inject" in str(exc_info.value)
|
||||
|
||||
def test_static_kwargs_collide_with_dependency(self) -> None:
|
||||
"""静态 kwargs 与依赖名冲突时应抛出 InjectionError."""
|
||||
|
||||
def test_static_kwargs_collide_with_dependency() -> None:
|
||||
def fn(a: int) -> int:
|
||||
return a
|
||||
|
||||
spec = px.TaskSpec("t", fn, ("a",), kwargs={"a": 99})
|
||||
spec = px.TaskSpec("t", fn, depends_on=("a",), kwargs={"a": 99})
|
||||
with pytest.raises(InjectionError):
|
||||
build_call_args(spec, {"a": 1})
|
||||
_ = build_call_args(spec, {"a": 1})
|
||||
|
||||
def test_var_positional_not_required(self) -> None:
|
||||
"""*args 参数不应触发 InjectionError."""
|
||||
|
||||
def fn(*args: Any) -> int: # pyright: ignore[reportExplicitAny, reportAny]
|
||||
return len(args)
|
||||
|
||||
spec = px.TaskSpec("t", fn, args=(1, 2, 3))
|
||||
args, kwargs = build_call_args(spec, {})
|
||||
assert args == (1, 2, 3)
|
||||
assert kwargs == {}
|
||||
|
||||
def test_var_keyword_consumes_leftover(self) -> None:
|
||||
"""**kwargs 应吞掉未被具名参数消费的依赖结果."""
|
||||
|
||||
def fn(a: int, **rest: Any) -> int: # pyright: ignore[reportExplicitAny, reportAny]
|
||||
return a + sum(rest.values())
|
||||
|
||||
spec = px.TaskSpec("t", fn, depends_on=("a", "b", "c"))
|
||||
_args, kwargs = build_call_args(spec, {"a": 1, "b": 2, "c": 3})
|
||||
assert kwargs == {"a": 1, "b": 2, "c": 3}
|
||||
|
||||
def test_no_var_keyword_drops_leftover(self) -> None:
|
||||
"""无 **kwargs 时,未被消费的依赖结果被丢弃(不报错)."""
|
||||
|
||||
def fn(a: int) -> int:
|
||||
return a
|
||||
|
||||
spec = px.TaskSpec("t", fn, depends_on=("a", "b"))
|
||||
# b 是依赖但 fn 不接收它 —— 应正常工作
|
||||
_args, kwargs = build_call_args(spec, {"a": 1, "b": 2})
|
||||
assert kwargs == {"a": 1}
|
||||
|
||||
def test_context_annotation_only_deps(self) -> None:
|
||||
"""Context 标注只接收该任务自身 depends_on 的结果."""
|
||||
|
||||
def fn(ctx: px.Context) -> int:
|
||||
return len(ctx)
|
||||
|
||||
spec = px.TaskSpec("t", fn, depends_on=("a", "b"))
|
||||
_args, kwargs = build_call_args(spec, {"a": 1, "b": 2, "c": 99})
|
||||
assert kwargs == {"ctx": {"a": 1, "b": 2}}
|
||||
|
||||
|
||||
def test_describe_injection() -> None:
|
||||
def fn(a: int, ctx: px.Context, flag: bool = False) -> None:
|
||||
class TestDescribeInjection:
|
||||
"""测试 describe_injection 函数."""
|
||||
|
||||
def test_describe_injection(self) -> None:
|
||||
"""应正确描述依赖注入、Context 标注和默认值."""
|
||||
|
||||
def fn(a: int, ctx: px.Context, flag: bool = False) -> None: # noqa: ARG001
|
||||
return None
|
||||
|
||||
spec = px.TaskSpec("t", fn, ("a",))
|
||||
spec = px.TaskSpec("t", fn, depends_on=("a",))
|
||||
desc = describe_injection(spec)
|
||||
assert "a=<result:a>" in desc
|
||||
assert "ctx=<Context>" in desc
|
||||
assert "flag=<default>" in desc
|
||||
|
||||
def test_var_positional(self) -> None:
|
||||
"""*args 参数应显示为 *args."""
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# _is_context_annotation 各分支
|
||||
# ---------------------------------------------------------------------- #
|
||||
def test_is_context_annotation_direct_object() -> None:
|
||||
"""直接传入 Context 别名对象应返回 True。"""
|
||||
def fn(*args: Any) -> None: # noqa: ARG001
|
||||
return None
|
||||
|
||||
spec = px.TaskSpec("t", fn)
|
||||
desc = describe_injection(spec)
|
||||
assert "*args" in desc
|
||||
|
||||
def test_var_keyword(self) -> None:
|
||||
"""**kwargs 参数应显示为 **kwargs=<all-deps>."""
|
||||
|
||||
def fn(**kwargs: Any) -> None: # pyright: ignore[reportExplicitAny, reportAny] # noqa: ARG001
|
||||
return None
|
||||
|
||||
spec = px.TaskSpec("t", fn, depends_on=("a",))
|
||||
desc = describe_injection(spec)
|
||||
assert "**kwargs=<all-deps>" in desc
|
||||
|
||||
def test_unresolved(self) -> None:
|
||||
"""无依赖、无静态值、无默认的参数应显示为 <UNRESOLVED>."""
|
||||
|
||||
def fn(missing: int) -> None: # noqa: ARG001
|
||||
return None
|
||||
|
||||
spec = px.TaskSpec("t", fn)
|
||||
desc = describe_injection(spec)
|
||||
assert "missing=<UNRESOLVED>" in desc
|
||||
|
||||
def test_static_kwargs(self) -> None:
|
||||
"""静态 kwargs 应显示具体值."""
|
||||
|
||||
def fn(flag: bool = False) -> None: # noqa: ARG001
|
||||
return None
|
||||
|
||||
spec = px.TaskSpec("t", fn, kwargs={"flag": True})
|
||||
desc = describe_injection(spec)
|
||||
assert "flag=True" in desc
|
||||
|
||||
def test_positional_args_filled(self) -> None:
|
||||
"""spec.args 填充的位置参数应显示具体值(覆盖 args_filled 分支)."""
|
||||
|
||||
def fn(a: int, b: str) -> None: # noqa: ARG001
|
||||
return None
|
||||
|
||||
spec = px.TaskSpec("t", fn, args=(1, "x"))
|
||||
desc = describe_injection(spec)
|
||||
assert "a=1" in desc
|
||||
assert "b='x'" in desc
|
||||
|
||||
|
||||
class TestIsContextAnnotation:
|
||||
"""测试 _is_context_annotation 函数."""
|
||||
|
||||
def test_direct_object(self) -> None:
|
||||
"""直接传入 Context 别名对象应返回 True."""
|
||||
assert _is_context_annotation(px.Context) is True
|
||||
|
||||
|
||||
def test_is_context_annotation_string() -> None:
|
||||
"""字符串形式的注解应被识别。"""
|
||||
def test_string(self) -> None:
|
||||
"""字符串形式的注解应被识别."""
|
||||
assert _is_context_annotation("Context") is True
|
||||
assert _is_context_annotation("px.Context") is True
|
||||
assert _is_context_annotation("pyflowx.Context") is True
|
||||
assert _is_context_annotation("NotContext") is False
|
||||
assert _is_context_annotation("int") is False
|
||||
|
||||
|
||||
def test_is_context_annotation_typing_alias() -> None:
|
||||
"""具有 __name__/_name 为 Context/Mapping 的 typing 别名应返回 True。"""
|
||||
def test_typing_alias(self) -> None:
|
||||
"""具有 __name__/_name 为 Context/Mapping 的 typing 别名应返回 True."""
|
||||
|
||||
class FakeAlias:
|
||||
__name__ = "Context"
|
||||
@@ -119,117 +225,8 @@ def test_is_context_annotation_typing_alias() -> None:
|
||||
|
||||
assert _is_context_annotation(FakeMapping()) is True
|
||||
|
||||
|
||||
def test_is_context_annotation_other() -> None:
|
||||
"""其他类型注解应返回 False。"""
|
||||
def test_other(self) -> None:
|
||||
"""其他类型注解应返回 False."""
|
||||
assert _is_context_annotation(int) is False
|
||||
assert _is_context_annotation(str) is False
|
||||
assert _is_context_annotation(None) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# describe_injection 其余分支
|
||||
# ---------------------------------------------------------------------- #
|
||||
def test_describe_injection_var_positional() -> None:
|
||||
"""*args 参数应显示为 *args。"""
|
||||
|
||||
def fn(*args: Any) -> None:
|
||||
return None
|
||||
|
||||
spec = px.TaskSpec("t", fn)
|
||||
desc = describe_injection(spec)
|
||||
assert "*args" in desc
|
||||
|
||||
|
||||
def test_describe_injection_var_keyword() -> None:
|
||||
"""**kwargs 参数应显示为 **kwargs=<all-deps>。"""
|
||||
|
||||
def fn(**kwargs: Any) -> None:
|
||||
return None
|
||||
|
||||
spec = px.TaskSpec("t", fn, ("a",))
|
||||
desc = describe_injection(spec)
|
||||
assert "**kwargs=<all-deps>" in desc
|
||||
|
||||
|
||||
def test_describe_injection_unresolved() -> None:
|
||||
"""无依赖、无静态值、无默认的参数应显示为 <UNRESOLVED>。"""
|
||||
|
||||
def fn(missing: int) -> None:
|
||||
return None
|
||||
|
||||
spec = px.TaskSpec("t", fn)
|
||||
desc = describe_injection(spec)
|
||||
assert "missing=<UNRESOLVED>" in desc
|
||||
|
||||
|
||||
def test_describe_injection_static_kwargs() -> None:
|
||||
"""静态 kwargs 应显示具体值。"""
|
||||
|
||||
def fn(flag: bool = False) -> None:
|
||||
return None
|
||||
|
||||
spec = px.TaskSpec("t", fn, kwargs={"flag": True})
|
||||
desc = describe_injection(spec)
|
||||
assert "flag=True" in desc
|
||||
|
||||
|
||||
def test_describe_injection_positional_args_filled() -> None:
|
||||
"""spec.args 填充的位置参数应显示具体值(覆盖 args_filled 分支)。"""
|
||||
|
||||
def fn(a: int, b: str) -> None:
|
||||
return None
|
||||
|
||||
spec = px.TaskSpec("t", fn, args=(1, "x"))
|
||||
desc = describe_injection(spec)
|
||||
assert "a=1" in desc
|
||||
assert "b='x'" in desc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# build_call_args 边界
|
||||
# ---------------------------------------------------------------------- #
|
||||
def test_build_call_args_var_positional_not_required() -> None:
|
||||
"""*args 参数不应触发 InjectionError。"""
|
||||
|
||||
def fn(*args: Any) -> int:
|
||||
return len(args)
|
||||
|
||||
spec = px.TaskSpec("t", fn, args=(1, 2, 3))
|
||||
args, kwargs = build_call_args(spec, {})
|
||||
assert args == (1, 2, 3)
|
||||
assert kwargs == {}
|
||||
|
||||
|
||||
def test_build_call_args_var_keyword_consumes_leftover() -> None:
|
||||
"""**kwargs 应吞掉未被具名参数消费的依赖结果。"""
|
||||
|
||||
def fn(a: int, **rest: Any) -> int:
|
||||
return a + sum(rest.values())
|
||||
|
||||
spec = px.TaskSpec("t", fn, ("a", "b", "c"))
|
||||
args, kwargs = build_call_args(spec, {"a": 1, "b": 2, "c": 3})
|
||||
assert kwargs == {"a": 1, "b": 2, "c": 3}
|
||||
|
||||
|
||||
def test_build_call_args_no_var_keyword_drops_leftover() -> None:
|
||||
"""无 **kwargs 时,未被消费的依赖结果被丢弃(不报错)。"""
|
||||
|
||||
def fn(a: int) -> int:
|
||||
return a
|
||||
|
||||
spec = px.TaskSpec("t", fn, ("a", "b"))
|
||||
# b 是依赖但 fn 不接收它 —— 应正常工作
|
||||
args, kwargs = build_call_args(spec, {"a": 1, "b": 2})
|
||||
assert kwargs == {"a": 1}
|
||||
|
||||
|
||||
def test_build_call_args_context_annotation_only_deps() -> None:
|
||||
"""Context 标注只接收该任务自身 depends_on 的结果。"""
|
||||
|
||||
def fn(ctx: px.Context) -> int:
|
||||
return len(ctx)
|
||||
|
||||
spec = px.TaskSpec("t", fn, ("a", "b"))
|
||||
args, kwargs = build_call_args(spec, {"a": 1, "b": 2, "c": 99})
|
||||
assert kwargs == {"ctx": {"a": 1, "b": 2}}
|
||||
|
||||
+38
-44
@@ -3,11 +3,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, List
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -29,7 +29,7 @@ def test_sequential_basic() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("extract", extract),
|
||||
px.TaskSpec("double", double, ("extract",)),
|
||||
px.TaskSpec("double", double, depends_on=("extract",)),
|
||||
]
|
||||
)
|
||||
report = px.run(graph, strategy="sequential")
|
||||
@@ -39,7 +39,7 @@ def test_sequential_basic() -> None:
|
||||
|
||||
|
||||
def test_sequential_diamond() -> None:
|
||||
order: List[str] = []
|
||||
order: list[str] = []
|
||||
|
||||
def make(name: str) -> Any:
|
||||
def fn() -> str:
|
||||
@@ -51,9 +51,9 @@ def test_sequential_diamond() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", make("a")),
|
||||
px.TaskSpec("b", make("b"), ("a",)),
|
||||
px.TaskSpec("c", make("c"), ("a",)),
|
||||
px.TaskSpec("d", make("d"), ("b", "c")),
|
||||
px.TaskSpec("b", make("b"), depends_on=("a",)),
|
||||
px.TaskSpec("c", make("c"), depends_on=("a",)),
|
||||
px.TaskSpec("d", make("d"), depends_on=("b", "c")),
|
||||
]
|
||||
)
|
||||
report = px.run(graph, strategy="sequential")
|
||||
@@ -66,17 +66,17 @@ def test_failure_propagates() -> None:
|
||||
def boom() -> None:
|
||||
raise ValueError("kaboom")
|
||||
|
||||
def downstream(boom: None) -> int:
|
||||
def downstream(_boom: None) -> int:
|
||||
return 1
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("boom", boom),
|
||||
px.TaskSpec("downstream", downstream, ("boom",)),
|
||||
px.TaskSpec("downstream", downstream, depends_on=("boom",)),
|
||||
]
|
||||
)
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
px.run(graph, strategy="sequential")
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
assert exc_info.value.task == "boom"
|
||||
assert isinstance(exc_info.value.cause, ValueError)
|
||||
|
||||
@@ -103,13 +103,14 @@ def test_retries_exhausted() -> None:
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("f", always_fail, retries=2)])
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
px.run(graph, strategy="sequential")
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
assert exc_info.value.attempts == 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# Threaded
|
||||
# ---------------------------------------------------------------------- #
|
||||
@pytest.mark.slow
|
||||
def test_threaded_parallelism() -> None:
|
||||
def slow() -> str:
|
||||
time.sleep(0.3)
|
||||
@@ -130,8 +131,9 @@ def test_threaded_parallelism() -> None:
|
||||
assert elapsed < 0.8
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_threaded_layer_barrier() -> None:
|
||||
finished: List[str] = []
|
||||
finished: list[str] = []
|
||||
lock = threading.Lock()
|
||||
|
||||
def make(name: str) -> Any:
|
||||
@@ -147,7 +149,7 @@ def test_threaded_layer_barrier() -> None:
|
||||
[
|
||||
px.TaskSpec("a", make("a")),
|
||||
px.TaskSpec("b", make("b")),
|
||||
px.TaskSpec("c", make("c"), ("a", "b")),
|
||||
px.TaskSpec("c", make("c"), depends_on=("a", "b")),
|
||||
]
|
||||
)
|
||||
report = px.run(graph, strategy="thread", max_workers=2)
|
||||
@@ -171,7 +173,7 @@ def test_async_basic() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("fetch", fetch),
|
||||
px.TaskSpec("transform", transform, ("fetch",)),
|
||||
px.TaskSpec("transform", transform, depends_on=("fetch",)),
|
||||
]
|
||||
)
|
||||
report = px.run(graph, strategy="async")
|
||||
@@ -179,6 +181,7 @@ def test_async_basic() -> None:
|
||||
assert report["transform"] == 84
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_async_parallelism() -> None:
|
||||
async def slow() -> str:
|
||||
await asyncio.sleep(0.3)
|
||||
@@ -209,7 +212,7 @@ def test_async_mixed_sync_and_async() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("sync_task", sync_task),
|
||||
px.TaskSpec("async_task", async_task, ("sync_task",)),
|
||||
px.TaskSpec("async_task", async_task, depends_on=("sync_task",)),
|
||||
]
|
||||
)
|
||||
report = px.run(graph, strategy="async")
|
||||
@@ -223,7 +226,7 @@ def test_async_timeout() -> None:
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("slow", slow, timeout=0.05)])
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
px.run(graph, strategy="async")
|
||||
_ = px.run(graph, strategy="async")
|
||||
assert isinstance(exc_info.value.cause, TaskTimeoutError)
|
||||
|
||||
|
||||
@@ -231,7 +234,7 @@ def test_async_timeout() -> None:
|
||||
# Dry run
|
||||
# ---------------------------------------------------------------------- #
|
||||
def test_dry_run_does_not_execute(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
called: List[str] = []
|
||||
called: list[str] = []
|
||||
|
||||
def fn() -> str:
|
||||
called.append("x")
|
||||
@@ -250,7 +253,7 @@ def test_dry_run_does_not_execute(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
# State / resume
|
||||
# ---------------------------------------------------------------------- #
|
||||
def test_memory_backend_resume() -> None:
|
||||
runs: List[str] = []
|
||||
runs: list[str] = []
|
||||
|
||||
def make(name: str) -> Any:
|
||||
def fn() -> str:
|
||||
@@ -262,30 +265,30 @@ def test_memory_backend_resume() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", make("a")),
|
||||
px.TaskSpec("b", make("b"), ("a",)),
|
||||
px.TaskSpec("b", make("b"), depends_on=("a",)),
|
||||
]
|
||||
)
|
||||
backend = MemoryBackend()
|
||||
px.run(graph, strategy="sequential", state=backend)
|
||||
_ = px.run(graph, strategy="sequential", state=backend)
|
||||
assert runs == ["a", "b"]
|
||||
|
||||
# Second run: both cached, neither re-executed.
|
||||
px.run(graph, strategy="sequential", state=backend)
|
||||
_ = px.run(graph, strategy="sequential", state=backend)
|
||||
assert runs == ["a", "b"] # unchanged
|
||||
|
||||
|
||||
def test_json_backend_persistence() -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "state.json")
|
||||
path = str(Path(tmp) / "state.json")
|
||||
|
||||
def fn() -> int:
|
||||
return 7
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", fn)])
|
||||
px.run(graph, strategy="sequential", state=JSONBackend(path))
|
||||
_ = px.run(graph, strategy="sequential", state=JSONBackend(path))
|
||||
|
||||
# New backend reads the file; task should be skipped.
|
||||
runs: List[str] = []
|
||||
runs: list[str] = []
|
||||
|
||||
def fn2() -> int:
|
||||
runs.append("ran")
|
||||
@@ -301,27 +304,18 @@ def test_json_backend_persistence() -> None:
|
||||
# Events
|
||||
# ---------------------------------------------------------------------- #
|
||||
def test_on_event_callback() -> None:
|
||||
events: List[px.TaskEvent] = []
|
||||
events: list[px.TaskEvent] = []
|
||||
|
||||
def fn() -> int:
|
||||
return 1
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", fn)])
|
||||
px.run(graph, strategy="sequential", on_event=events.append)
|
||||
_ = px.run(graph, strategy="sequential", on_event=events.append)
|
||||
statuses = [e.status for e in events]
|
||||
assert px.TaskStatus.SUCCESS in statuses
|
||||
assert all(e.task == "a" for e in events)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# Invalid strategy
|
||||
# ---------------------------------------------------------------------- #
|
||||
def test_invalid_strategy() -> None:
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", lambda: None)]) # type: ignore[arg-type]
|
||||
with pytest.raises(ValueError):
|
||||
px.run(graph, strategy="bogus") # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 异步策略:sync 任务无 timeout 分支 + timeout 重试分支
|
||||
# ---------------------------------------------------------------------- #
|
||||
@@ -390,7 +384,7 @@ def test_async_failure_retry_branch(caplog: pytest.LogCaptureFixture) -> None:
|
||||
# ---------------------------------------------------------------------- #
|
||||
def test_threaded_skips_cached_tasks() -> None:
|
||||
"""threaded 策略下命中缓存的任务应被跳过(覆盖 line 224-230)。"""
|
||||
runs: List[str] = []
|
||||
runs: list[str] = []
|
||||
|
||||
def make(name: str) -> Any:
|
||||
def fn() -> str:
|
||||
@@ -402,15 +396,15 @@ def test_threaded_skips_cached_tasks() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", make("a")),
|
||||
px.TaskSpec("b", make("b"), ("a",)),
|
||||
px.TaskSpec("b", make("b"), depends_on=("a",)),
|
||||
]
|
||||
)
|
||||
backend = px.MemoryBackend()
|
||||
# 第一次运行填充缓存
|
||||
px.run(graph, strategy="thread", max_workers=2, state=backend)
|
||||
_ = px.run(graph, strategy="thread", max_workers=2, state=backend)
|
||||
assert runs == ["a", "b"]
|
||||
# 第二次运行应全部跳过
|
||||
px.run(graph, strategy="thread", max_workers=2, state=backend)
|
||||
_ = px.run(graph, strategy="thread", max_workers=2, state=backend)
|
||||
assert runs == ["a", "b"] # 未再执行
|
||||
|
||||
|
||||
@@ -426,7 +420,7 @@ def test_threaded_all_cached_layer() -> None:
|
||||
|
||||
def test_async_skips_cached_tasks() -> None:
|
||||
"""async 策略下命中缓存的任务应被跳过(覆盖 line 268-274)。"""
|
||||
runs: List[str] = []
|
||||
runs: list[str] = []
|
||||
|
||||
async def make(name: str) -> Any:
|
||||
async def fn() -> str:
|
||||
@@ -447,13 +441,13 @@ def test_async_skips_cached_tasks() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", a),
|
||||
px.TaskSpec("b", b, ("a",)),
|
||||
px.TaskSpec("b", b, depends_on=("a",)),
|
||||
]
|
||||
)
|
||||
backend = px.MemoryBackend()
|
||||
px.run(graph, strategy="async", state=backend)
|
||||
_ = px.run(graph, strategy="async", state=backend)
|
||||
assert runs == ["a", "b"]
|
||||
px.run(graph, strategy="async", state=backend)
|
||||
_ = px.run(graph, strategy="async", state=backend)
|
||||
assert runs == ["a", "b"]
|
||||
|
||||
|
||||
@@ -480,7 +474,7 @@ def test_failure_marks_report_unsuccessful() -> None:
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", boom)])
|
||||
with pytest.raises(px.TaskFailedError):
|
||||
px.run(graph, strategy="sequential")
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
# report 在异常前未返回,但若捕获异常则 success 应为 False
|
||||
# 这里验证 run() 抛异常的行为本身
|
||||
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Tests for executors module edge cases."""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.task import TaskStatus
|
||||
|
||||
# 跨平台的 echo 命令
|
||||
if sys.platform == "win32":
|
||||
ECHO_CMD = ["cmd", "/c", "echo"]
|
||||
else:
|
||||
ECHO_CMD = ["echo"]
|
||||
|
||||
|
||||
def test_execute_sync_with_timeout():
|
||||
"""Test execute task with timeout correctly."""
|
||||
# Note: timeout for Python functions only works in async strategy
|
||||
# For sync functions, timeout is not enforced in sequential strategy
|
||||
# This test verifies that the task runs without timeout error
|
||||
spec = px.TaskSpec("quick", fn=lambda: "result", timeout=10)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
|
||||
# Should succeed without timeout error
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_execute_async_with_timeout():
|
||||
"""Test execute async task with timeout correctly."""
|
||||
|
||||
async def slow_async_function():
|
||||
await asyncio.sleep(2)
|
||||
return "result"
|
||||
|
||||
spec = px.TaskSpec("slow_async", fn=slow_async_function, timeout=0.5)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
|
||||
# This should timeout
|
||||
with pytest.raises(px.TaskFailedError):
|
||||
px.run(graph, strategy="async")
|
||||
|
||||
|
||||
def test_verbose_event_callback_running():
|
||||
"""Test verbose event callback for RUNNING status."""
|
||||
# Create a graph with verbose callback
|
||||
spec = px.TaskSpec("test", fn=lambda: "result", verbose=True)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
report = px.run(graph, strategy="sequential")
|
||||
# Should print without error
|
||||
assert report.success
|
||||
|
||||
|
||||
def test_verbose_run_with_success_lifecycle(capsys):
|
||||
"""Test px.run with verbose=True prints SUCCESS lifecycle."""
|
||||
spec = px.TaskSpec("test", fn=lambda: "result")
|
||||
graph = px.Graph.from_specs([spec])
|
||||
report = px.run(graph, strategy="sequential", verbose=True)
|
||||
assert report.success
|
||||
captured = capsys.readouterr()
|
||||
assert "成功" in captured.out
|
||||
|
||||
|
||||
def test_verbose_run_with_failed_lifecycle(capsys):
|
||||
"""Test px.run with verbose=True prints FAILED lifecycle with error."""
|
||||
|
||||
def raise_error():
|
||||
raise ValueError("test error")
|
||||
|
||||
spec = px.TaskSpec("test", fn=raise_error)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
|
||||
with pytest.raises(px.TaskFailedError):
|
||||
px.run(graph, strategy="sequential", verbose=True)
|
||||
captured = capsys.readouterr()
|
||||
assert "失败" in captured.out
|
||||
assert "test error" in captured.out
|
||||
|
||||
|
||||
def test_verbose_run_with_skipped_lifecycle(capsys):
|
||||
"""Test px.run with verbose=True prints SKIPPED lifecycle."""
|
||||
spec = px.TaskSpec(
|
||||
"test",
|
||||
fn=lambda: "result",
|
||||
conditions=(lambda: False,),
|
||||
)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
report = px.run(graph, strategy="sequential", verbose=True)
|
||||
assert report.success
|
||||
captured = capsys.readouterr()
|
||||
assert "跳过" in captured.out
|
||||
|
||||
|
||||
def test_verbose_run_with_user_callback():
|
||||
"""Test px.run with verbose=True and user callback both called."""
|
||||
events = []
|
||||
|
||||
def on_event(event):
|
||||
events.append(event)
|
||||
|
||||
spec = px.TaskSpec("test", fn=lambda: "result")
|
||||
graph = px.Graph.from_specs([spec])
|
||||
report = px.run(graph, strategy="sequential", verbose=True, on_event=on_event)
|
||||
assert report.success
|
||||
assert len(events) == 1
|
||||
assert events[0].status == px.TaskStatus.SUCCESS
|
||||
|
||||
|
||||
def test_verbose_event_callback_success():
|
||||
"""Test verbose event callback for SUCCESS status."""
|
||||
# Create a graph with verbose callback
|
||||
spec = px.TaskSpec("test", fn=lambda: "result", verbose=True)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
report = px.run(graph, strategy="sequential")
|
||||
# Should print without error
|
||||
assert report.success
|
||||
|
||||
|
||||
def test_verbose_event_callback_failed():
|
||||
"""Test verbose event callback for FAILED status."""
|
||||
# Create a graph with verbose callback and failing task
|
||||
|
||||
def raise_error():
|
||||
raise ValueError("test error")
|
||||
|
||||
spec = px.TaskSpec("test", fn=raise_error, verbose=True)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
|
||||
# Should print without error
|
||||
with pytest.raises(px.TaskFailedError):
|
||||
px.run(graph, strategy="sequential")
|
||||
|
||||
|
||||
def test_verbose_event_callback_skipped():
|
||||
"""Test verbose event callback for SKIPPED status."""
|
||||
# Create a graph with verbose callback and skipped task
|
||||
spec = px.TaskSpec(
|
||||
"test",
|
||||
fn=lambda: "result",
|
||||
conditions=(lambda: False,),
|
||||
verbose=True,
|
||||
)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
report = px.run(graph, strategy="sequential")
|
||||
# Should print without error
|
||||
assert report.success
|
||||
|
||||
|
||||
def test_execute_sync_with_retries():
|
||||
"""Test execute task with retries."""
|
||||
|
||||
call_count = 0
|
||||
|
||||
def failing_function():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count < 3:
|
||||
raise ValueError("temporary error")
|
||||
return "success"
|
||||
|
||||
spec = px.TaskSpec("retry_test", fn=failing_function, retries=3)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
|
||||
# Should succeed after retries
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert report.results["retry_test"].attempts == 3
|
||||
|
||||
|
||||
def test_execute_async_with_retries():
|
||||
"""Test execute async task with retries."""
|
||||
|
||||
call_count = 0
|
||||
|
||||
async def failing_async_function():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count < 3:
|
||||
raise ValueError("temporary error")
|
||||
return "success"
|
||||
|
||||
spec = px.TaskSpec("retry_async_test", fn=failing_async_function, retries=3)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
|
||||
# Should succeed after retries
|
||||
report = px.run(graph, strategy="async")
|
||||
assert report.success
|
||||
assert report.results["retry_async_test"].attempts == 3
|
||||
|
||||
|
||||
def test_execute_sync_skip_on_condition():
|
||||
"""Test execute task skips task when condition is false."""
|
||||
spec = px.TaskSpec(
|
||||
"skip_test",
|
||||
fn=lambda: "result",
|
||||
conditions=(lambda: False,),
|
||||
)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert report.results["skip_test"].status == TaskStatus.SKIPPED
|
||||
|
||||
|
||||
def test_execute_async_skip_on_condition():
|
||||
"""Test execute async task skips task when condition is false."""
|
||||
spec = px.TaskSpec(
|
||||
"skip_async_test",
|
||||
fn=lambda: "result",
|
||||
conditions=(lambda: False,),
|
||||
)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
|
||||
report = px.run(graph, strategy="async")
|
||||
assert report.success
|
||||
assert report.results["skip_async_test"].status == TaskStatus.SKIPPED
|
||||
|
||||
|
||||
def test_execute_sync_with_error():
|
||||
"""Test execute task handles errors correctly."""
|
||||
|
||||
def error_function():
|
||||
raise ValueError("test error")
|
||||
|
||||
spec = px.TaskSpec("error_test", fn=error_function)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
|
||||
with pytest.raises(px.TaskFailedError):
|
||||
px.run(graph, strategy="sequential")
|
||||
|
||||
|
||||
def test_execute_async_with_error():
|
||||
"""Test execute async task handles errors correctly."""
|
||||
|
||||
async def error_async_function():
|
||||
raise ValueError("test error")
|
||||
|
||||
spec = px.TaskSpec("error_async_test", fn=error_async_function)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
|
||||
with pytest.raises(px.TaskFailedError):
|
||||
px.run(graph, strategy="async")
|
||||
+27
-26
@@ -16,8 +16,8 @@ def test_from_specs_builds_graph() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", _fn),
|
||||
px.TaskSpec("b", _fn, ("a",)),
|
||||
px.TaskSpec("c", _fn, ("a", "b")),
|
||||
px.TaskSpec("b", _fn, depends_on=("a",)),
|
||||
px.TaskSpec("c", _fn, depends_on=("a", "b")),
|
||||
]
|
||||
)
|
||||
assert set(graph.names) == {"a", "b", "c"}
|
||||
@@ -30,7 +30,7 @@ def test_from_specs_allows_forward_references() -> None:
|
||||
# b depends on a, but a is declared after b — order should not matter.
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("b", _fn, ("a",)),
|
||||
px.TaskSpec("b", _fn, depends_on=("a",)),
|
||||
px.TaskSpec("a", _fn),
|
||||
]
|
||||
)
|
||||
@@ -39,7 +39,7 @@ def test_from_specs_allows_forward_references() -> None:
|
||||
|
||||
def test_duplicate_task_raises() -> None:
|
||||
with pytest.raises(DuplicateTaskError):
|
||||
px.Graph.from_specs(
|
||||
_ = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", _fn),
|
||||
px.TaskSpec("a", _fn),
|
||||
@@ -49,18 +49,19 @@ def test_duplicate_task_raises() -> None:
|
||||
|
||||
def test_missing_dependency_raises() -> None:
|
||||
with pytest.raises(MissingDependencyError) as exc_info:
|
||||
px.Graph.from_specs([px.TaskSpec("b", _fn, ("a",))])
|
||||
_ = px.Graph.from_specs([px.TaskSpec("b", _fn, depends_on=("a",))])
|
||||
|
||||
assert exc_info.value.task == "b"
|
||||
assert exc_info.value.dependency == "a"
|
||||
|
||||
|
||||
def test_cycle_detection() -> None:
|
||||
with pytest.raises(CycleError):
|
||||
px.Graph.from_specs(
|
||||
_ = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", _fn, ("c",)),
|
||||
px.TaskSpec("b", _fn, ("a",)),
|
||||
px.TaskSpec("c", _fn, ("b",)),
|
||||
px.TaskSpec("a", _fn, depends_on=("c",)),
|
||||
px.TaskSpec("b", _fn, depends_on=("a",)),
|
||||
px.TaskSpec("c", _fn, depends_on=("b",)),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -70,8 +71,8 @@ def test_layers_grouping() -> None:
|
||||
[
|
||||
px.TaskSpec("a", _fn),
|
||||
px.TaskSpec("b", _fn),
|
||||
px.TaskSpec("c", _fn, ("a", "b")),
|
||||
px.TaskSpec("d", _fn, ("c",)),
|
||||
px.TaskSpec("c", _fn, depends_on=("a", "b")),
|
||||
px.TaskSpec("d", _fn, depends_on=("c",)),
|
||||
]
|
||||
)
|
||||
layers = graph.layers()
|
||||
@@ -80,14 +81,14 @@ def test_layers_grouping() -> None:
|
||||
|
||||
def test_self_dependency_rejected() -> None:
|
||||
with pytest.raises(ValueError):
|
||||
px.TaskSpec("a", _fn, ("a",))
|
||||
_ = px.TaskSpec("a", _fn, depends_on=("a",))
|
||||
|
||||
|
||||
def test_to_mermaid() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", _fn),
|
||||
px.TaskSpec("b", _fn, ("a",)),
|
||||
px.TaskSpec("b", _fn, depends_on=("a",)),
|
||||
]
|
||||
)
|
||||
mermaid = graph.to_mermaid()
|
||||
@@ -99,15 +100,15 @@ def test_to_mermaid() -> None:
|
||||
def test_to_mermaid_invalid_orientation() -> None:
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", _fn)])
|
||||
with pytest.raises(ValueError):
|
||||
graph.to_mermaid("XX")
|
||||
_ = graph.to_mermaid("XX")
|
||||
|
||||
|
||||
def test_subgraph_by_tags() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", _fn, tags=("ingest",)),
|
||||
px.TaskSpec("b", _fn, ("a",), tags=("ingest",)),
|
||||
px.TaskSpec("c", _fn, ("b",), tags=("report",)),
|
||||
px.TaskSpec("b", _fn, depends_on=("a",), tags=("ingest",)),
|
||||
px.TaskSpec("c", _fn, depends_on=("b",), tags=("report",)),
|
||||
]
|
||||
)
|
||||
sub = graph.subgraph(["ingest"])
|
||||
@@ -121,8 +122,8 @@ def test_subgraph_by_names() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", _fn),
|
||||
px.TaskSpec("b", _fn, ("a",)),
|
||||
px.TaskSpec("c", _fn, ("b",)),
|
||||
px.TaskSpec("b", _fn, depends_on=("a",)),
|
||||
px.TaskSpec("c", _fn, depends_on=("b",)),
|
||||
]
|
||||
)
|
||||
sub = graph.subgraph_by_names(["a", "b"])
|
||||
@@ -134,14 +135,14 @@ def test_subgraph_by_names() -> None:
|
||||
def test_subgraph_by_names_unknown() -> None:
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", _fn)])
|
||||
with pytest.raises(KeyError):
|
||||
graph.subgraph_by_names(["nope"])
|
||||
_ = graph.subgraph_by_names(["nope"])
|
||||
|
||||
|
||||
def test_describe() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", _fn),
|
||||
px.TaskSpec("b", _fn, ("a",)),
|
||||
px.TaskSpec("b", _fn, depends_on=("a",)),
|
||||
]
|
||||
)
|
||||
desc = graph.describe()
|
||||
@@ -160,14 +161,14 @@ def test_add_chains_and_validates() -> None:
|
||||
assert "a" in graph
|
||||
# 缺失依赖应即时报错
|
||||
with pytest.raises(MissingDependencyError):
|
||||
graph.add(px.TaskSpec("b", _fn, ("missing",)))
|
||||
_ = graph.add(px.TaskSpec("b", _fn, depends_on=("missing",)))
|
||||
|
||||
|
||||
def test_add_duplicate_raises() -> None:
|
||||
graph = px.Graph()
|
||||
graph.add(px.TaskSpec("a", _fn))
|
||||
_ = graph.add(px.TaskSpec("a", _fn))
|
||||
with pytest.raises(DuplicateTaskError):
|
||||
graph.add(px.TaskSpec("a", _fn))
|
||||
_ = graph.add(px.TaskSpec("a", _fn))
|
||||
|
||||
|
||||
def test_all_specs_returns_view() -> None:
|
||||
@@ -182,14 +183,14 @@ def test_spec_accessor() -> None:
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", _fn)])
|
||||
assert graph.spec("a").name == "a"
|
||||
with pytest.raises(KeyError):
|
||||
graph.spec("missing")
|
||||
_ = graph.spec("missing")
|
||||
|
||||
|
||||
def test_dependencies_accessor() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", _fn),
|
||||
px.TaskSpec("b", _fn, ("a",)),
|
||||
px.TaskSpec("b", _fn, depends_on=("a",)),
|
||||
]
|
||||
)
|
||||
assert graph.dependencies("a") == ()
|
||||
@@ -213,7 +214,7 @@ def test_subgraph_preserves_metadata() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", _fn, tags=("x",), retries=3, timeout=5.0),
|
||||
px.TaskSpec("b", _fn, ("a",), tags=("y",)),
|
||||
px.TaskSpec("b", _fn, depends_on=("a",), tags=("y",)),
|
||||
]
|
||||
)
|
||||
sub = graph.subgraph(["x"])
|
||||
|
||||
+43
-37
@@ -1,9 +1,9 @@
|
||||
"""RunReport 测试。"""
|
||||
"""RunReport 测试."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.task import TaskResult, TaskSpec, TaskStatus
|
||||
@@ -16,18 +16,17 @@ def _fn() -> int:
|
||||
def _make_result(
|
||||
name: str = "a",
|
||||
status: TaskStatus = TaskStatus.SUCCESS,
|
||||
value: object = 42,
|
||||
error: Optional[object] = None,
|
||||
value: Any = 42,
|
||||
error: BaseException | None = None,
|
||||
duration: float = 0.5,
|
||||
attempts: int = 1,
|
||||
) -> TaskResult[object]:
|
||||
spec: TaskSpec[object] = TaskSpec[object](name, _fn)
|
||||
) -> TaskResult[Any]:
|
||||
"""构造测试用 TaskResult 实例."""
|
||||
spec: TaskSpec[Any] = TaskSpec[Any](name, _fn)
|
||||
start = datetime(2024, 1, 1, 0, 0, 0)
|
||||
# 用 timedelta 精确表达秒数,避免 int() 截断小数
|
||||
from datetime import timedelta
|
||||
|
||||
end = start + timedelta(seconds=duration) if duration else None
|
||||
return TaskResult[object](
|
||||
return TaskResult[Any](
|
||||
spec=spec,
|
||||
status=status,
|
||||
value=value,
|
||||
@@ -38,27 +37,31 @@ def _make_result(
|
||||
)
|
||||
|
||||
|
||||
def test_getitem_returns_value() -> None:
|
||||
class TestRunReportAccess:
|
||||
"""测试 RunReport 的访问接口."""
|
||||
|
||||
def test_getitem_returns_value(self) -> None:
|
||||
"""report[name] 应返回任务结果值."""
|
||||
report = px.RunReport()
|
||||
report.results["a"] = _make_result("a", value=7)
|
||||
assert report["a"] == 7
|
||||
|
||||
|
||||
def test_result_of_returns_full_result() -> None:
|
||||
def test_result_of_returns_full_result(self) -> None:
|
||||
"""result_of 应返回完整的 TaskResult 对象."""
|
||||
report = px.RunReport()
|
||||
r = _make_result("a")
|
||||
report.results["a"] = r
|
||||
assert report.result_of("a") is r
|
||||
|
||||
|
||||
def test_contains() -> None:
|
||||
def test_contains(self) -> None:
|
||||
"""in 运算符应正确判断任务是否存在."""
|
||||
report = px.RunReport()
|
||||
report.results["a"] = _make_result("a")
|
||||
assert "a" in report
|
||||
assert "b" not in report
|
||||
|
||||
|
||||
def test_iter_and_len() -> None:
|
||||
def test_iter_and_len(self) -> None:
|
||||
"""应支持迭代任务名并返回任务数量."""
|
||||
report = px.RunReport()
|
||||
report.results["a"] = _make_result("a")
|
||||
report.results["b"] = _make_result("b")
|
||||
@@ -66,7 +69,11 @@ def test_iter_and_len() -> None:
|
||||
assert len(report) == 2
|
||||
|
||||
|
||||
def test_summary_success() -> None:
|
||||
class TestRunReportSummary:
|
||||
"""测试 RunReport 的 summary 方法."""
|
||||
|
||||
def test_summary_success(self) -> None:
|
||||
"""应正确汇总成功和跳过的任务."""
|
||||
report = px.RunReport()
|
||||
report.results["a"] = _make_result("a", status=TaskStatus.SUCCESS, duration=1.0)
|
||||
report.results["b"] = _make_result("b", status=TaskStatus.SKIPPED, duration=0.0)
|
||||
@@ -76,26 +83,27 @@ def test_summary_success() -> None:
|
||||
assert s["by_status"] == {"success": 1, "skipped": 1}
|
||||
assert s["total_duration_seconds"] == 1.0
|
||||
|
||||
|
||||
def test_summary_with_none_duration() -> None:
|
||||
"""未开始/未结束的任务 duration 为 None,不应计入总时长。"""
|
||||
def test_summary_with_none_duration(self) -> None:
|
||||
"""未开始/未结束的任务 duration 为 None,不应计入总时长."""
|
||||
report = px.RunReport()
|
||||
spec: TaskSpec[object] = TaskSpec("a", _fn) # type: ignore[arg-type]
|
||||
spec: TaskSpec[Any] = TaskSpec[Any]("a", _fn) # type: ignore[arg-type]
|
||||
report.results["a"] = TaskResult(spec=spec, status=TaskStatus.FAILED)
|
||||
s = report.summary()
|
||||
assert s["total_duration_seconds"] == 0.0
|
||||
|
||||
|
||||
def test_failed_tasks() -> None:
|
||||
def test_failed_tasks(self) -> None:
|
||||
"""failed_tasks 应返回所有失败任务名."""
|
||||
report = px.RunReport()
|
||||
report.results["a"] = _make_result("a", status=TaskStatus.SUCCESS)
|
||||
report.results["b"] = _make_result(
|
||||
"b", status=TaskStatus.FAILED, error=ValueError("x")
|
||||
)
|
||||
report.results["b"] = _make_result("b", status=TaskStatus.FAILED, error=ValueError("x"))
|
||||
assert report.failed_tasks() == ["b"]
|
||||
|
||||
|
||||
def test_describe_success() -> None:
|
||||
class TestRunReportDescribe:
|
||||
"""测试 RunReport 的 describe 方法."""
|
||||
|
||||
def test_describe_success(self) -> None:
|
||||
"""应正确描述成功状态和耗时."""
|
||||
report = px.RunReport()
|
||||
report.results["a"] = _make_result("a", status=TaskStatus.SUCCESS, duration=0.5)
|
||||
desc = report.describe()
|
||||
@@ -103,20 +111,18 @@ def test_describe_success() -> None:
|
||||
assert "a: success" in desc
|
||||
assert "0.500s" in desc
|
||||
|
||||
|
||||
def test_describe_with_error() -> None:
|
||||
def test_describe_with_error(self) -> None:
|
||||
"""应正确描述失败状态和错误信息."""
|
||||
report = px.RunReport(success=False)
|
||||
report.results["a"] = _make_result(
|
||||
"a", status=TaskStatus.FAILED, error=ValueError("boom"), duration=0.1
|
||||
)
|
||||
report.results["a"] = _make_result("a", status=TaskStatus.FAILED, error=ValueError("boom"), duration=0.1)
|
||||
desc = report.describe()
|
||||
assert "success=False" in desc
|
||||
assert "error=ValueError" in desc
|
||||
|
||||
|
||||
def test_describe_no_duration() -> None:
|
||||
def test_describe_no_duration(self) -> None:
|
||||
"""无耗时的任务应显示为 '-'."""
|
||||
report = px.RunReport()
|
||||
spec: TaskSpec[object] = TaskSpec("a", _fn) # type: ignore[arg-type]
|
||||
report.results["a"] = TaskResult(spec=spec, status=TaskStatus.PENDING)
|
||||
spec: TaskSpec[Any] = TaskSpec[Any]("a", _fn) # type: ignore[arg-type]
|
||||
report.results["a"] = TaskResult[Any](spec=spec, status=TaskStatus.PENDING)
|
||||
desc = report.describe()
|
||||
assert "-" in desc # duration 显示为 "-"
|
||||
|
||||
@@ -0,0 +1,649 @@
|
||||
"""Tests for CliRunner: command dispatch, argument parsing, exit codes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx import CliExitCode
|
||||
from pyflowx.errors import TaskFailedError
|
||||
|
||||
# 跨平台的 echo 命令
|
||||
if sys.platform == "win32":
|
||||
ECHO_CMD = ["cmd", "/c", "echo"]
|
||||
else:
|
||||
ECHO_CMD = ["echo"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 辅助工厂
|
||||
# ---------------------------------------------------------------------- #
|
||||
def _echo_graph(name: str = "echo_task", msg: str = "hello") -> px.Graph:
|
||||
"""构造一个单任务 echo 图, 用于执行成功场景."""
|
||||
return px.Graph.from_specs([px.TaskSpec(name, cmd=[*ECHO_CMD, msg])])
|
||||
|
||||
|
||||
def _failing_graph() -> px.Graph:
|
||||
"""构造一个必定失败的单任务图."""
|
||||
return px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"fail",
|
||||
cmd=["python", "-c", "import sys; sys.exit(1)"],
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _multi_task_graph() -> px.Graph:
|
||||
"""构造一个带依赖的多任务图."""
|
||||
return px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", cmd=[*ECHO_CMD, "a"]),
|
||||
px.TaskSpec("b", cmd=[*ECHO_CMD, "b"], depends_on=("a",)),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 构造与校验
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestCliRunnerConstruction:
|
||||
"""测试 CliRunner 的构造与参数校验."""
|
||||
|
||||
def test_requires_at_least_one_command(self) -> None:
|
||||
"""没有命令时应抛出 ValueError."""
|
||||
with pytest.raises(ValueError, match="至少需要一个命令"):
|
||||
_ = px.CliRunner()
|
||||
|
||||
def test_accepts_single_graph(self) -> None:
|
||||
"""单个命令应正常构造."""
|
||||
runner = px.CliRunner(graphs={"clean": _echo_graph()})
|
||||
assert runner.commands == ["clean"]
|
||||
|
||||
def test_accepts_multiple_graphs(self) -> None:
|
||||
"""多个命令应按插入顺序保留."""
|
||||
runner = px.CliRunner(
|
||||
graphs={
|
||||
"clean": _echo_graph("c", "clean"),
|
||||
"build": _echo_graph("b", "build"),
|
||||
"test": _echo_graph("t", "test"),
|
||||
}
|
||||
)
|
||||
assert runner.commands == ["clean", "build", "test"]
|
||||
|
||||
def test_default_strategy_is_sequential(self) -> None:
|
||||
"""默认策略应为 Strategy.SEQUENTIAL."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
assert runner.strategy == "sequential"
|
||||
|
||||
def test_custom_strategy_string(self) -> None:
|
||||
"""应支持通过字符串指定策略."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()}, strategy="thread")
|
||||
assert runner.strategy == "thread"
|
||||
|
||||
def test_custom_strategy_enum(self) -> None:
|
||||
"""应支持通过 Strategy 枚举指定策略."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()}, strategy="async")
|
||||
assert runner.strategy == "async"
|
||||
|
||||
def test_default_verbose_is_true(self) -> None:
|
||||
"""默认 verbose 应为 True."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
assert runner.verbose is True
|
||||
|
||||
def test_custom_verbose_false(self) -> None:
|
||||
"""应支持关闭 verbose."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()}, verbose=False)
|
||||
assert runner.verbose is False
|
||||
|
||||
def test_default_description_is_empty(self) -> None:
|
||||
"""默认描述应为空字符串."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
assert runner.description == ""
|
||||
|
||||
def test_custom_description(self) -> None:
|
||||
"""应支持自定义描述."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()}, description="My CLI")
|
||||
assert runner.description == "My CLI"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 属性与内省
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestCliRunnerProperties:
|
||||
"""测试 CliRunner 的属性访问."""
|
||||
|
||||
def test_commands_returns_list(self) -> None:
|
||||
"""commands 应返回列表."""
|
||||
runner = px.CliRunner({"a": _echo_graph(), "b": _echo_graph()})
|
||||
assert isinstance(runner.commands, list)
|
||||
|
||||
def test_graphs_contains_original_graphs(self) -> None:
|
||||
"""graphs 应包含原始 Graph 实例."""
|
||||
g = _echo_graph()
|
||||
runner = px.CliRunner({"cmd": g})
|
||||
assert runner.graphs["cmd"] is g
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 参数解析
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestCliRunnerParser:
|
||||
"""测试参数解析器."""
|
||||
|
||||
def test_create_parser_returns_argument_parser(self) -> None:
|
||||
"""create_parser 应返回 ArgumentParser."""
|
||||
from argparse import ArgumentParser
|
||||
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
parser = runner.create_parser()
|
||||
assert isinstance(parser, ArgumentParser)
|
||||
|
||||
def test_parser_has_command_argument(self) -> None:
|
||||
"""解析器应有 command 位置参数."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
parser = runner.create_parser()
|
||||
parsed = parser.parse_args(["clean"])
|
||||
assert parsed.command == "clean"
|
||||
|
||||
def test_parser_command_is_optional(self) -> None:
|
||||
"""command 应为可选参数."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
parser = runner.create_parser()
|
||||
parsed = parser.parse_args([])
|
||||
assert parsed.command is None
|
||||
|
||||
def test_parser_has_strategy_option(self) -> None:
|
||||
"""解析器应有 --strategy 选项."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
parser = runner.create_parser()
|
||||
parsed = parser.parse_args(["clean", "--strategy", "thread"])
|
||||
assert parsed.strategy == "thread"
|
||||
|
||||
def test_parser_strategy_default(self) -> None:
|
||||
"""--strategy 默认值应与构造时一致."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()}, "async")
|
||||
parser = runner.create_parser()
|
||||
parsed = parser.parse_args(["clean"])
|
||||
assert parsed.strategy == "sequential"
|
||||
|
||||
def test_parser_has_dry_run_flag(self) -> None:
|
||||
"""解析器应有 --dry-run 标志."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
parser = runner.create_parser()
|
||||
parsed = parser.parse_args(["clean", "--dry-run"])
|
||||
assert parsed.dry_run is True
|
||||
|
||||
def test_parser_dry_run_default_false(self) -> None:
|
||||
"""--dry-run 默认为 False."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
parser = runner.create_parser()
|
||||
parsed = parser.parse_args(["clean"])
|
||||
assert parsed.dry_run is False
|
||||
|
||||
def test_parser_has_list_flag(self) -> None:
|
||||
"""解析器应有 --list 标志."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
parser = runner.create_parser()
|
||||
parsed = parser.parse_args(["--list"])
|
||||
assert parsed.list is True
|
||||
|
||||
def test_parser_has_quiet_flag(self) -> None:
|
||||
"""解析器应有 --quiet 标志."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
parser = runner.create_parser()
|
||||
parsed = parser.parse_args(["clean", "--quiet"])
|
||||
assert parsed.quiet is True
|
||||
|
||||
def test_parser_quiet_default_false(self) -> None:
|
||||
"""--quiet 默认为 False."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
parser = runner.create_parser()
|
||||
parsed = parser.parse_args(["clean"])
|
||||
assert parsed.quiet is False
|
||||
|
||||
def test_format_commands_help_contains_all_commands(self) -> None:
|
||||
"""帮助文本应包含所有命令."""
|
||||
runner = px.CliRunner(
|
||||
{"clean": _echo_graph("c", "clean"), "build": _echo_graph("b", "build")},
|
||||
)
|
||||
help_text = runner._format_commands_help()
|
||||
assert "clean" in help_text
|
||||
assert "build" in help_text
|
||||
assert "可用命令" in help_text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 执行: 成功路径
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestCliRunnerRunSuccess:
|
||||
"""测试 CliRunner.run 的成功执行路径."""
|
||||
|
||||
def test_run_valid_command_returns_zero(self) -> None:
|
||||
"""有效命令执行成功应返回 0."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
exit_code = runner.run(["clean"])
|
||||
assert exit_code == CliExitCode.SUCCESS.value
|
||||
|
||||
def test_run_executes_correct_graph(self) -> None:
|
||||
"""应执行用户指定的命令对应的图."""
|
||||
executed: list[str] = []
|
||||
|
||||
def track_a() -> None:
|
||||
executed.append("a")
|
||||
|
||||
def track_b() -> None:
|
||||
executed.append("b")
|
||||
|
||||
runner = px.CliRunner(
|
||||
{
|
||||
"a": px.Graph.from_specs([px.TaskSpec("a", track_a)]),
|
||||
"b": px.Graph.from_specs([px.TaskSpec("b", track_b)]),
|
||||
}
|
||||
)
|
||||
_ = runner.run(["b"])
|
||||
assert executed == ["b"]
|
||||
|
||||
def test_run_multi_task_graph(self) -> None:
|
||||
"""应能执行带依赖的多任务图."""
|
||||
runner = px.CliRunner({"multi": _multi_task_graph()})
|
||||
exit_code = runner.run(["multi"])
|
||||
assert exit_code == CliExitCode.SUCCESS.value
|
||||
|
||||
def test_run_with_strategy_override(self) -> None:
|
||||
"""应支持通过 --strategy 覆盖默认策略."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
exit_code = runner.run(["echo", "--strategy", "thread"])
|
||||
assert exit_code == CliExitCode.SUCCESS.value
|
||||
|
||||
def test_run_with_dry_run(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""--dry-run 应只打印计划不执行."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
exit_code = runner.run(["echo", "--dry-run"])
|
||||
assert exit_code == CliExitCode.SUCCESS.value
|
||||
captured = capsys.readouterr()
|
||||
assert "Dry run" in captured.out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 执行: verbose 模式
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestCliRunnerVerbose:
|
||||
"""测试 verbose 模式."""
|
||||
|
||||
def test_verbose_default_prints_lifecycle(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""默认 verbose=True 应打印任务生命周期."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
_ = runner.run(["echo"])
|
||||
captured = capsys.readouterr()
|
||||
# verbose 模式下应打印任务生命周期
|
||||
assert "[verbose]" in captured.out
|
||||
|
||||
def test_quiet_flag_disables_verbose(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""--quiet 应关闭 verbose 输出."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
_ = runner.run(["echo", "--quiet"])
|
||||
captured = capsys.readouterr()
|
||||
# quiet 模式下不应有 [verbose] 前缀的输出
|
||||
assert "[verbose]" not in captured.out
|
||||
|
||||
def test_verbose_false_constructor_disables_verbose(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""构造时 verbose=False 应关闭 verbose 输出."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()}, verbose=False)
|
||||
_ = runner.run(["echo"])
|
||||
captured = capsys.readouterr()
|
||||
assert "[verbose]" not in captured.out
|
||||
|
||||
def test_verbose_prints_command_for_cmd_task(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose 模式下 cmd 任务应打印执行的命令."""
|
||||
runner = px.CliRunner({"echo": _echo_graph(msg="verbose-test")})
|
||||
_ = runner.run(["echo"])
|
||||
captured = capsys.readouterr()
|
||||
# 应打印执行的命令
|
||||
assert "执行命令" in captured.out or "执行 Shell" in captured.out
|
||||
# 应打印返回码
|
||||
assert "返回码" in captured.out
|
||||
|
||||
def test_verbose_prints_success_lifecycle(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose 模式下成功任务应打印成功信息."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
_ = runner.run(["echo"])
|
||||
captured = capsys.readouterr()
|
||||
assert "成功" in captured.out
|
||||
|
||||
def test_verbose_prints_skip_lifecycle(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose 模式下跳过的任务应打印跳过信息."""
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"skip_me",
|
||||
cmd=[*ECHO_CMD, "skip"],
|
||||
conditions=(lambda: False,),
|
||||
),
|
||||
]
|
||||
)
|
||||
runner = px.CliRunner({"skip": graph})
|
||||
_ = runner.run(["skip"])
|
||||
captured = capsys.readouterr()
|
||||
assert "跳过" in captured.out
|
||||
|
||||
def test_verbose_prints_failure_lifecycle(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose 模式下失败任务应打印失败信息."""
|
||||
runner = px.CliRunner({"fail": _failing_graph()})
|
||||
_ = runner.run(["fail"])
|
||||
captured = capsys.readouterr()
|
||||
# 失败信息可能出现在 stdout (verbose) 或 stderr (PyFlowXError)
|
||||
combined = captured.out + captured.err
|
||||
assert "失败" in combined or "错误" in combined
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 执行: 失败路径
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestCliRunnerRunFailure:
|
||||
"""测试 CliRunner.run 的失败执行路径."""
|
||||
|
||||
def test_run_unknown_command_returns_failure(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""未知命令应返回 1 并打印错误."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
exit_code = runner.run(["unknown"])
|
||||
assert exit_code == CliExitCode.FAILURE.value
|
||||
captured = capsys.readouterr()
|
||||
assert "未知命令" in captured.err
|
||||
assert "clean" in captured.err
|
||||
|
||||
def test_run_no_command_returns_failure(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""无命令时应返回 1 并打印帮助."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
exit_code = runner.run([])
|
||||
assert exit_code == CliExitCode.FAILURE.value
|
||||
captured = capsys.readouterr()
|
||||
assert "可用命令" in captured.out or "可用命令" in captured.err
|
||||
|
||||
def test_run_failing_task_returns_failure(self) -> None:
|
||||
"""任务失败时应返回 1."""
|
||||
runner = px.CliRunner({"fail": _failing_graph()})
|
||||
exit_code = runner.run(["fail"])
|
||||
assert exit_code == CliExitCode.FAILURE.value
|
||||
|
||||
def test_run_failing_task_prints_error(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""任务失败时应打印错误信息."""
|
||||
runner = px.CliRunner({"fail": _failing_graph()})
|
||||
_ = runner.run(["fail"])
|
||||
captured = capsys.readouterr()
|
||||
# PyFlowXError 信息应输出到 stderr
|
||||
assert "错误" in captured.err or "失败" in captured.err
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 执行: --list 选项
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestCliRunnerList:
|
||||
"""测试 --list 选项."""
|
||||
|
||||
def test_list_returns_success(self) -> None:
|
||||
"""--list 应返回 0."""
|
||||
runner = px.CliRunner({"clean": _echo_graph(), "build": _echo_graph()})
|
||||
exit_code = runner.run(["--list"])
|
||||
assert exit_code == CliExitCode.SUCCESS.value
|
||||
|
||||
def test_list_prints_all_commands(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""--list 应打印所有命令."""
|
||||
runner = px.CliRunner(
|
||||
{
|
||||
"clean": _echo_graph("c", "clean"),
|
||||
"build": _echo_graph("b", "build"),
|
||||
"test": _echo_graph("t", "test"),
|
||||
}
|
||||
)
|
||||
_ = runner.run(["--list"])
|
||||
captured = capsys.readouterr()
|
||||
assert "clean" in captured.out
|
||||
assert "build" in captured.out
|
||||
assert "test" in captured.out
|
||||
|
||||
def test_list_does_not_execute_any_graph(self) -> None:
|
||||
"""--list 不应执行任何图."""
|
||||
executed: list[str] = []
|
||||
|
||||
def track() -> None:
|
||||
executed.append("ran")
|
||||
|
||||
runner = px.CliRunner({"a": px.Graph.from_specs([px.TaskSpec("a", track)])})
|
||||
_ = runner.run(["--list"])
|
||||
assert executed == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 错误处理
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestCliRunnerErrorHandling:
|
||||
"""测试错误处理."""
|
||||
|
||||
def test_keyboard_interrupt_returns_130(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""KeyboardInterrupt 应返回 130."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
|
||||
def raise_interrupt(*_args: Any, **_kwargs: Any) -> None:
|
||||
raise KeyboardInterrupt
|
||||
|
||||
with patch("pyflowx.runner.run", side_effect=raise_interrupt):
|
||||
exit_code = runner.run(["echo"])
|
||||
assert exit_code == CliExitCode.INTERRUPTED.value
|
||||
captured = capsys.readouterr()
|
||||
assert "取消" in captured.err
|
||||
|
||||
def test_pyflowx_error_returns_failure(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""PyFlowXError 应返回 1."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
|
||||
def raise_error(*_args: Any, **_kwargs: Any) -> None:
|
||||
raise TaskFailedError("echo", RuntimeError("boom"), 1)
|
||||
|
||||
with patch("pyflowx.runner.run", side_effect=raise_error):
|
||||
exit_code = runner.run(["echo"])
|
||||
assert exit_code == CliExitCode.FAILURE.value
|
||||
captured = capsys.readouterr()
|
||||
assert "错误" in captured.err
|
||||
|
||||
def test_generic_exception_propagates(self) -> None:
|
||||
"""非 PyFlowXError 的异常应向上传播."""
|
||||
|
||||
class CustomError(Exception):
|
||||
pass
|
||||
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
|
||||
def raise_custom(*_args: Any, **_kwargs: Any) -> None:
|
||||
raise CustomError("unexpected")
|
||||
|
||||
with patch("pyflowx.runner.run", side_effect=raise_custom), pytest.raises(CustomError):
|
||||
_ = runner.run(["echo"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# run_cli
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestCliRunnerRunCli:
|
||||
"""测试 run_cli 方法."""
|
||||
|
||||
def test_run_cli_calls_sys_exit(self) -> None:
|
||||
"""run_cli 应调用 sys.exit."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
runner.run_cli(["echo"])
|
||||
assert exc_info.value.code == CliExitCode.SUCCESS.value
|
||||
|
||||
def test_run_cli_exit_code_on_failure(self) -> None:
|
||||
"""run_cli 失败时应以非零码退出."""
|
||||
runner = px.CliRunner({"fail": _failing_graph()})
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
runner.run_cli(["fail"])
|
||||
assert exc_info.value.code == CliExitCode.FAILURE.value
|
||||
|
||||
def test_run_cli_no_args_uses_sys_argv(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""run_cli 无参数时应使用 sys.argv."""
|
||||
monkeypatch.setattr(sys, "argv", ["pymake", "echo"])
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
runner.run_cli()
|
||||
assert exc_info.value.code == CliExitCode.SUCCESS.value
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 退出码枚举
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestCliExitCode:
|
||||
"""测试 CliExitCode 枚举."""
|
||||
|
||||
def test_success_is_zero(self) -> None:
|
||||
assert CliExitCode.SUCCESS.value == 0
|
||||
|
||||
def test_failure_is_one(self) -> None:
|
||||
assert CliExitCode.FAILURE.value == 1
|
||||
|
||||
def test_interrupted_is_130(self) -> None:
|
||||
assert CliExitCode.INTERRUPTED.value == 130
|
||||
|
||||
def test_exit_codes_are_distinct(self) -> None:
|
||||
values = {e.value for e in CliExitCode}
|
||||
assert len(values) == 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 集成测试
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestCliRunnerIntegration:
|
||||
"""集成测试: CliRunner + Graph + TaskSpec + 条件."""
|
||||
|
||||
def test_condition_skipped_command_succeeds(self) -> None:
|
||||
"""条件不满足时任务跳过, 整体仍成功."""
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"skip_me",
|
||||
cmd=[*ECHO_CMD, "should not run"],
|
||||
conditions=(lambda: False,),
|
||||
),
|
||||
]
|
||||
)
|
||||
runner = px.CliRunner({"skip": graph})
|
||||
exit_code = runner.run(["skip"])
|
||||
assert exit_code == CliExitCode.SUCCESS.value
|
||||
|
||||
def test_condition_met_command_succeeds(self) -> None:
|
||||
"""条件满足时任务执行, 整体成功."""
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"run_me",
|
||||
cmd=[*ECHO_CMD, "should run"],
|
||||
conditions=(lambda: True,),
|
||||
),
|
||||
]
|
||||
)
|
||||
runner = px.CliRunner({"run": graph})
|
||||
exit_code = runner.run(["run"])
|
||||
assert exit_code == CliExitCode.SUCCESS.value
|
||||
|
||||
def test_diamond_dependency_graph(self) -> None:
|
||||
"""菱形依赖图应正确执行."""
|
||||
order: list[str] = []
|
||||
|
||||
def make(name: str) -> Any:
|
||||
def fn() -> str:
|
||||
order.append(name)
|
||||
return name
|
||||
|
||||
return fn
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", make("a")),
|
||||
px.TaskSpec("b", make("b"), depends_on=("a",)),
|
||||
px.TaskSpec("c", make("c"), depends_on=("a",)),
|
||||
px.TaskSpec("d", make("d"), depends_on=("b", "c")),
|
||||
]
|
||||
)
|
||||
runner = px.CliRunner({"diamond": graph})
|
||||
exit_code = runner.run(["diamond"])
|
||||
assert exit_code == CliExitCode.SUCCESS.value
|
||||
assert order == ["a", "b", "c", "d"]
|
||||
|
||||
def test_mixed_fn_and_cmd_commands(self) -> None:
|
||||
"""混合 fn 和 cmd 的命令应都能执行."""
|
||||
runner = px.CliRunner(
|
||||
{
|
||||
"fn_cmd": px.Graph.from_specs([px.TaskSpec("fn", fn=lambda: "fn-result")]),
|
||||
"cmd_cmd": px.Graph.from_specs([px.TaskSpec("cmd", cmd=[*ECHO_CMD, "cmd-result"])]),
|
||||
}
|
||||
)
|
||||
assert runner.run(["fn_cmd"]) == CliExitCode.SUCCESS.value
|
||||
assert runner.run(["cmd_cmd"]) == CliExitCode.SUCCESS.value
|
||||
|
||||
def test_command_with_cwd(self) -> None:
|
||||
"""带 cwd 的命令应正确执行."""
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
if sys.platform == "win32":
|
||||
ls_cmd = ["cmd", "/c", "dir"]
|
||||
else:
|
||||
ls_cmd = ["ls"]
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("ls", cmd=ls_cmd, cwd=Path(tmpdir))])
|
||||
runner = px.CliRunner({"ls": graph})
|
||||
exit_code = runner.run(["ls"])
|
||||
assert exit_code == CliExitCode.SUCCESS.value
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 构造校验 (补充覆盖)
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestCliRunnerConstructionValidation:
|
||||
"""测试 CliRunner 的构造校验 (补充覆盖)."""
|
||||
|
||||
def test_non_graph_value_raises_type_error(self) -> None:
|
||||
"""非 Graph 值应抛出 TypeError (覆盖 runner.py line 119)."""
|
||||
with pytest.raises(TypeError, match="必须是 Graph 实例"):
|
||||
_ = px.CliRunner(graphs={"bad": "not a graph"}) # type: ignore[dict-item]
|
||||
|
||||
def test_non_graph_value_dict_raises_type_error(self) -> None:
|
||||
"""dict 中包含非 Graph 值应抛出 TypeError."""
|
||||
with pytest.raises(TypeError, match="必须是 Graph 实例"):
|
||||
_ = px.CliRunner(graphs={"good": _echo_graph(), "bad": 123}) # type: ignore[dict-item]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# _apply_verbose_to_graph (补充覆盖)
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestApplyVerboseToGraph:
|
||||
"""测试 _apply_verbose_to_graph 函数 (补充覆盖)."""
|
||||
|
||||
def test_specs_with_matching_verbose_are_kept(self) -> None:
|
||||
"""spec.verbose 已与目标值匹配时应保留原 spec (覆盖 runner.py line 57)."""
|
||||
from pyflowx.runner import _apply_verbose_to_graph
|
||||
|
||||
# 创建 verbose=True 的 spec
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", cmd=[*ECHO_CMD, "a"], verbose=True)])
|
||||
# 应用 verbose=True, spec.verbose 已匹配, 应保留原 spec
|
||||
new_graph = _apply_verbose_to_graph(graph, verbose=True)
|
||||
new_spec = new_graph.spec("a")
|
||||
assert new_spec.verbose is True
|
||||
|
||||
def test_specs_with_non_matching_verbose_are_replaced(self) -> None:
|
||||
"""spec.verbose 与目标值不匹配时应替换 (覆盖 else 分支)."""
|
||||
from pyflowx.runner import _apply_verbose_to_graph
|
||||
|
||||
# 创建 verbose=False 的 spec
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", cmd=[*ECHO_CMD, "a"], verbose=False)])
|
||||
# 应用 verbose=True, spec.verbose 不匹配, 应替换
|
||||
new_graph = _apply_verbose_to_graph(graph, verbose=True)
|
||||
new_spec = new_graph.spec("a")
|
||||
assert new_spec.verbose is True
|
||||
+26
-19
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@@ -13,6 +14,14 @@ from pyflowx.errors import StorageError
|
||||
from pyflowx.storage import JSONBackend, MemoryBackend, StateBackend, resolve_backend
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tmp_json(tmp_path: Path) -> Path:
|
||||
"""模拟临时 JSON 文件。"""
|
||||
path = tmp_path / "state.json"
|
||||
path.touch()
|
||||
return path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# MemoryBackend
|
||||
# ---------------------------------------------------------------------- #
|
||||
@@ -39,7 +48,7 @@ def test_memory_backend_get_missing_raises() -> None:
|
||||
# ---------------------------------------------------------------------- #
|
||||
def test_json_backend_save_and_load() -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "state.json")
|
||||
path = str(Path(tmp) / "state.json")
|
||||
b = JSONBackend(path)
|
||||
b.save("a", {"x": 1})
|
||||
b.save("b", [1, 2, 3])
|
||||
@@ -53,20 +62,20 @@ def test_json_backend_save_and_load() -> None:
|
||||
|
||||
def test_json_backend_clear() -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "state.json")
|
||||
path = str(Path(tmp) / "state.json")
|
||||
b = JSONBackend(path)
|
||||
b.save("a", 1)
|
||||
b.clear()
|
||||
assert not b.has("a")
|
||||
# 文件应被写入空 dict
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
assert json.load(fh) == {}
|
||||
|
||||
|
||||
def test_json_backend_nonexistent_file_starts_empty() -> None:
|
||||
"""文件不存在时应正常初始化为空。"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "absent.json")
|
||||
path = str(Path(tmp) / "absent.json")
|
||||
b = JSONBackend(path)
|
||||
assert dict(b.load()) == {}
|
||||
assert not b.has("anything")
|
||||
@@ -75,7 +84,7 @@ def test_json_backend_nonexistent_file_starts_empty() -> None:
|
||||
def test_json_backend_non_serialisable_raises() -> None:
|
||||
"""不可 JSON 序列化的值应抛 StorageError,且不污染内存状态。"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "state.json")
|
||||
path = str(Path(tmp) / "state.json")
|
||||
b = JSONBackend(path)
|
||||
with pytest.raises(StorageError):
|
||||
b.save("a", object()) # object() 不可序列化
|
||||
@@ -91,12 +100,12 @@ def test_json_backend_flush_type_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import json as _json
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "state.json")
|
||||
path = str(Path(tmp) / "state.json")
|
||||
b = JSONBackend(path)
|
||||
|
||||
original_dump = _json.dump
|
||||
|
||||
def flaky_dump(*args: Any, **kwargs: Any) -> None:
|
||||
def flaky_dump(*_args: Any, **_kwargs: Any) -> None:
|
||||
raise TypeError("simulated flush failure")
|
||||
|
||||
monkeypatch.setattr(_json, "dump", flaky_dump)
|
||||
@@ -109,15 +118,15 @@ def test_json_backend_flush_type_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_json_backend_flush_os_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""_flush 时 OSError 应转为 StorageError。"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "state.json")
|
||||
path = str(Path(tmp) / "state.json")
|
||||
b = JSONBackend(path)
|
||||
|
||||
original_replace = os.replace
|
||||
|
||||
def fail_replace(*args: Any, **kwargs: Any) -> None:
|
||||
def fail_replace(*_args: Any, **_kwargs: Any) -> None:
|
||||
raise OSError("simulated os.replace failure")
|
||||
|
||||
monkeypatch.setattr(os, "replace", fail_replace)
|
||||
monkeypatch.setattr(Path, "replace", fail_replace)
|
||||
with pytest.raises(StorageError, match="cannot write"):
|
||||
b.save("a", 1)
|
||||
monkeypatch.setattr(os, "replace", original_replace)
|
||||
@@ -126,20 +135,18 @@ def test_json_backend_flush_os_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_json_backend_corrupt_file_raises() -> None:
|
||||
"""损坏的 JSON 文件应抛 StorageError。"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "state.json")
|
||||
path = str(Path(tmp) / "state.json")
|
||||
with open(path, "w", encoding="utf-8") as fh:
|
||||
fh.write("{not valid json")
|
||||
_ = fh.write("{not valid json")
|
||||
with pytest.raises(StorageError):
|
||||
JSONBackend(path)
|
||||
_ = JSONBackend(path)
|
||||
|
||||
|
||||
def test_json_backend_non_dict_content_ignored() -> None:
|
||||
def test_json_backend_non_dict_content_ignored(tmp_path: Path) -> None:
|
||||
"""文件内容是合法 JSON 但非 dict 时应被忽略(保持空)。"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "state.json")
|
||||
with open(path, "w", encoding="utf-8") as fh:
|
||||
json.dump([1, 2, 3], fh) # list 而非 dict
|
||||
b = JSONBackend(path)
|
||||
path = tmp_path / "state.json"
|
||||
_ = path.write_text(json.dumps([1, 2, 3])) # list 而非 dict
|
||||
b = JSONBackend(str(path))
|
||||
assert dict(b.load()) == {}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Tests for task module edge cases."""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.task import TaskSpec
|
||||
|
||||
# 跨平台的 echo 命令
|
||||
if sys.platform == "win32":
|
||||
ECHO_CMD = ["cmd", "/c", "echo"]
|
||||
else:
|
||||
ECHO_CMD = ["echo"]
|
||||
|
||||
|
||||
def test_taskspec_wrap_cmd_with_list():
|
||||
"""Test TaskSpec._wrap_cmd with command list."""
|
||||
spec = TaskSpec("test", cmd=[*ECHO_CMD, "hello"])
|
||||
wrapped_fn = spec.effective_fn
|
||||
assert wrapped_fn is not None
|
||||
|
||||
|
||||
def test_taskspec_wrap_cmd_with_string():
|
||||
"""Test TaskSpec._wrap_cmd with command string."""
|
||||
if sys.platform == "win32":
|
||||
cmd_str = "cmd /c echo hello"
|
||||
else:
|
||||
cmd_str = "echo hello"
|
||||
spec = TaskSpec("test", cmd=cmd_str)
|
||||
wrapped_fn = spec.effective_fn
|
||||
assert wrapped_fn is not None
|
||||
|
||||
|
||||
def test_taskspec_wrap_cmd_with_timeout():
|
||||
"""Test TaskSpec._wrap_cmd with timeout."""
|
||||
spec = TaskSpec("test", cmd=[*ECHO_CMD, "hello"], timeout=0.1)
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
# Should not raise timeout error for quick command
|
||||
result = wrapped_fn()
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_taskspec_wrap_cmd_with_cwd():
|
||||
"""Test TaskSpec._wrap_cmd with working directory."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
spec = TaskSpec("test", cmd=[*ECHO_CMD, "hello"], cwd=Path(tmpdir))
|
||||
wrapped_fn = spec.effective_fn
|
||||
result = wrapped_fn()
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_taskspec_wrap_cmd_verbose():
|
||||
"""Test TaskSpec._wrap_cmd with verbose=True."""
|
||||
spec = TaskSpec("test", cmd=[*ECHO_CMD, "hello"], verbose=True)
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
# Should print verbose output
|
||||
result = wrapped_fn()
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_taskspec_wrap_cmd_error():
|
||||
"""Test TaskSpec._wrap_cmd handles command error."""
|
||||
spec = TaskSpec("test", cmd=["python", "-c", "import sys; sys.exit(1)"])
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
with pytest.raises(RuntimeError, match="命令执行失败"):
|
||||
_ = wrapped_fn()
|
||||
|
||||
|
||||
def test_taskspec_wrap_cmd_file_not_found():
|
||||
"""Test TaskSpec._wrap_cmd handles file not found."""
|
||||
spec = TaskSpec("test", cmd=["nonexistent_command"])
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
with pytest.raises(RuntimeError, match="命令未找到"):
|
||||
_ = wrapped_fn()
|
||||
|
||||
|
||||
def test_taskspec_wrap_cmd_shell_file_not_found():
|
||||
"""Test TaskSpec._wrap_cmd handles shell command file not found."""
|
||||
spec = TaskSpec("test", cmd="nonexistent_shell_command")
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
# Shell commands don't raise FileNotFoundError
|
||||
# They just return non-zero exit code
|
||||
with pytest.raises(RuntimeError):
|
||||
_ = wrapped_fn()
|
||||
|
||||
|
||||
def test_taskspec_no_fn_no_cmd():
|
||||
"""Test TaskSpec raises error when no fn or cmd."""
|
||||
with pytest.raises(ValueError, match="必须提供 fn 或 cmd 参数"):
|
||||
_ = TaskSpec("test")
|
||||
|
||||
|
||||
def test_taskspec_conditions_check():
|
||||
"""Test TaskSpec.should_execute with conditions."""
|
||||
spec = px.TaskSpec(
|
||||
"test",
|
||||
fn=lambda: "result",
|
||||
conditions=(lambda: True,),
|
||||
)
|
||||
|
||||
assert spec.should_execute() is True
|
||||
|
||||
|
||||
def test_taskspec_conditions_false():
|
||||
"""Test TaskSpec.should_execute with false conditions."""
|
||||
spec = px.TaskSpec(
|
||||
"test",
|
||||
fn=lambda: "result",
|
||||
conditions=(lambda: False,),
|
||||
)
|
||||
|
||||
assert spec.should_execute() is False
|
||||
|
||||
|
||||
def test_taskspec_conditions_multiple():
|
||||
"""Test TaskSpec.should_execute with multiple conditions."""
|
||||
spec = px.TaskSpec(
|
||||
"test",
|
||||
fn=lambda: "result",
|
||||
conditions=(lambda: True, lambda: True, lambda: True),
|
||||
)
|
||||
|
||||
assert spec.should_execute() is True
|
||||
|
||||
|
||||
def test_taskspec_conditions_multiple_one_false():
|
||||
"""Test TaskSpec.should_execute with one false condition."""
|
||||
spec = px.TaskSpec(
|
||||
"test",
|
||||
fn=lambda: "result",
|
||||
conditions=(lambda: True, lambda: False, lambda: True),
|
||||
)
|
||||
|
||||
assert spec.should_execute() is False
|
||||
|
||||
|
||||
def test_taskspec_list_cmd_timeout_mocked():
|
||||
"""Test TaskSpec._wrap_cmd handles list command timeout (mocked)."""
|
||||
spec = TaskSpec("test", cmd=["sleep", "10"], timeout=0.1)
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
with patch(
|
||||
"subprocess.run", side_effect=subprocess.TimeoutExpired(cmd=["sleep", "10"], timeout=0.1)
|
||||
), pytest.raises(RuntimeError, match="命令执行超时"):
|
||||
_ = wrapped_fn()
|
||||
|
||||
|
||||
def test_taskspec_shell_cmd_timeout_mocked():
|
||||
"""Test TaskSpec._wrap_cmd handles shell command timeout (mocked)."""
|
||||
spec = TaskSpec("test", cmd="sleep 10", timeout=0.1)
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
with patch("subprocess.run", side_effect=subprocess.TimeoutExpired(cmd="sleep 10", timeout=0.1)), pytest.raises(
|
||||
RuntimeError, match="Shell 命令执行超时"
|
||||
):
|
||||
_ = wrapped_fn()
|
||||
|
||||
|
||||
def test_taskspec_shell_cmd_file_not_found_mocked():
|
||||
"""Test TaskSpec._wrap_cmd handles shell command FileNotFoundError (mocked)."""
|
||||
spec = TaskSpec("test", cmd="nonexistent_shell_command")
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
with patch("subprocess.run", side_effect=FileNotFoundError("not found")), pytest.raises(
|
||||
RuntimeError, match="Shell 命令未找到"
|
||||
):
|
||||
_ = wrapped_fn()
|
||||
|
||||
|
||||
def test_taskspec_shell_cmd_with_cwd_verbose(capsys):
|
||||
"""Test TaskSpec._wrap_cmd with shell command, cwd and verbose=True."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
if sys.platform == "win32":
|
||||
shell_cmd = "cmd /c echo hello"
|
||||
else:
|
||||
shell_cmd = "echo hello"
|
||||
spec = TaskSpec("test", cmd=shell_cmd, cwd=Path(tmpdir), verbose=True)
|
||||
wrapped_fn = spec.effective_fn
|
||||
result = wrapped_fn()
|
||||
assert result is None
|
||||
captured = capsys.readouterr()
|
||||
assert "执行 Shell" in captured.out
|
||||
assert "工作目录" in captured.out
|
||||
|
||||
|
||||
def test_taskspec_list_cmd_os_error_mocked():
|
||||
"""Test TaskSpec._wrap_cmd handles list command OSError (mocked)."""
|
||||
spec = TaskSpec("test", cmd=["ls"])
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
with patch("subprocess.run", side_effect=OSError("os error")), pytest.raises(RuntimeError, match="命令执行异常"):
|
||||
_ = wrapped_fn()
|
||||
|
||||
|
||||
def test_taskspec_shell_cmd_os_error_mocked():
|
||||
"""Test TaskSpec._wrap_cmd handles shell command OSError (mocked)."""
|
||||
spec = TaskSpec("test", cmd="ls")
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
with patch("subprocess.run", side_effect=OSError("os error")), pytest.raises(
|
||||
RuntimeError, match="Shell 命令执行异常"
|
||||
):
|
||||
_ = wrapped_fn()
|
||||
@@ -0,0 +1,506 @@
|
||||
"""测试 TaskSpec 的命令和条件执行功能."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import (
|
||||
IS_LINUX,
|
||||
IS_MACOS,
|
||||
IS_WINDOWS,
|
||||
BuiltinConditions,
|
||||
)
|
||||
|
||||
# 跨平台的 echo 命令
|
||||
if sys.platform == "win32":
|
||||
ECHO_CMD = ["cmd", "/c", "echo"]
|
||||
else:
|
||||
ECHO_CMD = ["echo"]
|
||||
|
||||
|
||||
def test_taskspec_with_cmd_list():
|
||||
"""测试使用命令列表的 TaskSpec."""
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("echo_test", cmd=[*ECHO_CMD, "hello"]),
|
||||
]
|
||||
)
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert "echo_test" in report.results
|
||||
assert report.results["echo_test"].status == px.TaskStatus.SUCCESS
|
||||
|
||||
|
||||
def test_taskspec_with_cmd_string():
|
||||
"""测试使用 shell 命令字符串的 TaskSpec."""
|
||||
if sys.platform == "win32":
|
||||
shell_cmd = 'cmd /c "echo hello from shell"'
|
||||
else:
|
||||
shell_cmd = "echo 'hello from shell'"
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("shell_test", cmd=shell_cmd),
|
||||
]
|
||||
)
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert "shell_test" in report.results
|
||||
assert report.results["shell_test"].status == px.TaskStatus.SUCCESS
|
||||
|
||||
|
||||
def test_taskspec_with_conditions_skip():
|
||||
"""测试条件不满足时任务被跳过."""
|
||||
|
||||
# 创建一个永远不会满足的条件
|
||||
def never_true():
|
||||
return False
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"should_skip",
|
||||
cmd=[*ECHO_CMD, "this should not run"],
|
||||
conditions=(never_true,),
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert "should_skip" in report.results
|
||||
assert report.results["should_skip"].status == px.TaskStatus.SKIPPED
|
||||
|
||||
|
||||
def test_taskspec_with_conditions_execute():
|
||||
"""测试条件满足时任务正常执行."""
|
||||
|
||||
# 创建一个总是满足的条件
|
||||
def always_true():
|
||||
return True
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"should_run",
|
||||
cmd=[*ECHO_CMD, "this should run"],
|
||||
conditions=(always_true,),
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert "should_run" in report.results
|
||||
assert report.results["should_run"].status == px.TaskStatus.SUCCESS
|
||||
|
||||
|
||||
def test_platform_conditions():
|
||||
"""测试平台条件."""
|
||||
if sys.platform == "win32":
|
||||
win_cmd = ["cmd", "/c", "echo", "Windows"]
|
||||
posix_cmd = ["echo", "POSIX"]
|
||||
else:
|
||||
win_cmd = ["echo", "Windows"]
|
||||
posix_cmd = ["echo", "POSIX"]
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"win_task",
|
||||
cmd=win_cmd,
|
||||
conditions=(IS_WINDOWS,),
|
||||
),
|
||||
px.TaskSpec(
|
||||
"linux_task",
|
||||
cmd=posix_cmd,
|
||||
conditions=(IS_LINUX,),
|
||||
),
|
||||
px.TaskSpec(
|
||||
"macos_task",
|
||||
cmd=posix_cmd,
|
||||
conditions=(IS_MACOS,),
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
|
||||
# 检查只有当前平台的任务执行了
|
||||
if sys.platform == "win32":
|
||||
assert report.results["win_task"].status == px.TaskStatus.SUCCESS
|
||||
assert report.results["linux_task"].status == px.TaskStatus.SKIPPED
|
||||
assert report.results["macos_task"].status == px.TaskStatus.SKIPPED
|
||||
elif sys.platform == "linux":
|
||||
assert report.results["win_task"].status == px.TaskStatus.SKIPPED
|
||||
assert report.results["linux_task"].status == px.TaskStatus.SUCCESS
|
||||
assert report.results["macos_task"].status == px.TaskStatus.SKIPPED
|
||||
elif sys.platform == "darwin":
|
||||
assert report.results["win_task"].status == px.TaskStatus.SKIPPED
|
||||
assert report.results["linux_task"].status == px.TaskStatus.SKIPPED
|
||||
assert report.results["macos_task"].status == px.TaskStatus.SUCCESS
|
||||
|
||||
|
||||
def test_app_installed_conditions():
|
||||
"""测试应用安装条件."""
|
||||
# 测试 python 应该总是安装的
|
||||
if sys.platform == "win32":
|
||||
python_cmd = ["python", "--version"]
|
||||
else:
|
||||
python_cmd = ["python3", "--version"]
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"python_check",
|
||||
cmd=python_cmd,
|
||||
conditions=(BuiltinConditions.HAS_INSTALLED("python"),),
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert "python_check" in report.results
|
||||
# python 应该总是安装的
|
||||
assert report.results["python_check"].status == px.TaskStatus.SUCCESS
|
||||
|
||||
|
||||
def test_combined_conditions():
|
||||
"""测试组合条件."""
|
||||
# AND 条件
|
||||
and_condition = BuiltinConditions.AND(
|
||||
lambda: True,
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
# OR 条件
|
||||
or_condition = BuiltinConditions.OR(
|
||||
lambda: True,
|
||||
lambda: False,
|
||||
)
|
||||
|
||||
# NOT 条件
|
||||
not_condition = BuiltinConditions.NOT(lambda: False)
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"and_test",
|
||||
cmd=[*ECHO_CMD, "AND"],
|
||||
conditions=(and_condition,),
|
||||
),
|
||||
px.TaskSpec(
|
||||
"or_test",
|
||||
cmd=[*ECHO_CMD, "OR"],
|
||||
conditions=(or_condition,),
|
||||
),
|
||||
px.TaskSpec(
|
||||
"not_test",
|
||||
cmd=[*ECHO_CMD, "NOT"],
|
||||
conditions=(not_condition,),
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert report.results["and_test"].status == px.TaskStatus.SUCCESS
|
||||
assert report.results["or_test"].status == px.TaskStatus.SUCCESS
|
||||
assert report.results["not_test"].status == px.TaskStatus.SUCCESS
|
||||
|
||||
|
||||
def test_taskspec_with_cwd():
|
||||
"""测试工作目录设置."""
|
||||
if sys.platform == "win32":
|
||||
ls_cmd = ["cmd", "/c", "dir"]
|
||||
else:
|
||||
ls_cmd = ["ls", "-la"]
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"list_current",
|
||||
cmd=ls_cmd,
|
||||
cwd=Path.cwd(),
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert "list_current" in report.results
|
||||
assert report.results["list_current"].status == px.TaskStatus.SUCCESS
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_taskspec_with_timeout():
|
||||
"""测试超时设置."""
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
# 短时间任务应该成功
|
||||
px.TaskSpec(
|
||||
"short_task",
|
||||
cmd=["python", "-c", "import time; time.sleep(0.1)"],
|
||||
timeout=1.0,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert "short_task" in report.results
|
||||
assert report.results["short_task"].status == px.TaskStatus.SUCCESS
|
||||
|
||||
|
||||
def test_taskspec_dependency_with_conditions():
|
||||
"""测试依赖和条件的组合."""
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"first",
|
||||
cmd=[*ECHO_CMD, "first"],
|
||||
conditions=(lambda: True,),
|
||||
),
|
||||
px.TaskSpec(
|
||||
"second",
|
||||
cmd=[*ECHO_CMD, "second"],
|
||||
depends_on=("first",),
|
||||
conditions=(lambda: True,),
|
||||
),
|
||||
px.TaskSpec(
|
||||
"third",
|
||||
cmd=[*ECHO_CMD, "third"],
|
||||
depends_on=("second",),
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert report.results["first"].status == px.TaskStatus.SUCCESS
|
||||
assert report.results["second"].status == px.TaskStatus.SUCCESS
|
||||
assert report.results["third"].status == px.TaskStatus.SUCCESS
|
||||
|
||||
|
||||
def test_taskspec_mixed_fn_and_cmd():
|
||||
"""测试混合使用 fn 和 cmd."""
|
||||
|
||||
def my_function():
|
||||
return "result from function"
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("fn_task", fn=my_function),
|
||||
px.TaskSpec("cmd_task", cmd=[*ECHO_CMD, "from command"]),
|
||||
]
|
||||
)
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert report.results["fn_task"].status == px.TaskStatus.SUCCESS
|
||||
assert report.results["fn_task"].value == "result from function"
|
||||
assert report.results["cmd_task"].status == px.TaskStatus.SUCCESS
|
||||
|
||||
|
||||
def test_taskspec_cmd_overrides_fn():
|
||||
"""测试 cmd 参数优先于 fn 参数."""
|
||||
|
||||
def my_function():
|
||||
return "should not run"
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"cmd_priority",
|
||||
fn=my_function,
|
||||
cmd=[*ECHO_CMD, "cmd takes priority"],
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert report.results["cmd_priority"].status == px.TaskStatus.SUCCESS
|
||||
# cmd 应该被执行,而不是 fn
|
||||
assert report.results["cmd_priority"].value is None
|
||||
|
||||
|
||||
def test_taskspec_callable_cmd():
|
||||
"""测试 cmd 参数使用可调用对象."""
|
||||
|
||||
def my_callable():
|
||||
return "callable result"
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("callable_cmd", cmd=my_callable),
|
||||
]
|
||||
)
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert report.results["callable_cmd"].status == px.TaskStatus.SUCCESS
|
||||
assert report.results["callable_cmd"].value == "callable result"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# verbose 模式测试
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestTaskSpecVerbose:
|
||||
"""测试 TaskSpec 的 verbose 字段."""
|
||||
|
||||
def test_verbose_default_is_false(self) -> None:
|
||||
"""verbose 默认应为 False."""
|
||||
spec: px.TaskSpec[Any] = px.TaskSpec[Any]("a", cmd=[*ECHO_CMD, "hi"])
|
||||
assert spec.verbose is False
|
||||
|
||||
def test_verbose_true_prints_command(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose=True 时应打印执行的命令."""
|
||||
graph = px.Graph.from_specs([px.TaskSpec("echo", cmd=[*ECHO_CMD, "verbose-output"], verbose=True)])
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
captured = capsys.readouterr()
|
||||
assert "执行命令" in captured.out
|
||||
assert "返回码" in captured.out
|
||||
|
||||
def test_verbose_false_silent(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose=False 时不应打印命令信息."""
|
||||
graph = px.Graph.from_specs([px.TaskSpec[Any]("echo", cmd=[*ECHO_CMD, "silent"], verbose=False)])
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
captured = capsys.readouterr()
|
||||
assert "执行命令" not in captured.out
|
||||
assert "返回码" not in captured.out
|
||||
|
||||
def test_verbose_true_shell_cmd(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose=True 时 shell 命令也应打印执行信息."""
|
||||
if sys.platform == "win32":
|
||||
shell_cmd = 'cmd /c "echo shell-verbose"'
|
||||
else:
|
||||
shell_cmd = "echo 'shell-verbose'"
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("shell", cmd=shell_cmd, verbose=True)])
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
captured = capsys.readouterr()
|
||||
assert "执行 Shell" in captured.out
|
||||
|
||||
def test_verbose_prints_cwd(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose=True 且设置了 cwd 时应打印工作目录."""
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
graph = px.Graph.from_specs([px.TaskSpec[Any]("ls", cmd=ECHO_CMD, cwd=Path(tmpdir), verbose=True)])
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
captured = capsys.readouterr()
|
||||
assert "工作目录" in captured.out
|
||||
|
||||
def test_verbose_failure_includes_returncode(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose=True 时失败也应打印返回码."""
|
||||
from pyflowx.errors import TaskFailedError
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"fail",
|
||||
cmd=["python", "-c", "import sys; sys.exit(1)"],
|
||||
verbose=True,
|
||||
)
|
||||
]
|
||||
)
|
||||
with pytest.raises(TaskFailedError):
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
captured = capsys.readouterr()
|
||||
assert "返回码" in captured.out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# _wrap_cmd 错误路径测试
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestTaskSpecCmdErrors:
|
||||
"""测试 _wrap_cmd 的错误处理路径."""
|
||||
|
||||
def test_cmd_list_file_not_found(self) -> None:
|
||||
"""命令不存在时应抛出 RuntimeError."""
|
||||
from pyflowx.errors import TaskFailedError
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("missing", cmd=["this-command-does-not-exist-xyz"])])
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
# 错误信息应包含命令未找到
|
||||
assert "命令未找到" in str(exc_info.value.cause) or "not found" in str(exc_info.value.cause).lower()
|
||||
|
||||
def test_cmd_list_failure_includes_stderr(self) -> None:
|
||||
"""命令失败时错误信息应包含 stderr."""
|
||||
from pyflowx.errors import TaskFailedError
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"fail",
|
||||
cmd=[
|
||||
"python",
|
||||
"-c",
|
||||
"import sys; sys.stderr.write('error-msg'); sys.exit(1)",
|
||||
],
|
||||
)
|
||||
]
|
||||
)
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
# 非 verbose 模式下, stderr 应包含在错误信息中
|
||||
assert "error-msg" in str(exc_info.value.cause)
|
||||
|
||||
def test_cmd_string_file_not_found(self) -> None:
|
||||
"""shell 命令不存在时应抛出 RuntimeError."""
|
||||
from pyflowx.errors import TaskFailedError
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("missing", cmd="this-command-does-not-exist-xyz-123")])
|
||||
with pytest.raises(TaskFailedError):
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
|
||||
def test_cmd_string_failure(self) -> None:
|
||||
"""shell 命令失败时应抛出 RuntimeError."""
|
||||
from pyflowx.errors import TaskFailedError
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("fail", cmd='python -c "import sys; sys.exit(1)"')])
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
assert "Shell 命令执行失败" in str(exc_info.value.cause)
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_cmd_timeout_raises(self) -> None:
|
||||
"""命令超时应抛出 RuntimeError."""
|
||||
from pyflowx.errors import TaskFailedError
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"slow",
|
||||
cmd=["python", "-c", "import time; time.sleep(5)"],
|
||||
timeout=0.1,
|
||||
)
|
||||
]
|
||||
)
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
assert "超时" in str(exc_info.value.cause)
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_cmd_string_timeout_raises(self) -> None:
|
||||
"""shell 命令超时应抛出 RuntimeError."""
|
||||
from pyflowx.errors import TaskFailedError
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("slow", cmd='python -c "import time; time.sleep(5)"', timeout=0.1)])
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
assert "超时" in str(exc_info.value.cause)
|
||||
|
||||
def test_no_fn_no_cmd_raises(self) -> None:
|
||||
"""没有 fn 和 cmd 时应抛出 ValueError."""
|
||||
with pytest.raises(ValueError, match="必须提供 fn 或 cmd"):
|
||||
_ = px.TaskSpec("empty")
|
||||
@@ -0,0 +1,7 @@
|
||||
"""
|
||||
This type stub file was generated by pyright.
|
||||
"""
|
||||
|
||||
from .graphlib import CycleError, TopologicalSorter
|
||||
|
||||
__all__ = ["CycleError", "TopologicalSorter"]
|
||||
@@ -0,0 +1,113 @@
|
||||
"""
|
||||
This type stub file was generated by pyright.
|
||||
"""
|
||||
|
||||
from typing import Any, Generator
|
||||
|
||||
__all__ = ["CycleError", "TopologicalSorter"]
|
||||
_NODE_OUT = ...
|
||||
_NODE_DONE = ...
|
||||
|
||||
class _NodeInfo:
|
||||
__slots__: list[str]
|
||||
|
||||
def __init__(self, node) -> None: ...
|
||||
|
||||
class CycleError(ValueError):
|
||||
"""Subclass of ValueError raised by TopologicalSorterif cycles exist in the graph
|
||||
|
||||
If multiple cycles exist, only one undefined choice among them will be reported
|
||||
and included in the exception. The detected cycle can be accessed via the second
|
||||
element in the *args* attribute of the exception instance and consists in a list
|
||||
of nodes, such that each node is, in the graph, an immediate predecessor of the
|
||||
next node in the list. In the reported list, the first and the last node will be
|
||||
the same, to make it clear that it is cyclic.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
class TopologicalSorter:
|
||||
"""Provides functionality to topologically sort a graph of hashable nodes"""
|
||||
|
||||
def __init__(self, graph=...) -> None: ...
|
||||
def add(self, node, *predecessors) -> None:
|
||||
"""Add a new node and its predecessors to the graph.
|
||||
|
||||
Both the *node* and all elements in *predecessors* must be hashable.
|
||||
|
||||
If called multiple times with the same node argument, the set of dependencies
|
||||
will be the union of all dependencies passed in.
|
||||
|
||||
It is possible to add a node with no dependencies (*predecessors* is not provided)
|
||||
as well as provide a dependency twice. If a node that has not been provided before
|
||||
is included among *predecessors* it will be automatically added to the graph with
|
||||
no predecessors of its own.
|
||||
|
||||
Raises ValueError if called after "prepare".
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
def prepare(self) -> None:
|
||||
"""Mark the graph as finished and check for cycles in the graph.
|
||||
|
||||
If any cycle is detected, "CycleError" will be raised, but "get_ready" can
|
||||
still be used to obtain as many nodes as possible until cycles block more
|
||||
progress. After a call to this function, the graph cannot be modified and
|
||||
therefore no more nodes can be added using "add".
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
def get_ready(self) -> tuple[Any, ...]:
|
||||
"""Return a tuple of all the nodes that are ready.
|
||||
|
||||
Initially it returns all nodes with no predecessors; once those are marked
|
||||
as processed by calling "done", further calls will return all new nodes that
|
||||
have all their predecessors already processed. Once no more progress can be made,
|
||||
empty tuples are returned.
|
||||
|
||||
Raises ValueError if called without calling "prepare" previously.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
def is_active(self) -> bool:
|
||||
"""Return True if more progress can be made and ``False`` otherwise.
|
||||
|
||||
Progress can be made if cycles do not block the resolution and either there
|
||||
are still nodes ready that haven't yet been returned by "get_ready" or the
|
||||
number of nodes marked "done" is less than the number that have been returned
|
||||
by "get_ready".
|
||||
|
||||
Raises ValueError if called without calling "prepare" previously.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
def __bool__(self) -> bool: ...
|
||||
def done(self, *nodes) -> None:
|
||||
"""Marks a set of nodes returned by "get_ready" as processed.
|
||||
|
||||
This method unblocks any successor of each node in *nodes* for being returned
|
||||
in the future by a a call to "get_ready"
|
||||
|
||||
Raises :exec:`ValueError` if any node in *nodes* has already been marked as
|
||||
processed by a previous call to this method, if a node was not added to the
|
||||
graph by using "add" or if called without calling "prepare" previously or if
|
||||
node has not yet been returned by "get_ready".
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
def static_order(self) -> Generator[Any]:
|
||||
"""Returns an iterable of nodes in a topological order.
|
||||
|
||||
The particular order that is returned may depend on the specific
|
||||
order in which the items were inserted in the graph.
|
||||
|
||||
Using this method does not require to call "prepare" or "done". If any
|
||||
cycle is detected, :exc:`CycleError` will be raised.
|
||||
"""
|
||||
|
||||
...
|
||||
@@ -214,6 +214,18 @@ wheels = [
|
||||
{ url = "https://mirrors.aliyun.com/pypi/packages/3c/56/70860ece85cd49b564305cbc22bf6c4183975427ff6dfe2097e855f5dd5e/backports_zstd-1.6.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:994167ff6551b9c1ce226e0aab16295b98c94507b5701aa60d2c32b7d50796b1" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "basedpyright"
|
||||
version = "1.39.8"
|
||||
source = { registry = "https://mirrors.aliyun.com/pypi/simple/" }
|
||||
dependencies = [
|
||||
{ name = "nodejs-wheel-binaries" },
|
||||
]
|
||||
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/d2/62/8550c75850b2185df984d1de437b4805b039ba856cacbee2966236203133/basedpyright-1.39.8.tar.gz", hash = "sha256:bb1a86d4d71425d52d1501b317fe23d45527baed06bd5d5e1a07cd4b60d07b55" }
|
||||
wheels = [
|
||||
{ url = "https://mirrors.aliyun.com/pypi/packages/f5/94/878454aefe94328ba7ad808ecd63da8311aae1198da46cfb29f5cfe130a8/basedpyright-1.39.8-py3-none-any.whl", hash = "sha256:a79d89928064bd9023d429b50c625d87d023bacc2fe3932ef6c7bd13b5426048" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cachetools"
|
||||
version = "5.5.2"
|
||||
@@ -2018,6 +2030,22 @@ wheels = [
|
||||
{ url = "https://mirrors.aliyun.com/pypi/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nodejs-wheel-binaries"
|
||||
version = "24.16.0"
|
||||
source = { registry = "https://mirrors.aliyun.com/pypi/simple/" }
|
||||
sdist = { url = "https://mirrors.aliyun.com/pypi/packages/a3/22/2a5beb4e21417c73233d9f65cf6f3e96e891b80d2f550a8f630ebc6b88c6/nodejs_wheel_binaries-24.16.0.tar.gz", hash = "sha256:c973cb69dc5fd16e6f6dc6e579e2c3d5534e2a1f57619dddf5ba070efa7dde37" }
|
||||
wheels = [
|
||||
{ url = "https://mirrors.aliyun.com/pypi/packages/83/d1/68b43b53cd0fa83ae6fd406705023ca988d9e0ca41c724d82e66fbeb2ef6/nodejs_wheel_binaries-24.16.0-py2.py3-none-macosx_13_0_arm64.whl", hash = "sha256:d9f8f677dcf30e37ac244f07869726abe043f01eb0f45722b1df31cc2af7093c" },
|
||||
{ url = "https://mirrors.aliyun.com/pypi/packages/e9/b2/40a989159599080da485de966c4c2d207e852ac7aa7864702626d96c8bf5/nodejs_wheel_binaries-24.16.0-py2.py3-none-macosx_13_0_x86_64.whl", hash = "sha256:3d0370fe7120ce9697a4f60d40480d2bd8808d9f30131458d5afc0040d4e5a51" },
|
||||
{ url = "https://mirrors.aliyun.com/pypi/packages/d7/a7/cd42174fb5ff6faff7fa8d326a18914d8f232098ab5de055b57c16fa13ca/nodejs_wheel_binaries-24.16.0-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:85dc92bbb79c851569c5925dcc2a4c915a034efab375f99e4e7e6bbe9cca8342" },
|
||||
{ url = "https://mirrors.aliyun.com/pypi/packages/2b/95/c8a1f9ae140aa28df8744d984d01d4b3af7cdd6555af12127f40ceb45a7d/nodejs_wheel_binaries-24.16.0-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:2f3036292811514ba847b3708492644764f88a833ac425c5f55007014308ddfd" },
|
||||
{ url = "https://mirrors.aliyun.com/pypi/packages/64/c9/7c35b3737f59e36d0249c265397b7bff570519b95301d6e16ea361e904ad/nodejs_wheel_binaries-24.16.0-py2.py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:db8a8a76ebd2b28ecbfc9ad464baa3707241b9e050a30e2efdf6f60c0f886502" },
|
||||
{ url = "https://mirrors.aliyun.com/pypi/packages/04/96/d931255cf9d11a84d6b54d882dba7434646467d568ccf070ea3418638df3/nodejs_wheel_binaries-24.16.0-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f1a3d8f7b4491cbbd023ba3fc4e901fcca2d9fb80d57f24ba3890de8b1dbac03" },
|
||||
{ url = "https://mirrors.aliyun.com/pypi/packages/a2/7b/8b7a3f41bc255411be30b6d7d288aab8ffd9ea2055db8555ced3548007b9/nodejs_wheel_binaries-24.16.0-py2.py3-none-win_amd64.whl", hash = "sha256:bb136be9944f0662dcf1120f45193a6b75b13fac378971a95cc42c9f879a81aa" },
|
||||
{ url = "https://mirrors.aliyun.com/pypi/packages/17/66/1ed71f1f529b8ca727d42c7ceb9db0bef145ce4a13dfc86fb50aa44f3be6/nodejs_wheel_binaries-24.16.0-py2.py3-none-win_arm64.whl", hash = "sha256:8308940b5edd0a50dc5267ea36ba21c9f668e83fe0d9f293937174d3a7e31c36" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "packaging"
|
||||
version = "26.2"
|
||||
@@ -2193,7 +2221,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pyflowx"
|
||||
version = "0.1.1"
|
||||
version = "0.1.3"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "graphlib-backport", marker = "python_full_version < '3.9'" },
|
||||
@@ -2201,6 +2229,7 @@ dependencies = [
|
||||
|
||||
[package.optional-dependencies]
|
||||
dev = [
|
||||
{ name = "basedpyright" },
|
||||
{ name = "hatch", version = "1.14.2", source = { registry = "https://mirrors.aliyun.com/pypi/simple/" }, marker = "python_full_version < '3.9'" },
|
||||
{ name = "hatch", version = "1.15.1", source = { registry = "https://mirrors.aliyun.com/pypi/simple/" }, marker = "python_full_version == '3.9.*'" },
|
||||
{ name = "hatch", version = "1.17.0", source = { registry = "https://mirrors.aliyun.com/pypi/simple/" }, marker = "python_full_version >= '3.10'" },
|
||||
@@ -2239,10 +2268,11 @@ dev = [
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "basedpyright", marker = "extra == 'dev'", specifier = ">=1.39.8" },
|
||||
{ name = "graphlib-backport", marker = "python_full_version < '3.9'", specifier = ">=1.0.0" },
|
||||
{ name = "hatch", marker = "extra == 'dev'", specifier = ">=1.14.2" },
|
||||
{ name = "httpx", marker = "extra == 'dev'", specifier = ">=0.28.0" },
|
||||
{ name = "mypy", marker = "extra == 'dev'", specifier = ">=1.0" },
|
||||
{ name = "mypy", marker = "extra == 'dev'", specifier = ">=1.14.1" },
|
||||
{ name = "prek", marker = "extra == 'dev'", specifier = ">=0.4.5" },
|
||||
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" },
|
||||
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24.0" },
|
||||
|
||||
Reference in New Issue
Block a user