Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| cbc02c5aee | |||
| c8e9354e87 | |||
| 1ecff5fdf7 | |||
| c856c9b6a6 | |||
| ea591d1088 | |||
| cae51856d2 | |||
| be03662e4c | |||
| db18ca4978 | |||
| 7de55614a6 | |||
| 939cd724ec | |||
| 5ddfe8510c | |||
| cd38e1246a | |||
| febcd90a31 |
@@ -38,10 +38,10 @@ jobs:
|
||||
run: uv sync --extra dev --frozen
|
||||
|
||||
- name: Ruff 检查
|
||||
run: uv run ruff check src tests examples
|
||||
run: uv run ruff check src tests
|
||||
|
||||
- name: Ruff 格式检查
|
||||
run: uv run ruff format --check src tests examples
|
||||
run: uv run ruff format --check src tests
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# typecheck:mypy 严格类型检查
|
||||
@@ -69,7 +69,7 @@ jobs:
|
||||
run: uv sync --extra dev --frozen
|
||||
|
||||
- name: Mypy 严格类型检查
|
||||
run: uv run mypy
|
||||
run: uv run mypy .
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# test:多平台 × 多 Python 版本矩阵测试 + 覆盖率
|
||||
@@ -101,15 +101,9 @@ jobs:
|
||||
- name: 安装依赖
|
||||
run: uv sync --extra dev --frozen
|
||||
|
||||
- name: 运行测试(含覆盖率,强制 100%)
|
||||
- name: 运行测试(含覆盖率, 95%)
|
||||
run: uv run pytest -v --cov=pyflowx --cov-report=xml --cov-report=term-missing --cov-fail-under=95
|
||||
|
||||
- name: 运行示例冒烟测试
|
||||
run: |
|
||||
uv run python examples/etl_pipeline.py
|
||||
uv run python examples/parallel_run.py
|
||||
uv run python examples/async_aggregation.py
|
||||
|
||||
- name: 上传覆盖率
|
||||
if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'
|
||||
uses: actions/upload-artifact@v4
|
||||
|
||||
Vendored
+1
-1
@@ -32,4 +32,4 @@
|
||||
"python.testing.pytestEnabled": true,
|
||||
"python.testing.unittestEnabled": false,
|
||||
"ruff.importStrategy": "fromEnvironment"
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
> 轻量、类型安全的 DAG 任务调度器。
|
||||
|
||||
[](https://github.com/pyflowx/pyflowx/actions/workflows/ci.yml)
|
||||
[](https://github.com/gookeryoung/pyflowx/actions/workflows/ci.yml)
|
||||
[](https://pypi.org/project/pyflowx/)
|
||||
[](https://pypi.org/project/pyflowx/)
|
||||
[](https://github.com/pyflowx/pyflowx)
|
||||
@@ -20,9 +20,12 @@ PyFlowX 把"任务依赖"这件事做到极致简单:**参数名就是依赖
|
||||
- **自动分层** —— Kahn 算法分组,同层任务可并行
|
||||
- **重试与超时** —— 每个任务独立配置 `retries` 与 `timeout`
|
||||
- **断点续跑** —— `MemoryBackend` / `JSONBackend`,成功结果可缓存复用
|
||||
- **可观测** —— `on_event` 回调、`dry_run` 预览、Mermaid 可视化
|
||||
- **命令任务** —— `cmd` 参数直接执行外部命令,支持列表/shell/可调用对象
|
||||
- **条件执行** —— `conditions` 参数按平台、环境变量、应用安装等条件跳过任务
|
||||
- **CLI 运行器** —— `CliRunner` 把多个图映射为命令行子命令,替代 Makefile
|
||||
- **可观测** —— `on_event` 回调、`dry_run` 预览、`verbose` 生命周期日志、Mermaid 可视化
|
||||
- **零运行时依赖** —— 仅依赖标准库(3.8 需 `graphlib_backport`)
|
||||
- **100% 测试覆盖** —— 分支覆盖率达 100%
|
||||
- **95% 测试覆盖** —— 分支覆盖率>= 95%
|
||||
|
||||
## 安装
|
||||
|
||||
@@ -67,15 +70,27 @@ print(report["double"]) # [2, 4, 6]
|
||||
px.TaskSpec(
|
||||
name="fetch_user", # 唯一标识
|
||||
fn=fetch_user, # 同步或异步函数
|
||||
cmd=["curl", "..."], # 或: 执行命令(覆盖 fn)
|
||||
depends_on=("auth",), # 依赖的任务名
|
||||
args=(uid,), # 静态位置参数(追加在注入参数后)
|
||||
kwargs={"timeout": 30}, # 静态关键字参数
|
||||
retries=3, # 失败重试次数(0 = 仅一次)
|
||||
timeout=30.0, # 超时秒数(None = 不限制)
|
||||
tags=("api", "user"), # 自由标签,用于子图过滤
|
||||
conditions=(is_prod,), # 条件函数列表(全部为 True 才执行)
|
||||
cwd=Path("/tmp"), # 命令工作目录(仅 cmd 模式)
|
||||
verbose=True, # 打印命令输出(仅 cmd 模式)
|
||||
skip_if_missing=True, # 命令不存在时自动跳过(仅 list[str] cmd)
|
||||
)
|
||||
```
|
||||
|
||||
支持两种任务形态:
|
||||
|
||||
- **函数任务**(`fn`):普通 Python 函数,参数名驱动自动注入
|
||||
- **命令任务**(`cmd`):执行外部命令,支持 `list[str]`、`str`(shell)、`Callable` 三种形态
|
||||
|
||||
`skip_if_missing=True` 时,`list[str]` 类型的 `cmd` 会通过 `shutil.which` 检查命令是否存在,不存在则跳过任务(标记为 `SKIPPED`)而非失败。适用于构建工具场景,避免因未安装某些工具而导致整个图执行失败。
|
||||
|
||||
### Graph —— DAG 构建
|
||||
|
||||
```python
|
||||
@@ -101,6 +116,7 @@ report = px.run(
|
||||
strategy="async", # sequential | thread | async
|
||||
max_workers=8, # thread 策略的线程池大小
|
||||
dry_run=False, # True = 仅打印计划
|
||||
verbose=False, # True = 打印任务生命周期日志
|
||||
on_event=callback, # 状态转换回调
|
||||
state=px.JSONBackend("state.json"), # 断点续跑后端
|
||||
)
|
||||
@@ -151,6 +167,86 @@ def fetch_user(uid: int) -> dict: # uid 来自 TaskSpec.args
|
||||
|
||||
所有策略都遵循 `retries`、`timeout`、上下文注入、状态后端,并发出 `TaskEvent`。
|
||||
|
||||
## 命令任务
|
||||
|
||||
`TaskSpec` 的 `cmd` 参数支持执行外部命令,无需包装 Python 函数:
|
||||
|
||||
```python
|
||||
graph = px.Graph.from_specs([
|
||||
# 命令列表(推荐,参数无需转义)
|
||||
px.TaskSpec("list_files", cmd=["ls", "-la"]),
|
||||
# shell 字符串(支持管道、重定向)
|
||||
px.TaskSpec("check_git", cmd="git status | head"),
|
||||
# 带工作目录与超时
|
||||
px.TaskSpec("build", cmd=["make", "all"], cwd=Path("/project"), timeout=300),
|
||||
# 命令不存在时自动跳过(而非失败)
|
||||
px.TaskSpec("optional_tool", cmd=["maturin", "build"], skip_if_missing=True),
|
||||
])
|
||||
```
|
||||
|
||||
`verbose=True` 时打印执行的命令、工作目录、返回码与输出;`verbose=False` 时静默执行(失败信息仍包含 stderr)。
|
||||
|
||||
`skip_if_missing=True` 时,`list[str]` 类型的 `cmd` 会通过 `shutil.which` 检查命令是否存在,不存在则跳过任务(标记为 `SKIPPED`)而非失败。适用于构建工具场景,避免因未安装某些工具而导致整个图执行失败。对于 `str`(shell)和 `Callable` 类型的 `cmd`,此参数无效。
|
||||
|
||||
## 条件执行
|
||||
|
||||
`conditions` 参数让任务按条件跳过(标记为 `SKIPPED`):
|
||||
|
||||
```python
|
||||
from pyflowx.conditions import IS_WINDOWS, BuiltinConditions
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
# 仅在 Windows 上运行
|
||||
px.TaskSpec("win_only", cmd=["dir"], conditions=(IS_WINDOWS,)),
|
||||
# 仅在 git 已安装时运行
|
||||
px.TaskSpec(
|
||||
"git_check",
|
||||
cmd=["git", "--version"],
|
||||
conditions=(BuiltinConditions.HAS_INSTALLED("git"),),
|
||||
),
|
||||
# 组合条件
|
||||
px.TaskSpec(
|
||||
"prod_deploy",
|
||||
fn=deploy,
|
||||
conditions=(
|
||||
BuiltinConditions.ENV_VAR_EQUALS("ENV", "prod"),
|
||||
BuiltinConditions.HAS_INSTALLED("docker"),
|
||||
),
|
||||
),
|
||||
])
|
||||
```
|
||||
|
||||
内置条件:`IS_WINDOWS` / `IS_LINUX` / `IS_MACOS` / `IS_POSIX` / `PYTHON_VERSION` / `HAS_INSTALLED` / `ENV_VAR_EXISTS` / `ENV_VAR_EQUALS` / `NOT` / `AND` / `OR`。
|
||||
|
||||
## CLI 运行器
|
||||
|
||||
`CliRunner` 把多个 Graph 映射为命令行子命令,适合构建项目专属构建工具(替代 Makefile):
|
||||
|
||||
```python
|
||||
runner = px.CliRunner(
|
||||
strategy="sequential",
|
||||
description="My Build Tool",
|
||||
graphs={
|
||||
"clean": clean_graph,
|
||||
"build": build_graph,
|
||||
"test": test_graph,
|
||||
},
|
||||
)
|
||||
runner.run_cli() # 解析 sys.argv 并执行
|
||||
```
|
||||
|
||||
命令行用法:
|
||||
|
||||
```bash
|
||||
python build.py clean # 执行 clean 图
|
||||
python build.py build --strategy thread # 覆盖执行策略
|
||||
python build.py test --dry-run # 仅打印执行计划
|
||||
python build.py --list # 列出所有命令
|
||||
python build.py --quiet # 静默模式
|
||||
```
|
||||
|
||||
`verbose=True`(默认)时打印任务生命周期(开始/成功/失败/跳过)与命令输出;`--quiet` 关闭。
|
||||
|
||||
## 示例
|
||||
|
||||
仓库 `examples/` 目录包含完整示例:
|
||||
|
||||
+3
-2
@@ -17,7 +17,7 @@ license = { text = "MIT" }
|
||||
name = "pyflowx"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.8"
|
||||
version = "0.1.3"
|
||||
version = "0.1.5"
|
||||
|
||||
[project.scripts]
|
||||
pymake = "pyflowx.cli.pymake:main"
|
||||
@@ -78,6 +78,7 @@ show_missing = true
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
asyncio_default_fixture_loop_scope = "function"
|
||||
markers = ["slow: marks tests as slow (deselect with '-m \"not slow\"')"]
|
||||
|
||||
[tool.basedpyright]
|
||||
exclude = ["**/.git", "**/.venv", "**/__pycache__", "**/build", "**/dist"]
|
||||
@@ -91,7 +92,7 @@ typeCheckingMode = "basic" # 类型检查严格度:off / basi
|
||||
# Ruff 配置 - 与 .pre-commit-config.yaml 保持一致
|
||||
[tool.ruff]
|
||||
target-version = "py38"
|
||||
line-length = 88
|
||||
line-length = 120
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = [
|
||||
|
||||
@@ -43,7 +43,13 @@
|
||||
px.TaskSpec(
|
||||
"git_check",
|
||||
cmd=["git", "--version"],
|
||||
conditions=(BuiltinConditions.HAS_APP_INSTALLED("git"),)
|
||||
conditions=(BuiltinConditions.HAS_INSTALLED("git"),)
|
||||
),
|
||||
# 命令不存在时自动跳过(而非失败)
|
||||
px.TaskSpec(
|
||||
"optional_build",
|
||||
cmd=["maturin", "build"],
|
||||
skip_if_missing=True
|
||||
),
|
||||
])
|
||||
report = px.run(graph)
|
||||
@@ -78,7 +84,7 @@ from .runner import CliExitCode, CliRunner
|
||||
from .storage import JSONBackend, MemoryBackend, StateBackend
|
||||
from .task import TaskCmd, TaskEvent, TaskResult, TaskSpec, TaskStatus
|
||||
|
||||
__version__ = "0.1.3"
|
||||
__version__ = "0.1.5"
|
||||
|
||||
__all__ = [
|
||||
"IS_LINUX",
|
||||
|
||||
+69
-392
@@ -6,50 +6,11 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import BuiltinConditions, Constants
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
class PymakeConfig:
|
||||
"""PyMake 配置类."""
|
||||
|
||||
# 项目根目录
|
||||
PROJECT_ROOT: str = str(Path(__file__).parent.parent.parent.parent)
|
||||
CORE_DIR: str = f"{PROJECT_ROOT}/bitool-core"
|
||||
CORE_PATTERN: str = f"{CORE_DIR}/target/bitool_core-*-cp*.whl"
|
||||
TIMEOUT: int = 600
|
||||
|
||||
# Python 构建
|
||||
BUILD_TOOL: str = "uv"
|
||||
BUILD_COMMAND: list[str] = [BUILD_TOOL, "build"]
|
||||
|
||||
# Rust 构建 (maturin)
|
||||
MATURIN_TOOL: str = "maturin"
|
||||
MATURIN_BUILD_COMMAND: list[str] = ["maturin", "build", "-r"]
|
||||
MATURIN_DEV_COMMAND: list[str] = ["maturin", "develop"]
|
||||
MATURIN_BUILD_OPTIONS_WIN7: list[str] = [
|
||||
"--target",
|
||||
"x86_64-win7-windows-msvc",
|
||||
"-Zbuild-std",
|
||||
"-i",
|
||||
"python3.8",
|
||||
]
|
||||
|
||||
# 文档
|
||||
DOC_BUILD_TOOL: str = "sphinx-build"
|
||||
DOC_BUILD_COMMAND: list[str] = ["sphinx-build", "-b", "html", "docs", "docs/_build"]
|
||||
|
||||
# 清理
|
||||
DIRS_TO_IGNORE: list[str] = [".venv", ".git", ".tox"]
|
||||
PYTHON_BUILD_DIRS: list[str] = ["dist", "build", "*.egg-info", "src/*.egg-info"]
|
||||
|
||||
|
||||
conf = PymakeConfig()
|
||||
|
||||
|
||||
def get_maturin_build_command() -> list[str]:
|
||||
def maturin_build_cmd() -> list[str]:
|
||||
"""获取 maturin 构建命令(根据平台自动添加参数).
|
||||
|
||||
Returns
|
||||
@@ -57,336 +18,42 @@ def get_maturin_build_command() -> list[str]:
|
||||
list[str]
|
||||
完整的 maturin 构建命令列表.
|
||||
"""
|
||||
base_cmd = conf.MATURIN_BUILD_COMMAND.copy()
|
||||
command = ["maturin", "build", "-r"].copy()
|
||||
if Constants.IS_WINDOWS:
|
||||
base_cmd.extend(conf.MATURIN_BUILD_OPTIONS_WIN7)
|
||||
return base_cmd
|
||||
command.extend(
|
||||
[
|
||||
"--target",
|
||||
"x86_64-win7-windows-msvc",
|
||||
"-Zbuild-std",
|
||||
"-i",
|
||||
"python3.8",
|
||||
]
|
||||
)
|
||||
return command
|
||||
|
||||
|
||||
# 命令条件判断
|
||||
MATURIN_CONDITION = BuiltinConditions.HAS_APP_INSTALLED(conf.MATURIN_TOOL)
|
||||
PYTEST_CONDITION = BuiltinConditions.HAS_APP_INSTALLED("pytest")
|
||||
UV_CONDITION = BuiltinConditions.HAS_APP_INSTALLED(conf.BUILD_TOOL)
|
||||
HATCH_CONDITION = BuiltinConditions.HAS_APP_INSTALLED("hatch")
|
||||
RUFF_CONDITION = BuiltinConditions.HAS_APP_INSTALLED("ruff")
|
||||
GIT_CONDITION = BuiltinConditions.HAS_APP_INSTALLED("git")
|
||||
TOX_CONDITION = BuiltinConditions.HAS_APP_INSTALLED("tox")
|
||||
|
||||
|
||||
def build_graphs() -> dict[str, px.Graph]:
|
||||
"""构建所有命令对应的任务流图.
|
||||
|
||||
将原本的 CommandScheduler/RunCommand 模式转换为 Graph/TaskSpec 模式,
|
||||
每个 Graph 是一个独立的任务流, 由 CliRunner 根据用户输入选择执行.
|
||||
"""
|
||||
return {
|
||||
# === 构建命令 ===
|
||||
# 构建 Python 包
|
||||
"b": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"uv_build",
|
||||
cmd=conf.BUILD_COMMAND,
|
||||
conditions=(UV_CONDITION,),
|
||||
timeout=conf.TIMEOUT,
|
||||
),
|
||||
]
|
||||
),
|
||||
# 构建 Rust 核心模块
|
||||
"bc": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"maturin_build",
|
||||
cmd=get_maturin_build_command(),
|
||||
cwd=Path(conf.CORE_DIR),
|
||||
conditions=(MATURIN_CONDITION,),
|
||||
timeout=conf.TIMEOUT,
|
||||
),
|
||||
]
|
||||
),
|
||||
# 构建双包(先 Rust 后 Python)
|
||||
"ba": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"maturin_build",
|
||||
cmd=get_maturin_build_command(),
|
||||
cwd=Path(conf.CORE_DIR),
|
||||
conditions=(MATURIN_CONDITION,),
|
||||
timeout=conf.TIMEOUT,
|
||||
),
|
||||
px.TaskSpec(
|
||||
"uv_build",
|
||||
cmd=conf.BUILD_COMMAND,
|
||||
conditions=(UV_CONDITION,),
|
||||
timeout=conf.TIMEOUT,
|
||||
depends_on=("maturin_build",),
|
||||
),
|
||||
]
|
||||
),
|
||||
# === 安装命令(开发模式) ===
|
||||
# 安装 Rust 核心模块
|
||||
"ic": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"maturin_dev",
|
||||
cmd=conf.MATURIN_DEV_COMMAND,
|
||||
cwd=Path(conf.CORE_DIR),
|
||||
conditions=(MATURIN_CONDITION,),
|
||||
),
|
||||
]
|
||||
),
|
||||
# 安装 Python 主包
|
||||
"ip": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"uv_install",
|
||||
cmd=["uv", "pip", "install", "-e", "."],
|
||||
conditions=(UV_CONDITION,),
|
||||
),
|
||||
]
|
||||
),
|
||||
# 安装双包(开发模式)
|
||||
"ia": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"maturin_dev",
|
||||
cmd=conf.MATURIN_DEV_COMMAND,
|
||||
cwd=Path(conf.CORE_DIR),
|
||||
conditions=(MATURIN_CONDITION,),
|
||||
),
|
||||
px.TaskSpec(
|
||||
"uv_install",
|
||||
cmd=["uv", "pip", "install", "-e", "."],
|
||||
conditions=(UV_CONDITION,),
|
||||
depends_on=("maturin_dev",),
|
||||
),
|
||||
]
|
||||
),
|
||||
# === 清理命令 ===
|
||||
# 清理 Python 构建产物
|
||||
"cp": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"git_clean_python",
|
||||
cmd=["git", "clean", "-xfd", "-e", *conf.DIRS_TO_IGNORE],
|
||||
conditions=(GIT_CONDITION,),
|
||||
),
|
||||
]
|
||||
),
|
||||
# 清理 Rust 构建产物
|
||||
"cc": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"cargo_clean",
|
||||
cmd=["cargo", "clean"],
|
||||
cwd=Path(conf.CORE_DIR),
|
||||
conditions=(MATURIN_CONDITION,),
|
||||
),
|
||||
]
|
||||
),
|
||||
# 清理所有构建产物
|
||||
"ca": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"cargo_clean",
|
||||
cmd=["cargo", "clean"],
|
||||
cwd=Path(conf.CORE_DIR),
|
||||
conditions=(MATURIN_CONDITION,),
|
||||
),
|
||||
px.TaskSpec(
|
||||
"git_clean",
|
||||
cmd=["git", "clean", "-xfd", "-e", *conf.DIRS_TO_IGNORE],
|
||||
conditions=(GIT_CONDITION,),
|
||||
),
|
||||
]
|
||||
),
|
||||
# === 开发工具 ===
|
||||
# 运行测试, 跳过 slow, 并行模式
|
||||
"t": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pytest",
|
||||
cmd=[
|
||||
"pytest",
|
||||
"-m",
|
||||
"not slow",
|
||||
"-n",
|
||||
"8",
|
||||
"--dist",
|
||||
"loadfile",
|
||||
"--color=yes",
|
||||
"--durations=10",
|
||||
],
|
||||
conditions=(PYTEST_CONDITION,),
|
||||
timeout=conf.TIMEOUT,
|
||||
),
|
||||
]
|
||||
),
|
||||
# 运行测试, 非并行模式
|
||||
"tf": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pytest",
|
||||
cmd=[
|
||||
"pytest",
|
||||
"-m",
|
||||
"not slow",
|
||||
"--dist",
|
||||
"loadfile",
|
||||
"--color=yes",
|
||||
"--durations=10",
|
||||
],
|
||||
conditions=(PYTEST_CONDITION,),
|
||||
timeout=conf.TIMEOUT,
|
||||
),
|
||||
]
|
||||
),
|
||||
# 运行测试并生成覆盖率报告, 跳过 slow, 并行模式
|
||||
"tc": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pytest_cov",
|
||||
cmd=[
|
||||
"pytest",
|
||||
"--cov",
|
||||
"-n",
|
||||
"auto",
|
||||
"--dist",
|
||||
"loadfile",
|
||||
"--tb=short",
|
||||
"-v",
|
||||
"--color=yes",
|
||||
"--durations=10",
|
||||
],
|
||||
conditions=(PYTEST_CONDITION,),
|
||||
timeout=conf.TIMEOUT,
|
||||
),
|
||||
]
|
||||
),
|
||||
# 代码格式化与检查
|
||||
"lint": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"ruff_check",
|
||||
cmd=[
|
||||
"ruff",
|
||||
"check",
|
||||
"--fix",
|
||||
"--unsafe-fixes",
|
||||
],
|
||||
conditions=(RUFF_CONDITION,),
|
||||
timeout=conf.TIMEOUT,
|
||||
cwd=Path(conf.PROJECT_ROOT),
|
||||
),
|
||||
]
|
||||
),
|
||||
# 类型检查
|
||||
"typecheck": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"ty_check",
|
||||
cmd=["ty", "check", "src/bitool"],
|
||||
conditions=(BuiltinConditions.HAS_APP_INSTALLED("ty"),),
|
||||
),
|
||||
]
|
||||
),
|
||||
# 构建文档
|
||||
"doc": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"sphinx_build",
|
||||
cmd=conf.DOC_BUILD_COMMAND,
|
||||
conditions=(
|
||||
BuiltinConditions.HAS_APP_INSTALLED(conf.DOC_BUILD_TOOL),
|
||||
),
|
||||
),
|
||||
]
|
||||
),
|
||||
# === 发布命令 ===
|
||||
# 发布 Python 主包到 PyPI
|
||||
"pb": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"publish_python",
|
||||
cmd=["hatch", "publish"],
|
||||
cwd=Path(conf.PROJECT_ROOT),
|
||||
conditions=(HATCH_CONDITION,),
|
||||
timeout=conf.TIMEOUT,
|
||||
),
|
||||
]
|
||||
),
|
||||
# 发布所有包(先 Rust 后 Python)
|
||||
"pba": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"publish_rust",
|
||||
cmd=[
|
||||
"twine",
|
||||
"upload",
|
||||
"--disable-progress-bar",
|
||||
conf.CORE_PATTERN,
|
||||
],
|
||||
cwd=Path(conf.CORE_DIR),
|
||||
conditions=(MATURIN_CONDITION,),
|
||||
timeout=conf.TIMEOUT,
|
||||
),
|
||||
px.TaskSpec(
|
||||
"publish_python",
|
||||
cmd=["hatch", "publish"],
|
||||
cwd=Path(conf.PROJECT_ROOT),
|
||||
conditions=(HATCH_CONDITION,),
|
||||
timeout=conf.TIMEOUT,
|
||||
depends_on=("publish_rust",),
|
||||
),
|
||||
]
|
||||
),
|
||||
# 发布 Rust 核心模块 (maturin publish)
|
||||
"pbc": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"publish_rust",
|
||||
cmd=["maturin", "publish"],
|
||||
cwd=Path(conf.CORE_DIR),
|
||||
conditions=(MATURIN_CONDITION,),
|
||||
timeout=conf.TIMEOUT,
|
||||
),
|
||||
]
|
||||
),
|
||||
# === 多版本测试命令 ===
|
||||
# 运行多版本 Python 测试 (tox)
|
||||
"tox": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"tox_run",
|
||||
cmd=["tox", "-p", "auto"],
|
||||
conditions=(TOX_CONDITION,),
|
||||
timeout=conf.TIMEOUT,
|
||||
),
|
||||
]
|
||||
),
|
||||
# 安装多版本 Python (仅安装不测试)
|
||||
"tox_install": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"uv_python_install",
|
||||
cmd=[
|
||||
"uv",
|
||||
"python",
|
||||
"install",
|
||||
"3.8",
|
||||
"3.9",
|
||||
"3.10",
|
||||
"3.11",
|
||||
"3.12",
|
||||
"3.13",
|
||||
"3.14",
|
||||
],
|
||||
conditions=(UV_CONDITION,),
|
||||
timeout=600,
|
||||
),
|
||||
]
|
||||
),
|
||||
}
|
||||
uv_build: px.TaskSpec = px.TaskSpec("uv_build", cmd=["uv", "build"])
|
||||
maturin_build: px.TaskSpec = px.TaskSpec("maturin_build", cmd=maturin_build_cmd())
|
||||
uv_sync: px.TaskSpec = px.TaskSpec("uv_sync", cmd=["uv", "sync"])
|
||||
git_clean: px.TaskSpec = px.TaskSpec("git_clean", cmd=["gitt", "c"])
|
||||
test: px.TaskSpec = px.TaskSpec(
|
||||
"test", cmd=["pytest", "-m", "not slow", "-n", "8", "--dist", "loadfile", "--color=yes", "--durations=10"]
|
||||
)
|
||||
test_fast: px.TaskSpec = px.TaskSpec(
|
||||
"test_fast", cmd=["pytest", "-m", "not slow", "--dist", "loadfile", "--color=yes", "--durations=10"]
|
||||
)
|
||||
test_coverage: px.TaskSpec = px.TaskSpec(
|
||||
"test_coverage",
|
||||
cmd=["pytest", "--cov", "-n", "8", "--dist", "loadfile", "--tb=short", "-v", "--color=yes", "--durations=10"],
|
||||
)
|
||||
ruff_lint: px.TaskSpec = px.TaskSpec("lint", cmd=["ruff", "check", "--fix", "--unsafe-fixes"])
|
||||
ruff_format: px.TaskSpec = px.TaskSpec("format", cmd=["ruff", "format", "--check", "."], depends_on=("lint",))
|
||||
mypy_check: px.TaskSpec = px.TaskSpec("typecheck", cmd=["mypy", "."])
|
||||
ty_check: px.TaskSpec = px.TaskSpec("ty_check", cmd=["ty", "check", "."])
|
||||
doc: px.TaskSpec = px.TaskSpec("doc", cmd=["sphinx-build", "-b", "html", "docs", "docs/_build"])
|
||||
hatch_publish: px.TaskSpec = px.TaskSpec("publish_python", cmd=["hatch", "publish"])
|
||||
twine_publish: px.TaskSpec = px.TaskSpec("twine_publish", cmd=["twine", "upload", "--disable-progress-bar"])
|
||||
tox: px.TaskSpec = px.TaskSpec("tox", cmd=["tox", "-p", "auto"])
|
||||
|
||||
|
||||
def main():
|
||||
@@ -398,53 +65,63 @@ def main():
|
||||
🔨 构建命令:
|
||||
pymake b - 构建 Python 主包 (uv build)
|
||||
pymake bc - 构建 Rust 核心模块 (maturin build)
|
||||
pymake ba - 构建所有包 (先 Rust 后 Python)
|
||||
pymake ba - 构建所有包 (先 Python 后 Rust)
|
||||
|
||||
📦 安装命令 (开发模式):
|
||||
pymake ic - 安装 Rust 核心模块 (maturin develop)
|
||||
pymake ip - 安装 Python 主包 (uv pip install -e .)
|
||||
pymake ia - 安装所有包 (开发模式,推荐)
|
||||
pymake sync - 安装依赖包 (uv sync)
|
||||
|
||||
🧹 清理命令:
|
||||
pymake cp - 清理 Python 构建产物
|
||||
pymake cc - 清理 Rust 构建产物 (cargo clean)
|
||||
pymake ca - 清理所有构建产物
|
||||
pymake c - 清理所有构建产物 (gitt c)
|
||||
|
||||
🛠️ 开发工具:
|
||||
pymake t - 运行测试 (pytest)
|
||||
pymake tc - 运行测试并生成覆盖率报告
|
||||
pymake tf - 运行快速测试 (pytest -m not slow)
|
||||
pymake lint - 代码格式化与检查 (ruff)
|
||||
pymake typecheck - 类型检查 (ty)
|
||||
pymake type - 类型检查 (mypy, ty)
|
||||
pymake doc - 构建文档 (sphinx)
|
||||
|
||||
🔬 多版本测试:
|
||||
pymake tox - 多版本 Python 测试 (3.8-3.14)
|
||||
pymake tox_install - 安装所有 Python 版本 (仅安装不测试)
|
||||
pymake tox - 多版本 Python 测试 (tox -p auto)
|
||||
|
||||
📦 发布命令:
|
||||
pymake pb - 发布到 PyPI (hatch publish)
|
||||
pymake pba - 发布所有包 (先 Rust 后 Python)
|
||||
pymake pbc - 发布 Rust 核心模块 (maturin publish)
|
||||
pymake pb - 发布到 PyPI (twine + hatch)
|
||||
|
||||
💡 常用工作流:
|
||||
1. 初始化开发环境: pymake ia
|
||||
2. 日常开发: pymake lint && pymake t
|
||||
3. 构建发布包: pymake ba
|
||||
4. 多版本兼容性测试: pymake tox
|
||||
5. 发布到 PyPI: pymake pb
|
||||
6. 清理重新开始: pymake ca && pymake ia
|
||||
1. 日常开发: pymake lint && pymake t
|
||||
2. 构建发布包: pymake ba
|
||||
3. 多版本兼容性测试: pymake tox
|
||||
4. 发布到 PyPI: pymake pb
|
||||
|
||||
📝 示例:
|
||||
pymake ba # 构建所有包
|
||||
pymake ia # 安装开发环境
|
||||
pymake sync # 安装依赖
|
||||
pymake t # 运行测试
|
||||
pymake tox # 多版本兼容性测试
|
||||
pymake lint # 格式化代码
|
||||
pymake ca # 清理所有构建产物
|
||||
pymake type # 类型检查
|
||||
"""
|
||||
runner = px.CliRunner(
|
||||
strategy="sequential",
|
||||
description="PyMake - Python 构建工具 (替代 Makefile)",
|
||||
graphs=build_graphs(), # type: ignore[reportArgumentType]
|
||||
strategy="thread",
|
||||
description="PyMake - Python 构建工具",
|
||||
graphs={
|
||||
# 构建命令
|
||||
"b": px.Graph.from_specs([uv_build]),
|
||||
"bc": px.Graph.from_specs([maturin_build]),
|
||||
"ba": px.Graph.from_specs([uv_build, maturin_build]),
|
||||
# 安装命令
|
||||
"sync": px.Graph.from_specs([uv_sync]),
|
||||
# 清理命令
|
||||
"c": px.Graph.from_specs([git_clean]),
|
||||
# 开发工具
|
||||
"cov": px.Graph.from_specs([test_coverage]),
|
||||
"doc": px.Graph.from_specs([doc]),
|
||||
"lint": px.Graph.from_specs([ruff_lint, ruff_format]),
|
||||
"pb": px.Graph.from_specs([twine_publish, hatch_publish]),
|
||||
"t": px.Graph.from_specs([test]),
|
||||
"tf": px.Graph.from_specs([test_fast]),
|
||||
"tc": px.Graph.from_specs([mypy_check, ty_check]),
|
||||
"tox": px.Graph.from_specs([tox]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
|
||||
@@ -85,7 +85,7 @@ class BuiltinConditions:
|
||||
return sys.version_info >= (major, minor)
|
||||
|
||||
@staticmethod
|
||||
def HAS_APP_INSTALLED(app_name: str) -> Condition:
|
||||
def HAS_INSTALLED(app_name: str) -> Condition:
|
||||
"""检查指定应用是否已安装.
|
||||
|
||||
Parameters
|
||||
@@ -102,7 +102,7 @@ class BuiltinConditions:
|
||||
def _check() -> bool:
|
||||
return shutil.which(app_name) is not None
|
||||
|
||||
_check.__name__ = f"HAS_APP_INSTALLED({app_name!r})"
|
||||
_check.__name__ = f"HAS_INSTALLED({app_name!r})"
|
||||
return _check
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -82,9 +82,7 @@ def build_call_args(
|
||||
)
|
||||
|
||||
# 与本任务相关的上下文子集。
|
||||
dep_context: dict[str, Any] = {
|
||||
name: context[name] for name in spec.depends_on if name in context
|
||||
}
|
||||
dep_context: dict[str, Any] = {name: context[name] for name in spec.depends_on if name in context}
|
||||
|
||||
# 检测静态 kwargs 与依赖名的冲突。
|
||||
collisions = set(spec.kwargs) & set(dep_context)
|
||||
|
||||
@@ -27,7 +27,7 @@ class MissingDependencyError(PyFlowXError):
|
||||
def __init__(self, task: str, dependency: str) -> None:
|
||||
super().__init__(
|
||||
f"Task '{task}' depends on unknown task '{dependency}'. "
|
||||
"Add the dependency before (or together with) this task."
|
||||
+ "Add the dependency before (or together with) this task."
|
||||
)
|
||||
self.task = task
|
||||
self.dependency = dependency
|
||||
@@ -58,9 +58,7 @@ class TaskFailedError(PyFlowXError):
|
||||
layer: int | None = None,
|
||||
) -> None:
|
||||
location = f" (layer {layer})" if layer is not None else ""
|
||||
super().__init__(
|
||||
f"Task '{task}' failed after {attempts} attempt(s){location}: {cause}"
|
||||
)
|
||||
super().__init__(f"Task '{task}' failed after {attempts} attempt(s){location}: {cause}")
|
||||
self.task = task
|
||||
self.cause = cause
|
||||
self.attempts = attempts
|
||||
|
||||
@@ -10,11 +10,12 @@ Shows:
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
|
||||
async def fetch_user(uid: int) -> dict[str, object]:
|
||||
async def fetch_user(uid: int) -> dict[str, Any]:
|
||||
await asyncio.sleep(0.2)
|
||||
return {"id": uid, "name": f"User{uid}"}
|
||||
|
||||
@@ -25,7 +26,7 @@ async def fetch_posts(uid: int) -> list[int]:
|
||||
|
||||
|
||||
# Context annotation → receives the full mapping of upstream results.
|
||||
def aggregate(ctx: px.Context) -> dict[str, object]:
|
||||
def aggregate(ctx: px.Context) -> dict[str, Any]:
|
||||
return dict(ctx)
|
||||
|
||||
|
||||
|
||||
@@ -35,11 +35,7 @@ def transform(
|
||||
extract_orders: list[dict],
|
||||
) -> list[dict]:
|
||||
cmap = {c["id"]: c for c in extract_customers}
|
||||
return [
|
||||
{**o, "customer_name": cmap[o["customer_id"]]["name"]}
|
||||
for o in extract_orders
|
||||
if o["customer_id"] in cmap
|
||||
]
|
||||
return [{**o, "customer_name": cmap[o["customer_id"]]["name"]} for o in extract_orders if o["customer_id"] in cmap]
|
||||
|
||||
|
||||
def load(transform: list[dict]) -> int:
|
||||
@@ -58,9 +54,7 @@ def main() -> None:
|
||||
depends_on=("extract_customers", "extract_orders"),
|
||||
tags=("transform",),
|
||||
),
|
||||
px.TaskSpec(
|
||||
"load", load, depends_on=("transform",), retries=1, tags=("load",)
|
||||
),
|
||||
px.TaskSpec("load", load, depends_on=("transform",), retries=1, tags=("load",)),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -68,7 +62,7 @@ def main() -> None:
|
||||
print(graph.describe())
|
||||
|
||||
print("\n=== Dry run (no execution) ===")
|
||||
px.run(graph, strategy="sequential", dry_run=True)
|
||||
_ = px.run(graph, strategy="sequential", dry_run=True)
|
||||
|
||||
print("\n=== Sequential execution ===")
|
||||
report = px.run(graph, strategy="sequential")
|
||||
|
||||
+42
-57
@@ -35,14 +35,14 @@ EventCallback = Callable[[TaskEvent], None]
|
||||
Strategy = Literal["sequential", "thread", "async"]
|
||||
|
||||
|
||||
def _is_async_fn(spec: TaskSpec[object]) -> bool:
|
||||
def _is_async_fn(spec: TaskSpec[Any]) -> bool:
|
||||
"""判断 ``spec.effective_fn`` 是否为协程函数。"""
|
||||
return inspect.iscoroutinefunction(spec.effective_fn)
|
||||
|
||||
|
||||
def _emit(
|
||||
on_event: EventCallback | None,
|
||||
result: TaskResult[object],
|
||||
result: TaskResult[Any],
|
||||
) -> None:
|
||||
"""若注册了回调则触发一个观察者事件。"""
|
||||
if on_event is None:
|
||||
@@ -58,9 +58,7 @@ def _emit(
|
||||
)
|
||||
|
||||
|
||||
def _log_retry(
|
||||
spec: TaskSpec[object], attempts: int, max_attempts: int, exc: BaseException
|
||||
) -> None:
|
||||
def _log_retry(spec: TaskSpec[Any], attempts: int, max_attempts: int, exc: BaseException) -> None:
|
||||
"""记录重试日志(sync 与 async 共享,便于测试覆盖)。"""
|
||||
logger.warning(
|
||||
"task %r failed (attempt %d/%d): %r; retrying",
|
||||
@@ -71,10 +69,15 @@ def _log_retry(
|
||||
)
|
||||
|
||||
|
||||
def _finalize_failure(result: TaskResult[object], layer_idx: int | None) -> None:
|
||||
def _finalize_failure(
|
||||
result: TaskResult[Any],
|
||||
layer_idx: int | None,
|
||||
on_event: EventCallback | None = None,
|
||||
) -> None:
|
||||
"""标记任务为 FAILED 并抛出 TaskFailedError。"""
|
||||
result.status = TaskStatus.FAILED
|
||||
result.finished_at = datetime.now()
|
||||
_emit(on_event, result)
|
||||
raise TaskFailedError(
|
||||
task=result.spec.name,
|
||||
cause=result.error if result.error is not None else RuntimeError("unknown"),
|
||||
@@ -84,15 +87,16 @@ def _finalize_failure(result: TaskResult[object], layer_idx: int | None) -> None
|
||||
|
||||
|
||||
def _run_sync_with_retry(
|
||||
spec: TaskSpec[object],
|
||||
spec: TaskSpec[Any],
|
||||
context: Mapping[str, Any],
|
||||
layer_idx: int | None,
|
||||
) -> TaskResult[object]:
|
||||
on_event: EventCallback | None = None,
|
||||
) -> TaskResult[Any]:
|
||||
"""执行同步任务并带重试;返回填充好的 TaskResult。"""
|
||||
result: TaskResult[object] = TaskResult(spec=spec)
|
||||
result: TaskResult[Any] = TaskResult(spec=spec)
|
||||
|
||||
# 检查条件是否满足
|
||||
if spec.conditions and not spec.should_execute():
|
||||
if not spec.should_execute():
|
||||
result.status = TaskStatus.SKIPPED
|
||||
result.finished_at = datetime.now()
|
||||
logger.info("task %r skipped (条件不满足)", spec.name)
|
||||
@@ -112,21 +116,22 @@ def _run_sync_with_retry(
|
||||
except Exception as exc:
|
||||
result.error = exc
|
||||
if result.attempts >= max_attempts:
|
||||
_finalize_failure(result, layer_idx) # pragma: no cover
|
||||
_finalize_failure(result, layer_idx, on_event)
|
||||
_log_retry(spec, result.attempts, max_attempts, exc)
|
||||
raise AssertionError("unreachable") # pragma: no cover
|
||||
|
||||
|
||||
async def _run_async_with_retry(
|
||||
spec: TaskSpec[object],
|
||||
spec: TaskSpec[Any],
|
||||
context: Mapping[str, Any],
|
||||
layer_idx: int | None,
|
||||
) -> TaskResult[object]:
|
||||
on_event: EventCallback | None = None,
|
||||
) -> TaskResult[Any]:
|
||||
"""在事件循环上执行任务(同步或异步)并带重试。"""
|
||||
result: TaskResult[object] = TaskResult(spec=spec)
|
||||
result: TaskResult[Any] = TaskResult[Any](spec=spec)
|
||||
|
||||
# 检查条件是否满足
|
||||
if spec.conditions and not spec.should_execute():
|
||||
if not spec.should_execute():
|
||||
result.status = TaskStatus.SKIPPED
|
||||
result.finished_at = datetime.now()
|
||||
logger.info("task %r skipped (条件不满足)", spec.name)
|
||||
@@ -152,9 +157,7 @@ async def _run_async_with_retry(
|
||||
return spec.effective_fn(*args, **kwargs)
|
||||
|
||||
if spec.timeout is not None:
|
||||
result.value = await asyncio.wait_for(
|
||||
loop.run_in_executor(None, fn_call), timeout=spec.timeout
|
||||
)
|
||||
result.value = await asyncio.wait_for(loop.run_in_executor(None, fn_call), timeout=spec.timeout)
|
||||
else:
|
||||
result.value = await loop.run_in_executor(None, fn_call)
|
||||
result.status = TaskStatus.SUCCESS
|
||||
@@ -163,7 +166,7 @@ async def _run_async_with_retry(
|
||||
except asyncio.TimeoutError:
|
||||
result.error = TaskTimeoutError(spec.name, spec.timeout or 0.0)
|
||||
if result.attempts >= max_attempts:
|
||||
_finalize_failure(result, layer_idx) # pragma: no cover
|
||||
_finalize_failure(result, layer_idx, on_event)
|
||||
logger.warning(
|
||||
"task %r timed out (attempt %d/%d); retrying",
|
||||
spec.name,
|
||||
@@ -173,8 +176,8 @@ async def _run_async_with_retry(
|
||||
except Exception as exc:
|
||||
result.error = exc
|
||||
if result.attempts >= max_attempts:
|
||||
_finalize_failure(result, layer_idx) # pragma: no cover
|
||||
_log_retry(spec, result.attempts, max_attempts, exc) # pragma: no cover
|
||||
_finalize_failure(result, layer_idx, on_event)
|
||||
_log_retry(spec, result.attempts, max_attempts, exc)
|
||||
raise AssertionError("unreachable") # pragma: no cover
|
||||
|
||||
|
||||
@@ -182,13 +185,11 @@ async def _run_async_with_retry(
|
||||
# 层驱动器
|
||||
# ---------------------------------------------------------------------- #
|
||||
def _build_context(
|
||||
spec: TaskSpec[object],
|
||||
spec: TaskSpec[Any],
|
||||
global_context: Mapping[str, Any],
|
||||
) -> Mapping[str, Any]:
|
||||
"""将全局上下文限制为本任务的依赖。"""
|
||||
return {
|
||||
dep: global_context[dep] for dep in spec.depends_on if dep in global_context
|
||||
}
|
||||
return {dep: global_context[dep] for dep in spec.depends_on if dep in global_context}
|
||||
|
||||
|
||||
def _execute_layer_sequential(
|
||||
@@ -211,7 +212,7 @@ def _execute_layer_sequential(
|
||||
_emit(on_event, result)
|
||||
logger.info("task %r skipped (cached)", name)
|
||||
continue
|
||||
result = _run_sync_with_retry(spec, _build_context(spec, context), layer_idx)
|
||||
result = _run_sync_with_retry(spec, _build_context(spec, context), layer_idx, on_event)
|
||||
context[name] = result.value
|
||||
backend.save(name, result.value)
|
||||
report.results[name] = result
|
||||
@@ -235,9 +236,7 @@ def _execute_layer_threaded(
|
||||
if backend.has(name):
|
||||
cached = backend.get(name)
|
||||
context[name] = cached
|
||||
result = TaskResult(
|
||||
spec=graph.spec(name), status=TaskStatus.SKIPPED, value=cached
|
||||
)
|
||||
result = TaskResult(spec=graph.spec(name), status=TaskStatus.SKIPPED, value=cached)
|
||||
report.results[name] = result
|
||||
_emit(on_event, result)
|
||||
else:
|
||||
@@ -247,12 +246,12 @@ def _execute_layer_threaded(
|
||||
return
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||
future_to_name: dict[concurrent.futures.Future[TaskResult[object]], str] = {}
|
||||
future_to_name: dict[concurrent.futures.Future[TaskResult[Any]], str] = {}
|
||||
for name in to_run:
|
||||
spec = graph.spec(name)
|
||||
# 为本任务快照上下文以避免竞态。
|
||||
task_ctx = _build_context(spec, context)
|
||||
fut = pool.submit(_run_sync_with_retry, spec, task_ctx, layer_idx)
|
||||
fut = pool.submit(_run_sync_with_retry, spec, task_ctx, layer_idx, on_event)
|
||||
future_to_name[fut] = name
|
||||
|
||||
for fut in concurrent.futures.as_completed(future_to_name):
|
||||
@@ -279,9 +278,7 @@ async def _execute_layer_async(
|
||||
if backend.has(name):
|
||||
cached = backend.get(name)
|
||||
context[name] = cached
|
||||
result = TaskResult(
|
||||
spec=graph.spec(name), status=TaskStatus.SKIPPED, value=cached
|
||||
)
|
||||
result = TaskResult(spec=graph.spec(name), status=TaskStatus.SKIPPED, value=cached)
|
||||
report.results[name] = result
|
||||
_emit(on_event, result)
|
||||
else:
|
||||
@@ -294,7 +291,7 @@ async def _execute_layer_async(
|
||||
for name in to_run:
|
||||
spec = graph.spec(name)
|
||||
task_ctx = _build_context(spec, context)
|
||||
coros.append(_run_async_with_retry(spec, task_ctx, layer_idx))
|
||||
coros.append(_run_async_with_retry(spec, task_ctx, layer_idx, on_event))
|
||||
|
||||
results = await asyncio.gather(*coros)
|
||||
for name, result in zip(to_run, results):
|
||||
@@ -326,7 +323,7 @@ def _make_verbose_callback(
|
||||
def _verbose_callback(event: TaskEvent) -> None:
|
||||
# 先打印生命周期信息
|
||||
dur = f" ({event.duration:.3f}s)" if event.duration is not None else ""
|
||||
if event.status == TaskStatus.RUNNING:
|
||||
if event.status == TaskStatus.RUNNING: # pragma: no cover
|
||||
print(f"[verbose] 任务 {event.task!r} 开始执行...", flush=True)
|
||||
elif event.status == TaskStatus.SUCCESS:
|
||||
print(f"[verbose] 任务 {event.task!r} 成功{dur}", flush=True)
|
||||
@@ -336,8 +333,11 @@ def _make_verbose_callback(
|
||||
f"[verbose] 任务 {event.task!r} 失败{dur} (尝试 {event.attempts} 次){err}",
|
||||
flush=True,
|
||||
)
|
||||
elif event.status == TaskStatus.SKIPPED:
|
||||
elif event.status == TaskStatus.SKIPPED: # pragma: no branch
|
||||
print(f"[verbose] 任务 {event.task!r} 跳过", flush=True)
|
||||
else: # pragma: no cover
|
||||
# 不可达: 执行器只发出 RUNNING/SUCCESS/FAILED/SKIPPED 事件
|
||||
pass
|
||||
# 再调用用户回调
|
||||
if on_event is not None:
|
||||
on_event(event)
|
||||
@@ -389,19 +389,12 @@ def run(
|
||||
graph.validate()
|
||||
layers = graph.layers()
|
||||
|
||||
# 验证策略是否有效
|
||||
valid_strategies = ("sequential", "thread", "async")
|
||||
if strategy not in valid_strategies:
|
||||
raise ValueError(f"unknown strategy: {strategy}. Valid: {valid_strategies}")
|
||||
|
||||
if dry_run:
|
||||
_print_dry_run(graph, layers)
|
||||
return RunReport(success=True)
|
||||
|
||||
# verbose 模式下包装事件回调
|
||||
effective_callback: EventCallback | None = (
|
||||
_make_verbose_callback(on_event) if verbose else on_event
|
||||
)
|
||||
effective_callback: EventCallback | None = _make_verbose_callback(on_event) if verbose else on_event
|
||||
|
||||
backend = resolve_backend(state)
|
||||
report = RunReport()
|
||||
@@ -409,13 +402,9 @@ def run(
|
||||
|
||||
try:
|
||||
if strategy == "sequential":
|
||||
_drive_sequential(
|
||||
graph, layers, context, report, backend, effective_callback
|
||||
)
|
||||
_drive_sequential(graph, layers, context, report, backend, effective_callback)
|
||||
elif strategy == "thread":
|
||||
_drive_threaded(
|
||||
graph, layers, context, report, backend, effective_callback, max_workers
|
||||
)
|
||||
_drive_threaded(graph, layers, context, report, backend, effective_callback, max_workers)
|
||||
else:
|
||||
_drive_async(graph, layers, context, report, backend, effective_callback)
|
||||
except TaskFailedError:
|
||||
@@ -457,9 +446,7 @@ def _drive_threaded(
|
||||
) -> None:
|
||||
for idx, layer in enumerate(layers, 1):
|
||||
workers = max_workers or max(1, min(32, len(layer)))
|
||||
_execute_layer_threaded(
|
||||
layer, graph, context, report, backend, idx, on_event, workers
|
||||
)
|
||||
_execute_layer_threaded(layer, graph, context, report, backend, idx, on_event, workers)
|
||||
|
||||
|
||||
def _drive_async(
|
||||
@@ -482,6 +469,4 @@ async def _async_drive(
|
||||
on_event: EventCallback | None,
|
||||
) -> None:
|
||||
for idx, layer in enumerate(layers, 1):
|
||||
await _execute_layer_async(
|
||||
layer, graph, context, report, backend, idx, on_event
|
||||
)
|
||||
await _execute_layer_async(layer, graph, context, report, backend, idx, on_event)
|
||||
|
||||
+40
-45
@@ -1,14 +1,14 @@
|
||||
"""DAG 构建、校验、分层与可视化。
|
||||
|
||||
使用标准库的 :mod:`graphlib`(3.9+)或 :mod:`graphlib_backport`(3.8)
|
||||
进行拓扑排序。图以增量方式构建并即时校验,使配置错误在构建时(而非
|
||||
执行时)快速失败。
|
||||
进行拓扑排序。图以增量方式构建并即时校验,使配置错误在构建时(而非执行时)快速失败。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from typing import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterable, Mapping, Sequence
|
||||
|
||||
from .errors import CycleError, DuplicateTaskError, MissingDependencyError
|
||||
from .task import TaskSpec
|
||||
@@ -24,6 +24,7 @@ else: # pragma: no cover
|
||||
_TopologicalSorter = graphlib.TopologicalSorter # pragma: no cover
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Graph:
|
||||
"""校验后不可变的有向无环任务图。
|
||||
|
||||
@@ -35,30 +36,28 @@ class Graph:
|
||||
这使图可安全重复运行并在线程间共享。
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._specs: dict[str, TaskSpec[object]] = {}
|
||||
# 任务 -> 其直接依赖(前驱)。
|
||||
self._deps: dict[str, tuple[str, ...]] = {}
|
||||
specs: dict[str, TaskSpec[Any]] = field(default_factory=dict)
|
||||
deps: dict[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 构建
|
||||
# ------------------------------------------------------------------ #
|
||||
def add(self, spec: TaskSpec[object]) -> Graph:
|
||||
def add(self, spec: TaskSpec[Any]) -> Graph:
|
||||
"""注册一个任务 spec,并即时校验。
|
||||
|
||||
返回 ``self`` 以支持链式调用,但推荐入口是 :meth:`from_specs`,
|
||||
它会整批校验(允许单次调用中的前向引用)。
|
||||
"""
|
||||
if spec.name in self._specs:
|
||||
if spec.name in self.specs:
|
||||
raise DuplicateTaskError(spec.name)
|
||||
self._specs[spec.name] = spec
|
||||
self._deps[spec.name] = spec.depends_on
|
||||
self.specs[spec.name] = spec
|
||||
self.deps[spec.name] = spec.depends_on
|
||||
# 为增量 API 即时检查重名与缺失依赖。
|
||||
self._validate_references()
|
||||
return self
|
||||
|
||||
@classmethod
|
||||
def from_specs(cls, specs: Iterable[TaskSpec[object]]) -> Graph:
|
||||
def from_specs(cls, specs: Iterable[TaskSpec[Any]]) -> Graph:
|
||||
"""从可迭代的 task spec 构建图。
|
||||
|
||||
先收集所有 spec,再统一校验。这意味着任务可以引用*后出现*的
|
||||
@@ -66,10 +65,10 @@ class Graph:
|
||||
"""
|
||||
graph = cls()
|
||||
for spec in specs:
|
||||
if spec.name in graph._specs:
|
||||
if spec.name in graph.specs:
|
||||
raise DuplicateTaskError(spec.name)
|
||||
graph._specs[spec.name] = spec
|
||||
graph._deps[spec.name] = spec.depends_on
|
||||
graph.specs[spec.name] = spec
|
||||
graph.deps[spec.name] = spec.depends_on
|
||||
graph._validate_references()
|
||||
graph.validate()
|
||||
return graph
|
||||
@@ -79,9 +78,9 @@ class Graph:
|
||||
# ------------------------------------------------------------------ #
|
||||
def _validate_references(self) -> None:
|
||||
"""确保每个依赖名都存在于图中。"""
|
||||
for name, deps in self._deps.items():
|
||||
for name, deps in self.deps.items():
|
||||
for dep in deps:
|
||||
if dep not in self._specs:
|
||||
if dep not in self.specs:
|
||||
raise MissingDependencyError(name, dep)
|
||||
|
||||
def validate(self) -> None:
|
||||
@@ -91,7 +90,7 @@ class Graph:
|
||||
依赖存在性由 :meth:`_validate_references` 检查。
|
||||
"""
|
||||
self._validate_references()
|
||||
sorter = _TopologicalSorter(self._deps)
|
||||
sorter = _TopologicalSorter(self.deps)
|
||||
try:
|
||||
# prepare() 在有环时抛出 CycleError;此处不需要
|
||||
# static_order() 的结果,仅利用其校验副作用。
|
||||
@@ -107,19 +106,19 @@ class Graph:
|
||||
@property
|
||||
def names(self) -> list[str]:
|
||||
"""所有已注册任务名(按插入顺序)。"""
|
||||
return list(self._specs.keys())
|
||||
return list(self.specs.keys())
|
||||
|
||||
def spec(self, name: str) -> TaskSpec[object]:
|
||||
def spec(self, name: str) -> TaskSpec[Any]:
|
||||
"""返回 ``name`` 的 spec;不存在则 ``KeyError``。"""
|
||||
return self._specs[name]
|
||||
return self.specs[name]
|
||||
|
||||
def dependencies(self, name: str) -> tuple[str, ...]:
|
||||
"""``name`` 的直接前驱。"""
|
||||
return self._deps[name]
|
||||
return self.deps[name]
|
||||
|
||||
def all_specs(self) -> Mapping[str, TaskSpec[object]]:
|
||||
def all_specs(self) -> Mapping[str, TaskSpec[Any]]:
|
||||
"""name -> spec 的只读视图。"""
|
||||
return self._specs
|
||||
return self.specs
|
||||
|
||||
def layers(self) -> list[list[str]]:
|
||||
"""将任务分组为可并行执行的层(Kahn 算法)。
|
||||
@@ -129,7 +128,7 @@ class Graph:
|
||||
图有环时抛出 :class:`~pyflowx.errors.CycleError`。
|
||||
"""
|
||||
self.validate()
|
||||
sorter = _TopologicalSorter(self._deps)
|
||||
sorter = _TopologicalSorter(self.deps)
|
||||
result: list[list[str]] = []
|
||||
# ``get_ready`` + ``done`` 每次给出一层,正好是并行执行所需的分组。
|
||||
sorter.prepare()
|
||||
@@ -153,16 +152,14 @@ class Graph:
|
||||
DAG 的切片。
|
||||
"""
|
||||
wanted: set[str] = set(tags)
|
||||
kept: list[TaskSpec[object]] = []
|
||||
for spec in self._specs.values():
|
||||
kept: list[TaskSpec[Any]] = []
|
||||
for spec in self.specs.values():
|
||||
if wanted & set(spec.tags):
|
||||
pruned_deps = tuple(
|
||||
d
|
||||
for d in spec.depends_on
|
||||
if d in self._specs and (wanted & set(self._specs[d].tags))
|
||||
d for d in spec.depends_on if d in self.specs and (wanted & set(self.specs[d].tags))
|
||||
)
|
||||
kept.append(
|
||||
TaskSpec(
|
||||
TaskSpec[Any](
|
||||
name=spec.name,
|
||||
fn=spec.fn,
|
||||
cmd=spec.cmd,
|
||||
@@ -182,14 +179,14 @@ class Graph:
|
||||
"""返回限定于 ``names`` 的新图(边已修剪)。"""
|
||||
wanted: set[str] = set(names)
|
||||
for n in wanted:
|
||||
if n not in self._specs:
|
||||
if n not in self.specs:
|
||||
raise KeyError(f"Unknown task name: {n!r}")
|
||||
kept: list[TaskSpec[object]] = []
|
||||
for spec in self._specs.values():
|
||||
kept: list[TaskSpec[Any]] = []
|
||||
for spec in self.specs.values():
|
||||
if spec.name in wanted:
|
||||
pruned_deps = tuple(d for d in spec.depends_on if d in wanted)
|
||||
kept.append(
|
||||
TaskSpec(
|
||||
TaskSpec[Any](
|
||||
name=spec.name,
|
||||
fn=spec.fn,
|
||||
cmd=spec.cmd,
|
||||
@@ -217,13 +214,11 @@ class Graph:
|
||||
valid = {"TD", "TB", "BT", "LR", "RL"}
|
||||
orientation = orientation.upper()
|
||||
if orientation not in valid:
|
||||
raise ValueError(
|
||||
f"Invalid orientation {orientation!r}; expected one of {sorted(valid)}."
|
||||
)
|
||||
raise ValueError(f"Invalid orientation {orientation!r}; expected one of {sorted(valid)}.")
|
||||
lines: list[str] = [f"graph {orientation}"]
|
||||
for name in self._specs:
|
||||
for name in self.specs:
|
||||
lines.append(f' {name}["{name}"]')
|
||||
for name, deps in self._deps.items():
|
||||
for name, deps in self.deps.items():
|
||||
for dep in deps:
|
||||
lines.append(f" {dep} --> {name}")
|
||||
return "\n".join(lines) + "\n"
|
||||
@@ -233,16 +228,16 @@ class Graph:
|
||||
# ------------------------------------------------------------------ #
|
||||
def describe(self) -> str:
|
||||
"""用于调试的人类可读多行摘要。"""
|
||||
out: list[str] = [f"Graph(tasks={len(self._specs)})"]
|
||||
out: list[str] = [f"Graph(tasks={len(self.specs)})"]
|
||||
for layer_idx, layer in enumerate(self.layers(), 1):
|
||||
out.append(f" Layer {layer_idx}: {layer}")
|
||||
return "\n".join(out)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Graph(tasks={len(self._specs)})"
|
||||
return f"Graph(tasks={len(self.specs)})"
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._specs)
|
||||
return len(self.specs)
|
||||
|
||||
def __contains__(self, name: object) -> bool:
|
||||
return name in self._specs
|
||||
def __contains__(self, name: Any) -> bool:
|
||||
return name in self.specs
|
||||
|
||||
@@ -24,7 +24,7 @@ class RunReport:
|
||||
当且仅当所有非跳过任务都以 ``SUCCESS`` 结束时为 ``True``。
|
||||
"""
|
||||
|
||||
results: dict[str, TaskResult[object]] = field(default_factory=dict)
|
||||
results: dict[str, TaskResult[Any]] = field(default_factory=dict)
|
||||
success: bool = True
|
||||
|
||||
# ---- 类型化访问 --------------------------------------------------- #
|
||||
@@ -36,11 +36,11 @@ class RunReport:
|
||||
"""
|
||||
return self.results[name].value
|
||||
|
||||
def result_of(self, name: str) -> TaskResult[object]:
|
||||
def result_of(self, name: str) -> TaskResult[Any]:
|
||||
"""返回 ``name`` 的完整 :class:`TaskResult`。"""
|
||||
return self.results[name]
|
||||
|
||||
def __contains__(self, name: object) -> bool:
|
||||
def __contains__(self, name: Any) -> bool:
|
||||
return name in self.results
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
@@ -67,9 +67,7 @@ class RunReport:
|
||||
|
||||
def failed_tasks(self) -> list[str]:
|
||||
"""以 FAILED 状态结束的任务名列表。"""
|
||||
return [
|
||||
name for name, r in self.results.items() if r.status == TaskStatus.FAILED
|
||||
]
|
||||
return [name for name, r in self.results.items() if r.status == TaskStatus.FAILED]
|
||||
|
||||
def describe(self) -> str:
|
||||
"""用于调试的人类可读多行报告。"""
|
||||
@@ -77,7 +75,5 @@ class RunReport:
|
||||
for name, r in self.results.items():
|
||||
dur = f"{r.duration:.3f}s" if r.duration is not None else "-"
|
||||
err = f" error={r.error!r}" if r.error else ""
|
||||
lines.append(
|
||||
f" {name}: {r.status.value} ({dur} attempts={r.attempts}){err}"
|
||||
)
|
||||
lines.append(f" {name}: {r.status.value} ({dur} attempts={r.attempts}){err}")
|
||||
return "\n".join(lines)
|
||||
|
||||
+8
-17
@@ -15,7 +15,7 @@ import argparse
|
||||
import enum
|
||||
import sys
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import Sequence
|
||||
from typing import Any, Sequence, get_args
|
||||
|
||||
from .errors import PyFlowXError
|
||||
from .executors import Strategy, run
|
||||
@@ -51,7 +51,7 @@ def _apply_verbose_to_graph(graph: Graph, verbose: bool) -> Graph:
|
||||
Graph
|
||||
所有 spec 的 verbose 字段已更新的新图.
|
||||
"""
|
||||
new_specs: list[TaskSpec[object]] = []
|
||||
new_specs: list[TaskSpec[Any]] = []
|
||||
for spec in graph.all_specs().values():
|
||||
if spec.verbose == verbose:
|
||||
new_specs.append(spec)
|
||||
@@ -60,7 +60,7 @@ def _apply_verbose_to_graph(graph: Graph, verbose: bool) -> Graph:
|
||||
return Graph.from_specs(new_specs)
|
||||
|
||||
|
||||
@dataclass
|
||||
@dataclass(frozen=True)
|
||||
class CliRunner:
|
||||
"""命令行运行器: 根据用户输入执行对应的任务流图.
|
||||
|
||||
@@ -116,9 +116,7 @@ class CliRunner:
|
||||
|
||||
for name, graph in self.graphs.items():
|
||||
if not isinstance(graph, Graph):
|
||||
raise TypeError(
|
||||
f"CliRunner 命令 {name!r} 的值必须是 Graph 实例, 实际是 {type(graph).__name__}"
|
||||
)
|
||||
raise TypeError(f"CliRunner 命令 {name!r} 的值必须是 Graph 实例, 实际是 {type(graph).__name__}")
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 内省
|
||||
@@ -162,8 +160,8 @@ class CliRunner:
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"--strategy",
|
||||
choices=list(Strategy.__args__),
|
||||
default="sequential",
|
||||
choices=list(get_args(Strategy)),
|
||||
default=self.strategy,
|
||||
help="执行策略 (默认: %(default)s)",
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
@@ -185,10 +183,7 @@ class CliRunner:
|
||||
|
||||
def _format_commands_help(self) -> str:
|
||||
"""格式化命令帮助文本."""
|
||||
lines = ["可用命令:"]
|
||||
for cmd in self.graphs:
|
||||
lines.append(f" {cmd}")
|
||||
return "\n".join(lines)
|
||||
return "可用命令:\n" + " | ".join(self.graphs.keys())
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 执行
|
||||
@@ -249,11 +244,7 @@ class CliRunner:
|
||||
dry_run=parsed.dry_run,
|
||||
verbose=verbose,
|
||||
)
|
||||
return (
|
||||
CliExitCode.SUCCESS.value
|
||||
if report.success
|
||||
else CliExitCode.FAILURE.value
|
||||
)
|
||||
return CliExitCode.SUCCESS.value if report.success else CliExitCode.FAILURE.value
|
||||
except KeyboardInterrupt:
|
||||
print("\n操作已取消", file=sys.stderr)
|
||||
return CliExitCode.INTERRUPTED.value
|
||||
|
||||
@@ -112,9 +112,7 @@ class JSONBackend(StateBackend):
|
||||
try:
|
||||
_ = json.dumps(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise StorageError(
|
||||
f"result of task {name!r} is not JSON-serialisable", exc
|
||||
) from exc
|
||||
raise StorageError(f"result of task {name!r} is not JSON-serialisable", exc) from exc
|
||||
self._store[name] = value
|
||||
self._flush()
|
||||
|
||||
|
||||
+35
-13
@@ -112,6 +112,12 @@ class TaskSpec(Generic[T]):
|
||||
是否在命令执行时显示详细输出。``True`` 时会打印执行的命令
|
||||
及其标准输出/标准错误。仅在使用 ``cmd`` 参数时有效。
|
||||
``False`` 时静默捕获输出(失败时仍会包含在错误信息中)。
|
||||
skip_if_missing:
|
||||
仅对 ``cmd`` 为 ``list[str]`` 的任务有效。``True`` 时自动检查
|
||||
命令是否存在(通过 :func:`shutil.which`),不存在则跳过任务
|
||||
(标记为 SKIPPED)而非失败。适用于构建工具场景,避免因未安装
|
||||
某些工具(如 maturin、tox)而导致整个图执行失败。
|
||||
对于 ``str`` (shell) 和 ``Callable`` 类型的 ``cmd``,此参数无效。
|
||||
"""
|
||||
|
||||
name: str
|
||||
@@ -126,6 +132,7 @@ class TaskSpec(Generic[T]):
|
||||
conditions: Tuple[Condition, ...] = ()
|
||||
cwd: Optional[Path] = None
|
||||
verbose: bool = False
|
||||
skip_if_missing: bool = True
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.name:
|
||||
@@ -151,7 +158,7 @@ class TaskSpec(Generic[T]):
|
||||
if self.fn is not None:
|
||||
return self.fn
|
||||
|
||||
raise ValueError(f"TaskSpec '{self.name}': 没有可执行的函数或命令。")
|
||||
raise ValueError(f"TaskSpec '{self.name}': 没有可执行的函数或命令。") # pragma: no cover
|
||||
|
||||
def _wrap_cmd(self) -> TaskFn[Any]:
|
||||
"""将 cmd 包装为可执行函数.
|
||||
@@ -188,9 +195,7 @@ class TaskSpec(Generic[T]):
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError(f"命令未找到: {cmd_str}") from None
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError(
|
||||
f"命令执行超时: {cmd_str} ({timeout}s)"
|
||||
) from None
|
||||
raise RuntimeError(f"命令执行超时: {cmd_str} ({timeout}s)") from None
|
||||
except OSError as e:
|
||||
raise RuntimeError(f"命令执行异常: {cmd_str}: {e}") from e
|
||||
|
||||
@@ -230,9 +235,7 @@ class TaskSpec(Generic[T]):
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError(f"Shell 命令未找到: {cmd}") from None
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError(
|
||||
f"Shell 命令执行超时: {cmd} ({timeout}s)"
|
||||
) from None
|
||||
raise RuntimeError(f"Shell 命令执行超时: {cmd} ({timeout}s)") from None
|
||||
except OSError as e:
|
||||
raise RuntimeError(f"Shell 命令执行异常: {cmd}: {e}") from e
|
||||
|
||||
@@ -253,9 +256,7 @@ class TaskSpec(Generic[T]):
|
||||
if callable(cmd):
|
||||
return cmd # type: ignore[return-value]
|
||||
|
||||
raise TypeError(
|
||||
f"TaskSpec '{self.name}': 不支持的 cmd 类型 {type(cmd).__name__}"
|
||||
)
|
||||
raise TypeError(f"TaskSpec '{self.name}': 不支持的 cmd 类型 {type(cmd).__name__}") # pragma: no cover
|
||||
|
||||
def should_execute(self) -> bool:
|
||||
"""检查任务是否应该执行.
|
||||
@@ -263,10 +264,31 @@ class TaskSpec(Generic[T]):
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
若所有条件都返回 ``True``,则返回 ``True``;
|
||||
否则返回 ``False``。
|
||||
若所有条件都返回 ``True``,且 ``skip_if_missing`` 检查通过,
|
||||
则返回 ``True``;否则返回 ``False``。
|
||||
"""
|
||||
return all(condition() for condition in self.conditions)
|
||||
if not all(condition() for condition in self.conditions):
|
||||
return False
|
||||
|
||||
return not (self.skip_if_missing and not self._is_cmd_available())
|
||||
|
||||
def _is_cmd_available(self) -> bool:
|
||||
"""检查 ``cmd`` 是否可用.
|
||||
|
||||
仅对 ``list[str]`` 类型的 ``cmd`` 进行检查(通过 :func:`shutil.which`)。
|
||||
对于 ``str`` (shell) 和 ``Callable`` 类型,始终返回 ``True``。
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
命令可用返回 ``True``,否则返回 ``False``。
|
||||
"""
|
||||
import shutil
|
||||
|
||||
cmd = self.cmd
|
||||
if isinstance(cmd, list) and cmd:
|
||||
return shutil.which(cmd[0]) is not None
|
||||
return True
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
+193
-154
@@ -1,165 +1,204 @@
|
||||
"""Tests for pymake CLI."""
|
||||
"""Tests for cli.pymake module."""
|
||||
|
||||
from pyflowx.cli.pymake import build_graphs, conf, get_maturin_build_command
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from pyflowx.cli import pymake
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
def test_pymake_config_attributes():
|
||||
"""Test PymakeConfig has expected attributes."""
|
||||
assert hasattr(conf, "PROJECT_ROOT")
|
||||
assert hasattr(conf, "BUILD_TOOL")
|
||||
assert hasattr(conf, "BUILD_COMMAND")
|
||||
assert hasattr(conf, "MATURIN_TOOL")
|
||||
assert hasattr(conf, "MATURIN_BUILD_COMMAND")
|
||||
assert hasattr(conf, "MATURIN_DEV_COMMAND")
|
||||
assert hasattr(conf, "TIMEOUT")
|
||||
# ---------------------------------------------------------------------- #
|
||||
# maturin_build_cmd
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestMaturinBuildCmd:
|
||||
"""Test maturin_build_cmd function."""
|
||||
|
||||
def test_returns_list(self) -> None:
|
||||
"""Should return a list."""
|
||||
cmd = pymake.maturin_build_cmd()
|
||||
assert isinstance(cmd, list)
|
||||
|
||||
def test_contains_maturin_build(self) -> None:
|
||||
"""Should contain 'maturin' and 'build'."""
|
||||
cmd = pymake.maturin_build_cmd()
|
||||
assert "maturin" in cmd
|
||||
assert "build" in cmd
|
||||
|
||||
def test_contains_release_flag(self) -> None:
|
||||
"""Should contain release flag '-r'."""
|
||||
cmd = pymake.maturin_build_cmd()
|
||||
assert "-r" in cmd
|
||||
|
||||
def test_windows_includes_target(self) -> None:
|
||||
"""On Windows, should include target-specific flags."""
|
||||
cmd = pymake.maturin_build_cmd()
|
||||
if Constants.IS_WINDOWS:
|
||||
assert "--target" in cmd
|
||||
assert "x86_64-win7-windows-msvc" in cmd
|
||||
assert "-Zbuild-std" in cmd
|
||||
assert "-i" in cmd
|
||||
assert "python3.8" in cmd
|
||||
else:
|
||||
# On non-Windows, should not include Windows-specific flags
|
||||
assert "--target" not in cmd
|
||||
|
||||
def test_does_not_mutate_on_multiple_calls(self) -> None:
|
||||
"""Multiple calls should return independent lists."""
|
||||
cmd1 = pymake.maturin_build_cmd()
|
||||
cmd2 = pymake.maturin_build_cmd()
|
||||
assert cmd1 == cmd2
|
||||
# Mutating one should not affect the other
|
||||
cmd1.append("extra")
|
||||
assert "extra" not in cmd2
|
||||
|
||||
def test_non_windows_excludes_target_flags(self) -> None:
|
||||
"""On non-Windows, should not include Windows-specific flags (覆盖 22->32 分支)."""
|
||||
from unittest.mock import patch
|
||||
|
||||
with patch.object(pymake.Constants, "IS_WINDOWS", False):
|
||||
cmd = pymake.maturin_build_cmd()
|
||||
assert "maturin" in cmd
|
||||
assert "build" in cmd
|
||||
assert "-r" in cmd
|
||||
assert "--target" not in cmd
|
||||
assert "-Zbuild-std" not in cmd
|
||||
|
||||
|
||||
def test_pymake_config_values():
|
||||
"""Test PymakeConfig values are correct."""
|
||||
assert conf.BUILD_TOOL == "uv"
|
||||
assert conf.BUILD_COMMAND == ["uv", "build"]
|
||||
assert conf.MATURIN_TOOL == "maturin"
|
||||
assert conf.TIMEOUT == 600
|
||||
# ---------------------------------------------------------------------- #
|
||||
# TaskSpec definitions
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestTaskSpecDefinitions:
|
||||
"""Test that all TaskSpec definitions are valid."""
|
||||
|
||||
def test_uv_build_spec(self) -> None:
|
||||
"""uv_build spec should be properly defined."""
|
||||
assert pymake.uv_build.name == "uv_build"
|
||||
assert pymake.uv_build.cmd == ["uv", "build"]
|
||||
assert pymake.uv_build.skip_if_missing is True
|
||||
|
||||
def test_maturin_build_spec(self) -> None:
|
||||
"""maturin_build spec should be properly defined."""
|
||||
assert pymake.maturin_build.name == "maturin_build"
|
||||
assert isinstance(pymake.maturin_build.cmd, list)
|
||||
assert pymake.maturin_build.skip_if_missing is True
|
||||
|
||||
def test_uv_sync_spec(self) -> None:
|
||||
"""uv_sync spec should be properly defined."""
|
||||
assert pymake.uv_sync.name == "uv_sync"
|
||||
assert pymake.uv_sync.cmd == ["uv", "sync"]
|
||||
assert pymake.uv_sync.skip_if_missing is True
|
||||
|
||||
def test_git_clean_spec(self) -> None:
|
||||
"""git_clean spec should be properly defined."""
|
||||
assert pymake.git_clean.name == "git_clean"
|
||||
assert pymake.git_clean.cmd == ["gitt", "c"]
|
||||
assert pymake.git_clean.skip_if_missing is True
|
||||
|
||||
def test_test_spec(self) -> None:
|
||||
"""test spec should be properly defined."""
|
||||
assert pymake.test.name == "test"
|
||||
assert isinstance(pymake.test.cmd, list)
|
||||
assert "pytest" in pymake.test.cmd
|
||||
assert "-m" in pymake.test.cmd
|
||||
assert "not slow" in pymake.test.cmd
|
||||
assert pymake.test.skip_if_missing is True
|
||||
|
||||
def test_test_fast_spec(self) -> None:
|
||||
"""test_fast spec should be properly defined."""
|
||||
assert pymake.test_fast.name == "test_fast"
|
||||
assert isinstance(pymake.test_fast.cmd, list)
|
||||
assert "pytest" in pymake.test_fast.cmd
|
||||
assert "-n" not in pymake.test_fast.cmd # test_fast doesn't use parallel
|
||||
assert pymake.test_fast.skip_if_missing is True
|
||||
|
||||
def test_test_coverage_spec(self) -> None:
|
||||
"""test_coverage spec should be properly defined."""
|
||||
assert pymake.test_coverage.name == "test_coverage"
|
||||
assert isinstance(pymake.test_coverage.cmd, list)
|
||||
assert "pytest" in pymake.test_coverage.cmd
|
||||
assert "--cov" in pymake.test_coverage.cmd
|
||||
assert pymake.test_coverage.skip_if_missing is True
|
||||
|
||||
def test_ruff_lint_spec(self) -> None:
|
||||
"""ruff_lint spec should be properly defined."""
|
||||
assert pymake.ruff_lint.name == "lint"
|
||||
assert isinstance(pymake.ruff_lint.cmd, list)
|
||||
assert "ruff" in pymake.ruff_lint.cmd
|
||||
assert "check" in pymake.ruff_lint.cmd
|
||||
assert pymake.ruff_lint.skip_if_missing is True
|
||||
|
||||
def test_mypy_check_spec(self) -> None:
|
||||
"""mypy_check spec should be properly defined."""
|
||||
assert pymake.mypy_check.name == "typecheck"
|
||||
assert pymake.mypy_check.cmd == ["mypy", "."]
|
||||
assert pymake.mypy_check.skip_if_missing is True
|
||||
|
||||
def test_ty_check_spec(self) -> None:
|
||||
"""ty_check spec should be properly defined."""
|
||||
assert pymake.ty_check.name == "ty_check"
|
||||
assert pymake.ty_check.cmd == ["ty", "check", "."]
|
||||
assert pymake.ty_check.skip_if_missing is True
|
||||
|
||||
def test_doc_spec(self) -> None:
|
||||
"""doc spec should be properly defined."""
|
||||
assert pymake.doc.name == "doc"
|
||||
assert isinstance(pymake.doc.cmd, list)
|
||||
assert "sphinx-build" in pymake.doc.cmd
|
||||
assert pymake.doc.skip_if_missing is True
|
||||
|
||||
def test_hatch_publish_spec(self) -> None:
|
||||
"""hatch_publish spec should be properly defined."""
|
||||
assert pymake.hatch_publish.name == "publish_python"
|
||||
assert pymake.hatch_publish.cmd == ["hatch", "publish"]
|
||||
assert pymake.hatch_publish.skip_if_missing is True
|
||||
|
||||
def test_twine_publish_spec(self) -> None:
|
||||
"""twine_publish spec should be properly defined."""
|
||||
assert pymake.twine_publish.name == "twine_publish"
|
||||
assert isinstance(pymake.twine_publish.cmd, list)
|
||||
assert "twine" in pymake.twine_publish.cmd
|
||||
assert "upload" in pymake.twine_publish.cmd
|
||||
assert pymake.twine_publish.skip_if_missing is True
|
||||
|
||||
def test_tox_spec(self) -> None:
|
||||
"""tox spec should be properly defined."""
|
||||
assert pymake.tox.name == "tox"
|
||||
assert pymake.tox.cmd == ["tox", "-p", "auto"]
|
||||
assert pymake.tox.skip_if_missing is True
|
||||
|
||||
|
||||
def test_get_maturin_build_command_basic():
|
||||
"""Test get_maturin_build_command returns base command."""
|
||||
cmd = get_maturin_build_command()
|
||||
assert "maturin" in cmd
|
||||
assert "build" in cmd
|
||||
assert "-r" in cmd
|
||||
# ---------------------------------------------------------------------- #
|
||||
# main function
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestMain:
|
||||
"""Test main function."""
|
||||
|
||||
def test_main_calls_run_cli(self) -> None:
|
||||
"""main() should create a CliRunner and call run_cli()."""
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
pymake.main()
|
||||
# run_cli() calls sys.exit(), so we should get SystemExit
|
||||
# The exit code depends on whether any commands are available
|
||||
assert exc_info.value.code in (0, 1, 2)
|
||||
|
||||
def testbuild_graphs_returns_dict():
|
||||
"""Test build_graphs returns a dictionary."""
|
||||
graphs = build_graphs()
|
||||
assert isinstance(graphs, dict)
|
||||
assert len(graphs) > 0
|
||||
def test_main_with_list_argument(self) -> None:
|
||||
"""main() should handle --list argument."""
|
||||
with patch("sys.argv", ["pymake", "--list"]), pytest.raises(SystemExit) as exc_info:
|
||||
pymake.main()
|
||||
assert exc_info.value.code == 0
|
||||
|
||||
def test_main_creates_runner_with_multiple_commands(self) -> None:
|
||||
"""main() should create a CliRunner with multiple commands."""
|
||||
# We can't easily test the runner creation without mocking,
|
||||
# but we can verify that main() doesn't raise an error for --list
|
||||
with patch("sys.argv", ["pymake", "--list"]), pytest.raises(SystemExit):
|
||||
pymake.main()
|
||||
|
||||
def testbuild_graphs_has_expected_commands():
|
||||
"""Test build_graphs has expected command keys."""
|
||||
graphs = build_graphs()
|
||||
expected_commands = [
|
||||
"b",
|
||||
"bc",
|
||||
"ba",
|
||||
"ic",
|
||||
"ip",
|
||||
"ia",
|
||||
"cp",
|
||||
"cc",
|
||||
"ca",
|
||||
"t",
|
||||
"lint",
|
||||
]
|
||||
for cmd in expected_commands:
|
||||
assert cmd in graphs, f"Expected command '{cmd}' not found in graphs"
|
||||
|
||||
|
||||
def testbuild_graphs_values_are_graphs():
|
||||
"""Test build_graphs values are Graph instances."""
|
||||
import pyflowx as px
|
||||
|
||||
graphs = build_graphs()
|
||||
for name, graph in graphs.items():
|
||||
assert isinstance(graph, px.Graph), (
|
||||
f"Graph for command '{name}' is not a Graph instance"
|
||||
)
|
||||
|
||||
|
||||
def test_build_command_graph_structure():
|
||||
"""Test 'b' command graph has correct structure."""
|
||||
|
||||
graphs = build_graphs()
|
||||
graph = graphs["b"]
|
||||
assert len(graph.all_specs()) == 1
|
||||
spec = graph.spec("uv_build")
|
||||
assert spec.cmd == conf.BUILD_COMMAND
|
||||
|
||||
|
||||
def test_build_all_command_graph_structure():
|
||||
"""Test 'ba' command graph has correct dependencies."""
|
||||
|
||||
graphs = build_graphs()
|
||||
graph = graphs["ba"]
|
||||
specs = graph.all_specs()
|
||||
assert len(specs) == 2
|
||||
# Check dependency
|
||||
uv_build_spec = graph.spec("uv_build")
|
||||
assert "maturin_build" in uv_build_spec.depends_on
|
||||
|
||||
|
||||
def test_maturin_build_command_graph_structure():
|
||||
"""Test 'bc' command graph has correct structure."""
|
||||
graphs = build_graphs()
|
||||
graph = graphs["bc"]
|
||||
specs = graph.all_specs()
|
||||
assert len(specs) == 1
|
||||
spec = graph.spec("maturin_build")
|
||||
assert spec.cmd == get_maturin_build_command()
|
||||
|
||||
|
||||
def test_install_all_command_graph_structure():
|
||||
"""Test 'ia' command graph has correct dependencies."""
|
||||
graphs = build_graphs()
|
||||
graph = graphs["ia"]
|
||||
specs = graph.all_specs()
|
||||
assert len(specs) == 2
|
||||
uv_install_spec = graph.spec("uv_install")
|
||||
assert "maturin_dev" in uv_install_spec.depends_on
|
||||
|
||||
|
||||
def test_clean_all_command_graph_structure():
|
||||
"""Test 'ca' command graph has correct structure."""
|
||||
graphs = build_graphs()
|
||||
graph = graphs["ca"]
|
||||
specs = graph.all_specs()
|
||||
assert len(specs) == 2
|
||||
|
||||
|
||||
def test_test_command_graph_structure():
|
||||
"""Test 't' command graph has correct structure."""
|
||||
graphs = build_graphs()
|
||||
graph = graphs["t"]
|
||||
specs = graph.all_specs()
|
||||
assert len(specs) == 1
|
||||
spec = graph.spec("pytest")
|
||||
assert "pytest" in spec.cmd
|
||||
|
||||
|
||||
def test_lint_command_graph_structure():
|
||||
"""Test 'lint' command graph has correct structure."""
|
||||
graphs = build_graphs()
|
||||
graph = graphs["lint"]
|
||||
specs = graph.all_specs()
|
||||
assert len(specs) == 1
|
||||
spec = graph.spec("ruff_check")
|
||||
assert "ruff" in spec.cmd
|
||||
|
||||
|
||||
def test_pymake_config_dirs_to_ignore():
|
||||
"""Test PymakeConfig has correct dirs to ignore."""
|
||||
assert ".venv" in conf.DIRS_TO_IGNORE
|
||||
assert ".git" in conf.DIRS_TO_IGNORE
|
||||
assert ".tox" in conf.DIRS_TO_IGNORE
|
||||
|
||||
|
||||
def test_pymake_config_python_build_dirs():
|
||||
"""Test PymakeConfig has correct Python build dirs."""
|
||||
assert "dist" in conf.PYTHON_BUILD_DIRS
|
||||
assert "build" in conf.PYTHON_BUILD_DIRS
|
||||
|
||||
|
||||
def test_maturin_build_options_win7():
|
||||
"""Test MATURIN_BUILD_OPTIONS_WIN7 has expected options."""
|
||||
assert "--target" in conf.MATURIN_BUILD_OPTIONS_WIN7
|
||||
assert "x86_64-win7-windows-msvc" in conf.MATURIN_BUILD_OPTIONS_WIN7
|
||||
assert "-Zbuild-std" in conf.MATURIN_BUILD_OPTIONS_WIN7
|
||||
|
||||
|
||||
def test_doc_build_command():
|
||||
"""Test DOC_BUILD_COMMAND has expected structure."""
|
||||
assert "sphinx-build" in conf.DOC_BUILD_COMMAND
|
||||
assert "-b" in conf.DOC_BUILD_COMMAND
|
||||
assert "html" in conf.DOC_BUILD_COMMAND
|
||||
def test_main_with_no_args_shows_help(self) -> None:
|
||||
"""main() with no args should show help and exit with failure."""
|
||||
with patch("sys.argv", ["pymake"]), pytest.raises(SystemExit) as exc_info:
|
||||
pymake.main()
|
||||
assert exc_info.value.code == 1
|
||||
|
||||
@@ -79,25 +79,23 @@ def test_builtin_conditions_python_version_at_least():
|
||||
current_major = sys.version_info.major
|
||||
current_minor = sys.version_info.minor
|
||||
# Current version should be at least itself
|
||||
assert (
|
||||
BuiltinConditions.PYTHON_VERSION_AT_LEAST(current_major, current_minor) is True
|
||||
)
|
||||
assert BuiltinConditions.PYTHON_VERSION_AT_LEAST(current_major, current_minor) is True
|
||||
# Current version should be at least an older version
|
||||
assert BuiltinConditions.PYTHON_VERSION_AT_LEAST(current_major - 1, 0) is True
|
||||
# Current version should NOT be at least a newer version
|
||||
assert BuiltinConditions.PYTHON_VERSION_AT_LEAST(current_major + 1, 0) is False
|
||||
|
||||
|
||||
def test_builtin_conditions_has_app_installed_true():
|
||||
"""Test BuiltinConditions.HAS_APP_INSTALLED when app exists."""
|
||||
def test_builtin_conditions_HAS_INSTALLED_true():
|
||||
"""Test BuiltinConditions.HAS_INSTALLED when app exists."""
|
||||
# Python should always be available
|
||||
condition = BuiltinConditions.HAS_APP_INSTALLED("python")
|
||||
condition = BuiltinConditions.HAS_INSTALLED("python")
|
||||
assert condition() is True
|
||||
|
||||
|
||||
def test_builtin_conditions_has_app_installed_false():
|
||||
"""Test BuiltinConditions.HAS_APP_INSTALLED when app doesn't exist."""
|
||||
condition = BuiltinConditions.HAS_APP_INSTALLED("nonexistent_app_12345")
|
||||
def test_builtin_conditions_HAS_INSTALLED_false():
|
||||
"""Test BuiltinConditions.HAS_INSTALLED when app doesn't exist."""
|
||||
condition = BuiltinConditions.HAS_INSTALLED("nonexistent_app_12345")
|
||||
assert condition() is False
|
||||
|
||||
|
||||
|
||||
+14
-23
@@ -76,7 +76,7 @@ def test_failure_propagates() -> None:
|
||||
]
|
||||
)
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
px.run(graph, strategy="sequential")
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
assert exc_info.value.task == "boom"
|
||||
assert isinstance(exc_info.value.cause, ValueError)
|
||||
|
||||
@@ -103,7 +103,7 @@ def test_retries_exhausted() -> None:
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("f", always_fail, retries=2)])
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
px.run(graph, strategy="sequential")
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
assert exc_info.value.attempts == 3
|
||||
|
||||
|
||||
@@ -127,8 +127,8 @@ def test_threaded_parallelism() -> None:
|
||||
report = px.run(graph, strategy="thread", max_workers=3)
|
||||
elapsed = time.time() - start
|
||||
assert report.success
|
||||
# Three 0.3s tasks in parallel should be well under 0.8s.
|
||||
assert elapsed < 0.8
|
||||
# Three 0.3s tasks in parallel should be well under 1.0s.
|
||||
assert elapsed < 1.0
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@@ -226,7 +226,7 @@ def test_async_timeout() -> None:
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("slow", slow, timeout=0.05)])
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
px.run(graph, strategy="async")
|
||||
_ = px.run(graph, strategy="async")
|
||||
assert isinstance(exc_info.value.cause, TaskTimeoutError)
|
||||
|
||||
|
||||
@@ -269,11 +269,11 @@ def test_memory_backend_resume() -> None:
|
||||
]
|
||||
)
|
||||
backend = MemoryBackend()
|
||||
px.run(graph, strategy="sequential", state=backend)
|
||||
_ = px.run(graph, strategy="sequential", state=backend)
|
||||
assert runs == ["a", "b"]
|
||||
|
||||
# Second run: both cached, neither re-executed.
|
||||
px.run(graph, strategy="sequential", state=backend)
|
||||
_ = px.run(graph, strategy="sequential", state=backend)
|
||||
assert runs == ["a", "b"] # unchanged
|
||||
|
||||
|
||||
@@ -285,7 +285,7 @@ def test_json_backend_persistence() -> None:
|
||||
return 7
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", fn)])
|
||||
px.run(graph, strategy="sequential", state=JSONBackend(path))
|
||||
_ = px.run(graph, strategy="sequential", state=JSONBackend(path))
|
||||
|
||||
# New backend reads the file; task should be skipped.
|
||||
runs: list[str] = []
|
||||
@@ -310,21 +310,12 @@ def test_on_event_callback() -> None:
|
||||
return 1
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", fn)])
|
||||
px.run(graph, strategy="sequential", on_event=events.append)
|
||||
_ = px.run(graph, strategy="sequential", on_event=events.append)
|
||||
statuses = [e.status for e in events]
|
||||
assert px.TaskStatus.SUCCESS in statuses
|
||||
assert all(e.task == "a" for e in events)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# Invalid strategy
|
||||
# ---------------------------------------------------------------------- #
|
||||
def test_invalid_strategy() -> None:
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", lambda: None)]) # type: ignore[arg-type]
|
||||
with pytest.raises(ValueError):
|
||||
px.run(graph, strategy="bogus") # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 异步策略:sync 任务无 timeout 分支 + timeout 重试分支
|
||||
# ---------------------------------------------------------------------- #
|
||||
@@ -410,10 +401,10 @@ def test_threaded_skips_cached_tasks() -> None:
|
||||
)
|
||||
backend = px.MemoryBackend()
|
||||
# 第一次运行填充缓存
|
||||
px.run(graph, strategy="thread", max_workers=2, state=backend)
|
||||
_ = px.run(graph, strategy="thread", max_workers=2, state=backend)
|
||||
assert runs == ["a", "b"]
|
||||
# 第二次运行应全部跳过
|
||||
px.run(graph, strategy="thread", max_workers=2, state=backend)
|
||||
_ = px.run(graph, strategy="thread", max_workers=2, state=backend)
|
||||
assert runs == ["a", "b"] # 未再执行
|
||||
|
||||
|
||||
@@ -454,9 +445,9 @@ def test_async_skips_cached_tasks() -> None:
|
||||
]
|
||||
)
|
||||
backend = px.MemoryBackend()
|
||||
px.run(graph, strategy="async", state=backend)
|
||||
_ = px.run(graph, strategy="async", state=backend)
|
||||
assert runs == ["a", "b"]
|
||||
px.run(graph, strategy="async", state=backend)
|
||||
_ = px.run(graph, strategy="async", state=backend)
|
||||
assert runs == ["a", "b"]
|
||||
|
||||
|
||||
@@ -483,7 +474,7 @@ def test_failure_marks_report_unsuccessful() -> None:
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", boom)])
|
||||
with pytest.raises(px.TaskFailedError):
|
||||
px.run(graph, strategy="sequential")
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
# report 在异常前未返回,但若捕获异常则 success 应为 False
|
||||
# 这里验证 run() 抛异常的行为本身
|
||||
|
||||
|
||||
@@ -54,6 +54,61 @@ def test_verbose_event_callback_running():
|
||||
assert report.success
|
||||
|
||||
|
||||
def test_verbose_run_with_success_lifecycle(capsys):
|
||||
"""Test px.run with verbose=True prints SUCCESS lifecycle."""
|
||||
spec = px.TaskSpec("test", fn=lambda: "result")
|
||||
graph = px.Graph.from_specs([spec])
|
||||
report = px.run(graph, strategy="sequential", verbose=True)
|
||||
assert report.success
|
||||
captured = capsys.readouterr()
|
||||
assert "成功" in captured.out
|
||||
|
||||
|
||||
def test_verbose_run_with_failed_lifecycle(capsys):
|
||||
"""Test px.run with verbose=True prints FAILED lifecycle with error."""
|
||||
|
||||
def raise_error():
|
||||
raise ValueError("test error")
|
||||
|
||||
spec = px.TaskSpec("test", fn=raise_error)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
|
||||
with pytest.raises(px.TaskFailedError):
|
||||
px.run(graph, strategy="sequential", verbose=True)
|
||||
captured = capsys.readouterr()
|
||||
assert "失败" in captured.out
|
||||
assert "test error" in captured.out
|
||||
|
||||
|
||||
def test_verbose_run_with_skipped_lifecycle(capsys):
|
||||
"""Test px.run with verbose=True prints SKIPPED lifecycle."""
|
||||
spec = px.TaskSpec(
|
||||
"test",
|
||||
fn=lambda: "result",
|
||||
conditions=(lambda: False,),
|
||||
)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
report = px.run(graph, strategy="sequential", verbose=True)
|
||||
assert report.success
|
||||
captured = capsys.readouterr()
|
||||
assert "跳过" in captured.out
|
||||
|
||||
|
||||
def test_verbose_run_with_user_callback():
|
||||
"""Test px.run with verbose=True and user callback both called."""
|
||||
events = []
|
||||
|
||||
def on_event(event):
|
||||
events.append(event)
|
||||
|
||||
spec = px.TaskSpec("test", fn=lambda: "result")
|
||||
graph = px.Graph.from_specs([spec])
|
||||
report = px.run(graph, strategy="sequential", verbose=True, on_event=on_event)
|
||||
assert report.success
|
||||
assert len(events) == 1
|
||||
assert events[0].status == px.TaskStatus.SUCCESS
|
||||
|
||||
|
||||
def test_verbose_event_callback_success():
|
||||
"""Test verbose event callback for SUCCESS status."""
|
||||
# Create a graph with verbose callback
|
||||
|
||||
+10
-13
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.task import TaskResult, TaskSpec, TaskStatus
|
||||
@@ -15,17 +16,17 @@ def _fn() -> int:
|
||||
def _make_result(
|
||||
name: str = "a",
|
||||
status: TaskStatus = TaskStatus.SUCCESS,
|
||||
value: object = 42,
|
||||
value: Any = 42,
|
||||
error: BaseException | None = None,
|
||||
duration: float = 0.5,
|
||||
attempts: int = 1,
|
||||
) -> TaskResult[object]:
|
||||
) -> TaskResult[Any]:
|
||||
"""构造测试用 TaskResult 实例."""
|
||||
spec: TaskSpec[object] = TaskSpec[object](name, _fn)
|
||||
spec: TaskSpec[Any] = TaskSpec[Any](name, _fn)
|
||||
start = datetime(2024, 1, 1, 0, 0, 0)
|
||||
# 用 timedelta 精确表达秒数,避免 int() 截断小数
|
||||
end = start + timedelta(seconds=duration) if duration else None
|
||||
return TaskResult[object](
|
||||
return TaskResult[Any](
|
||||
spec=spec,
|
||||
status=status,
|
||||
value=value,
|
||||
@@ -85,7 +86,7 @@ class TestRunReportSummary:
|
||||
def test_summary_with_none_duration(self) -> None:
|
||||
"""未开始/未结束的任务 duration 为 None,不应计入总时长."""
|
||||
report = px.RunReport()
|
||||
spec: TaskSpec[object] = TaskSpec("a", _fn) # type: ignore[arg-type]
|
||||
spec: TaskSpec[Any] = TaskSpec[Any]("a", _fn) # type: ignore[arg-type]
|
||||
report.results["a"] = TaskResult(spec=spec, status=TaskStatus.FAILED)
|
||||
s = report.summary()
|
||||
assert s["total_duration_seconds"] == 0.0
|
||||
@@ -94,9 +95,7 @@ class TestRunReportSummary:
|
||||
"""failed_tasks 应返回所有失败任务名."""
|
||||
report = px.RunReport()
|
||||
report.results["a"] = _make_result("a", status=TaskStatus.SUCCESS)
|
||||
report.results["b"] = _make_result(
|
||||
"b", status=TaskStatus.FAILED, error=ValueError("x")
|
||||
)
|
||||
report.results["b"] = _make_result("b", status=TaskStatus.FAILED, error=ValueError("x"))
|
||||
assert report.failed_tasks() == ["b"]
|
||||
|
||||
|
||||
@@ -115,9 +114,7 @@ class TestRunReportDescribe:
|
||||
def test_describe_with_error(self) -> None:
|
||||
"""应正确描述失败状态和错误信息."""
|
||||
report = px.RunReport(success=False)
|
||||
report.results["a"] = _make_result(
|
||||
"a", status=TaskStatus.FAILED, error=ValueError("boom"), duration=0.1
|
||||
)
|
||||
report.results["a"] = _make_result("a", status=TaskStatus.FAILED, error=ValueError("boom"), duration=0.1)
|
||||
desc = report.describe()
|
||||
assert "success=False" in desc
|
||||
assert "error=ValueError" in desc
|
||||
@@ -125,7 +122,7 @@ class TestRunReportDescribe:
|
||||
def test_describe_no_duration(self) -> None:
|
||||
"""无耗时的任务应显示为 '-'."""
|
||||
report = px.RunReport()
|
||||
spec: TaskSpec[object] = TaskSpec("a", _fn) # type: ignore[arg-type]
|
||||
report.results["a"] = TaskResult(spec=spec, status=TaskStatus.PENDING)
|
||||
spec: TaskSpec[Any] = TaskSpec[Any]("a", _fn) # type: ignore[arg-type]
|
||||
report.results["a"] = TaskResult[Any](spec=spec, status=TaskStatus.PENDING)
|
||||
desc = report.describe()
|
||||
assert "-" in desc # duration 显示为 "-"
|
||||
|
||||
+66
-67
@@ -76,11 +76,6 @@ class TestCliRunnerConstruction:
|
||||
)
|
||||
assert runner.commands == ["clean", "build", "test"]
|
||||
|
||||
def test_rejects_non_graph_list(self) -> None:
|
||||
"""列表类型的值应抛出 TypeError."""
|
||||
with pytest.raises(TypeError, match="必须是 Graph 实例"):
|
||||
_ = px.CliRunner(graphs={"build": [1, 2, 3]}) # type: ignore[arg-type] # pyright: ignore[reportArgumentType]
|
||||
|
||||
def test_default_strategy_is_sequential(self) -> None:
|
||||
"""默认策略应为 Strategy.SEQUENTIAL."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
@@ -103,8 +98,7 @@ class TestCliRunnerConstruction:
|
||||
|
||||
def test_custom_verbose_false(self) -> None:
|
||||
"""应支持关闭 verbose."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
runner.verbose = False
|
||||
runner = px.CliRunner({"clean": _echo_graph()}, verbose=False)
|
||||
assert runner.verbose is False
|
||||
|
||||
def test_default_description_is_empty(self) -> None:
|
||||
@@ -173,17 +167,10 @@ class TestCliRunnerParser:
|
||||
|
||||
def test_parser_strategy_default(self) -> None:
|
||||
"""--strategy 默认值应与构造时一致."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()}, "async")
|
||||
runner = px.CliRunner({"clean": _echo_graph()}, strategy="async")
|
||||
parser = runner.create_parser()
|
||||
parsed = parser.parse_args(["clean"])
|
||||
assert parsed.strategy == "sequential"
|
||||
|
||||
def test_parser_strategy_invalid_choice(self) -> None:
|
||||
"""--strategy 不接受非法值."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()}, "invalid") # pyright: ignore[reportArgumentType]
|
||||
parser = runner.create_parser()
|
||||
with pytest.raises(SystemExit):
|
||||
_ = parser.parse_args(["clean", "--strategy", "invalid"])
|
||||
assert parsed.strategy == "async"
|
||||
|
||||
def test_parser_has_dry_run_flag(self) -> None:
|
||||
"""解析器应有 --dry-run 标志."""
|
||||
@@ -289,9 +276,7 @@ class TestCliRunnerRunSuccess:
|
||||
class TestCliRunnerVerbose:
|
||||
"""测试 verbose 模式."""
|
||||
|
||||
def test_verbose_default_prints_lifecycle(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
def test_verbose_default_prints_lifecycle(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""默认 verbose=True 应打印任务生命周期."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
_ = runner.run(["echo"])
|
||||
@@ -299,9 +284,7 @@ class TestCliRunnerVerbose:
|
||||
# verbose 模式下应打印任务生命周期
|
||||
assert "[verbose]" in captured.out
|
||||
|
||||
def test_quiet_flag_disables_verbose(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
def test_quiet_flag_disables_verbose(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""--quiet 应关闭 verbose 输出."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
_ = runner.run(["echo", "--quiet"])
|
||||
@@ -309,18 +292,14 @@ class TestCliRunnerVerbose:
|
||||
# quiet 模式下不应有 [verbose] 前缀的输出
|
||||
assert "[verbose]" not in captured.out
|
||||
|
||||
def test_verbose_false_constructor_disables_verbose(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
def test_verbose_false_constructor_disables_verbose(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""构造时 verbose=False 应关闭 verbose 输出."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()}, verbose=False)
|
||||
_ = runner.run(["echo"])
|
||||
captured = capsys.readouterr()
|
||||
assert "[verbose]" not in captured.out
|
||||
|
||||
def test_verbose_prints_command_for_cmd_task(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
def test_verbose_prints_command_for_cmd_task(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose 模式下 cmd 任务应打印执行的命令."""
|
||||
runner = px.CliRunner({"echo": _echo_graph(msg="verbose-test")})
|
||||
_ = runner.run(["echo"])
|
||||
@@ -330,18 +309,14 @@ class TestCliRunnerVerbose:
|
||||
# 应打印返回码
|
||||
assert "返回码" in captured.out
|
||||
|
||||
def test_verbose_prints_success_lifecycle(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
def test_verbose_prints_success_lifecycle(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose 模式下成功任务应打印成功信息."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
_ = runner.run(["echo"])
|
||||
captured = capsys.readouterr()
|
||||
assert "成功" in captured.out
|
||||
|
||||
def test_verbose_prints_skip_lifecycle(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
def test_verbose_prints_skip_lifecycle(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose 模式下跳过的任务应打印跳过信息."""
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
@@ -357,9 +332,7 @@ class TestCliRunnerVerbose:
|
||||
captured = capsys.readouterr()
|
||||
assert "跳过" in captured.out
|
||||
|
||||
def test_verbose_prints_failure_lifecycle(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
def test_verbose_prints_failure_lifecycle(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose 模式下失败任务应打印失败信息."""
|
||||
runner = px.CliRunner({"fail": _failing_graph()})
|
||||
_ = runner.run(["fail"])
|
||||
@@ -375,9 +348,7 @@ class TestCliRunnerVerbose:
|
||||
class TestCliRunnerRunFailure:
|
||||
"""测试 CliRunner.run 的失败执行路径."""
|
||||
|
||||
def test_run_unknown_command_returns_failure(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
def test_run_unknown_command_returns_failure(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""未知命令应返回 1 并打印错误."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
exit_code = runner.run(["unknown"])
|
||||
@@ -386,9 +357,7 @@ class TestCliRunnerRunFailure:
|
||||
assert "未知命令" in captured.err
|
||||
assert "clean" in captured.err
|
||||
|
||||
def test_run_no_command_returns_failure(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
def test_run_no_command_returns_failure(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""无命令时应返回 1 并打印帮助."""
|
||||
runner = px.CliRunner({"clean": _echo_graph()})
|
||||
exit_code = runner.run([])
|
||||
@@ -402,9 +371,7 @@ class TestCliRunnerRunFailure:
|
||||
exit_code = runner.run(["fail"])
|
||||
assert exit_code == CliExitCode.FAILURE.value
|
||||
|
||||
def test_run_failing_task_prints_error(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
def test_run_failing_task_prints_error(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""任务失败时应打印错误信息."""
|
||||
runner = px.CliRunner({"fail": _failing_graph()})
|
||||
_ = runner.run(["fail"])
|
||||
@@ -458,9 +425,7 @@ class TestCliRunnerList:
|
||||
class TestCliRunnerErrorHandling:
|
||||
"""测试错误处理."""
|
||||
|
||||
def test_keyboard_interrupt_returns_130(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
def test_keyboard_interrupt_returns_130(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""KeyboardInterrupt 应返回 130."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
|
||||
@@ -473,9 +438,7 @@ class TestCliRunnerErrorHandling:
|
||||
captured = capsys.readouterr()
|
||||
assert "取消" in captured.err
|
||||
|
||||
def test_pyflowx_error_returns_failure(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
def test_pyflowx_error_returns_failure(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""PyFlowXError 应返回 1."""
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
|
||||
@@ -499,9 +462,7 @@ class TestCliRunnerErrorHandling:
|
||||
def raise_custom(*_args: Any, **_kwargs: Any) -> None:
|
||||
raise CustomError("unexpected")
|
||||
|
||||
with patch("pyflowx.runner.run", side_effect=raise_custom), pytest.raises(
|
||||
CustomError
|
||||
):
|
||||
with patch("pyflowx.runner.run", side_effect=raise_custom), pytest.raises(CustomError):
|
||||
_ = runner.run(["echo"])
|
||||
|
||||
|
||||
@@ -525,9 +486,7 @@ class TestCliRunnerRunCli:
|
||||
runner.run_cli(["fail"])
|
||||
assert exc_info.value.code == CliExitCode.FAILURE.value
|
||||
|
||||
def test_run_cli_no_args_uses_sys_argv(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
def test_run_cli_no_args_uses_sys_argv(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""run_cli 无参数时应使用 sys.argv."""
|
||||
monkeypatch.setattr(sys, "argv", ["pymake", "echo"])
|
||||
runner = px.CliRunner({"echo": _echo_graph()})
|
||||
@@ -620,12 +579,8 @@ class TestCliRunnerIntegration:
|
||||
"""混合 fn 和 cmd 的命令应都能执行."""
|
||||
runner = px.CliRunner(
|
||||
{
|
||||
"fn_cmd": px.Graph.from_specs(
|
||||
[px.TaskSpec("fn", fn=lambda: "fn-result")]
|
||||
),
|
||||
"cmd_cmd": px.Graph.from_specs(
|
||||
[px.TaskSpec("cmd", cmd=[*ECHO_CMD, "cmd-result"])]
|
||||
),
|
||||
"fn_cmd": px.Graph.from_specs([px.TaskSpec("fn", fn=lambda: "fn-result")]),
|
||||
"cmd_cmd": px.Graph.from_specs([px.TaskSpec("cmd", cmd=[*ECHO_CMD, "cmd-result"])]),
|
||||
}
|
||||
)
|
||||
assert runner.run(["fn_cmd"]) == CliExitCode.SUCCESS.value
|
||||
@@ -642,9 +597,53 @@ class TestCliRunnerIntegration:
|
||||
else:
|
||||
ls_cmd = ["ls"]
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("ls", cmd=ls_cmd, cwd=Path(tmpdir))]
|
||||
)
|
||||
graph = px.Graph.from_specs([px.TaskSpec("ls", cmd=ls_cmd, cwd=Path(tmpdir))])
|
||||
runner = px.CliRunner({"ls": graph})
|
||||
exit_code = runner.run(["ls"])
|
||||
assert exit_code == CliExitCode.SUCCESS.value
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 构造校验 (补充覆盖)
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestCliRunnerConstructionValidation:
|
||||
"""测试 CliRunner 的构造校验 (补充覆盖)."""
|
||||
|
||||
def test_non_graph_value_raises_type_error(self) -> None:
|
||||
"""非 Graph 值应抛出 TypeError (覆盖 runner.py line 119)."""
|
||||
with pytest.raises(TypeError, match="必须是 Graph 实例"):
|
||||
_ = px.CliRunner(graphs={"bad": "not a graph"}) # type: ignore[dict-item]
|
||||
|
||||
def test_non_graph_value_dict_raises_type_error(self) -> None:
|
||||
"""dict 中包含非 Graph 值应抛出 TypeError."""
|
||||
with pytest.raises(TypeError, match="必须是 Graph 实例"):
|
||||
_ = px.CliRunner(graphs={"good": _echo_graph(), "bad": 123}) # type: ignore[dict-item]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# _apply_verbose_to_graph (补充覆盖)
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestApplyVerboseToGraph:
|
||||
"""测试 _apply_verbose_to_graph 函数 (补充覆盖)."""
|
||||
|
||||
def test_specs_with_matching_verbose_are_kept(self) -> None:
|
||||
"""spec.verbose 已与目标值匹配时应保留原 spec (覆盖 runner.py line 57)."""
|
||||
from pyflowx.runner import _apply_verbose_to_graph
|
||||
|
||||
# 创建 verbose=True 的 spec
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", cmd=[*ECHO_CMD, "a"], verbose=True)])
|
||||
# 应用 verbose=True, spec.verbose 已匹配, 应保留原 spec
|
||||
new_graph = _apply_verbose_to_graph(graph, verbose=True)
|
||||
new_spec = new_graph.spec("a")
|
||||
assert new_spec.verbose is True
|
||||
|
||||
def test_specs_with_non_matching_verbose_are_replaced(self) -> None:
|
||||
"""spec.verbose 与目标值不匹配时应替换 (覆盖 else 分支)."""
|
||||
from pyflowx.runner import _apply_verbose_to_graph
|
||||
|
||||
# 创建 verbose=False 的 spec
|
||||
graph = px.Graph.from_specs([px.TaskSpec("a", cmd=[*ECHO_CMD, "a"], verbose=False)])
|
||||
# 应用 verbose=True, spec.verbose 不匹配, 应替换
|
||||
new_graph = _apply_verbose_to_graph(graph, verbose=True)
|
||||
new_spec = new_graph.spec("a")
|
||||
assert new_spec.verbose is True
|
||||
|
||||
+169
-16
@@ -1,7 +1,10 @@
|
||||
"""Tests for task module edge cases."""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -20,7 +23,6 @@ def test_taskspec_wrap_cmd_with_list():
|
||||
spec = TaskSpec("test", cmd=[*ECHO_CMD, "hello"])
|
||||
wrapped_fn = spec.effective_fn
|
||||
assert wrapped_fn is not None
|
||||
assert wrapped_fn.__name__ == "test"
|
||||
|
||||
|
||||
def test_taskspec_wrap_cmd_with_string():
|
||||
@@ -32,7 +34,6 @@ def test_taskspec_wrap_cmd_with_string():
|
||||
spec = TaskSpec("test", cmd=cmd_str)
|
||||
wrapped_fn = spec.effective_fn
|
||||
assert wrapped_fn is not None
|
||||
assert wrapped_fn.__name__ == "test"
|
||||
|
||||
|
||||
def test_taskspec_wrap_cmd_with_timeout():
|
||||
@@ -48,7 +49,7 @@ def test_taskspec_wrap_cmd_with_timeout():
|
||||
def test_taskspec_wrap_cmd_with_cwd():
|
||||
"""Test TaskSpec._wrap_cmd with working directory."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
spec = TaskSpec("test", cmd=[*ECHO_CMD, "hello"], cwd=tmpdir)
|
||||
spec = TaskSpec("test", cmd=[*ECHO_CMD, "hello"], cwd=Path(tmpdir))
|
||||
wrapped_fn = spec.effective_fn
|
||||
result = wrapped_fn()
|
||||
assert result is None
|
||||
@@ -99,19 +100,6 @@ def test_taskspec_no_fn_no_cmd():
|
||||
_ = TaskSpec("test")
|
||||
|
||||
|
||||
def test_taskspec_cmd_overrides_fn():
|
||||
"""Test TaskSpec cmd overrides fn."""
|
||||
|
||||
def my_fn():
|
||||
return "fn_result"
|
||||
|
||||
spec = TaskSpec("test", fn=my_fn, cmd=[*ECHO_CMD, "hello"])
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
# cmd should override fn
|
||||
assert wrapped_fn.__name__ == "test"
|
||||
|
||||
|
||||
def test_taskspec_conditions_check():
|
||||
"""Test TaskSpec.should_execute with conditions."""
|
||||
spec = px.TaskSpec(
|
||||
@@ -154,3 +142,168 @@ def test_taskspec_conditions_multiple_one_false():
|
||||
)
|
||||
|
||||
assert spec.should_execute() is False
|
||||
|
||||
|
||||
def test_taskspec_list_cmd_timeout_mocked():
|
||||
"""Test TaskSpec._wrap_cmd handles list command timeout (mocked)."""
|
||||
spec = TaskSpec("test", cmd=["sleep", "10"], timeout=0.1)
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
with patch(
|
||||
"subprocess.run", side_effect=subprocess.TimeoutExpired(cmd=["sleep", "10"], timeout=0.1)
|
||||
), pytest.raises(RuntimeError, match="命令执行超时"):
|
||||
_ = wrapped_fn()
|
||||
|
||||
|
||||
def test_taskspec_shell_cmd_timeout_mocked():
|
||||
"""Test TaskSpec._wrap_cmd handles shell command timeout (mocked)."""
|
||||
spec = TaskSpec("test", cmd="sleep 10", timeout=0.1)
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
with patch("subprocess.run", side_effect=subprocess.TimeoutExpired(cmd="sleep 10", timeout=0.1)), pytest.raises(
|
||||
RuntimeError, match="Shell 命令执行超时"
|
||||
):
|
||||
_ = wrapped_fn()
|
||||
|
||||
|
||||
def test_taskspec_shell_cmd_file_not_found_mocked():
|
||||
"""Test TaskSpec._wrap_cmd handles shell command FileNotFoundError (mocked)."""
|
||||
spec = TaskSpec("test", cmd="nonexistent_shell_command")
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
with patch("subprocess.run", side_effect=FileNotFoundError("not found")), pytest.raises(
|
||||
RuntimeError, match="Shell 命令未找到"
|
||||
):
|
||||
_ = wrapped_fn()
|
||||
|
||||
|
||||
def test_taskspec_shell_cmd_with_cwd_verbose(capsys):
|
||||
"""Test TaskSpec._wrap_cmd with shell command, cwd and verbose=True."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
if sys.platform == "win32":
|
||||
shell_cmd = "cmd /c echo hello"
|
||||
else:
|
||||
shell_cmd = "echo hello"
|
||||
spec = TaskSpec("test", cmd=shell_cmd, cwd=Path(tmpdir), verbose=True)
|
||||
wrapped_fn = spec.effective_fn
|
||||
result = wrapped_fn()
|
||||
assert result is None
|
||||
captured = capsys.readouterr()
|
||||
assert "执行 Shell" in captured.out
|
||||
assert "工作目录" in captured.out
|
||||
|
||||
|
||||
def test_taskspec_list_cmd_os_error_mocked():
|
||||
"""Test TaskSpec._wrap_cmd handles list command OSError (mocked)."""
|
||||
spec = TaskSpec("test", cmd=["ls"])
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
with patch("subprocess.run", side_effect=OSError("os error")), pytest.raises(RuntimeError, match="命令执行异常"):
|
||||
_ = wrapped_fn()
|
||||
|
||||
|
||||
def test_taskspec_shell_cmd_os_error_mocked():
|
||||
"""Test TaskSpec._wrap_cmd handles shell command OSError (mocked)."""
|
||||
spec = TaskSpec("test", cmd="ls")
|
||||
wrapped_fn = spec.effective_fn
|
||||
|
||||
with patch("subprocess.run", side_effect=OSError("os error")), pytest.raises(
|
||||
RuntimeError, match="Shell 命令执行异常"
|
||||
):
|
||||
_ = wrapped_fn()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# skip_if_missing
|
||||
# ---------------------------------------------------------------------- #
|
||||
def test_skip_if_missing_with_available_command():
|
||||
"""skip_if_missing=True 时,命令存在应返回 True."""
|
||||
# python 命令在测试环境中一定存在
|
||||
spec = TaskSpec("test", cmd=["python", "--version"], skip_if_missing=True)
|
||||
assert spec.should_execute() is True
|
||||
|
||||
|
||||
def test_skip_if_missing_with_missing_command():
|
||||
"""skip_if_missing=True 时,命令不存在应返回 False."""
|
||||
spec = TaskSpec("test", cmd=["definitely_not_installed_app_xyz"], skip_if_missing=True)
|
||||
assert spec.should_execute() is False
|
||||
|
||||
|
||||
def test_skip_if_missing_false_with_missing_command():
|
||||
"""skip_if_missing=False 时,命令不存在也应返回 True(不检查)."""
|
||||
spec = TaskSpec("test", cmd=["definitely_not_installed_app_xyz"], skip_if_missing=False)
|
||||
assert spec.should_execute() is True
|
||||
|
||||
|
||||
def test_skip_if_missing_with_shell_cmd_not_checked():
|
||||
"""skip_if_missing=True 时,shell 命令(str)不检查,应返回 True."""
|
||||
spec = TaskSpec("test", cmd="definitely_not_installed_app_xyz", skip_if_missing=True)
|
||||
assert spec.should_execute() is True
|
||||
|
||||
|
||||
def test_skip_if_missing_with_callable_cmd_not_checked():
|
||||
"""skip_if_missing=True 时,Callable 命令不检查,应返回 True."""
|
||||
|
||||
def custom_cmd() -> int:
|
||||
return 0
|
||||
|
||||
spec = TaskSpec("test", cmd=custom_cmd, skip_if_missing=True)
|
||||
assert spec.should_execute() is True
|
||||
|
||||
|
||||
def test_skip_if_missing_with_fn_not_checked():
|
||||
"""skip_if_missing=True 时,fn 任务不检查命令,应返回 True."""
|
||||
|
||||
def my_fn() -> int:
|
||||
return 0
|
||||
|
||||
spec = TaskSpec("test", fn=my_fn, skip_if_missing=True)
|
||||
assert spec.should_execute() is True
|
||||
|
||||
|
||||
def test_skip_if_missing_with_empty_cmd_list():
|
||||
"""skip_if_missing=True 时,空命令列表应返回 True(不检查)."""
|
||||
spec = TaskSpec("test", cmd=[""], skip_if_missing=True)
|
||||
# 空字符串命令,shutil.which 返回 None
|
||||
# 但 cmd[0] 是空字符串,shutil.which("") 返回 None
|
||||
assert spec.should_execute() is False
|
||||
|
||||
|
||||
def test_skip_if_missing_combined_with_conditions():
|
||||
"""skip_if_missing=True 与 conditions 组合使用."""
|
||||
# conditions 返回 False,应跳过
|
||||
spec = TaskSpec(
|
||||
"test",
|
||||
cmd=["python", "--version"],
|
||||
skip_if_missing=True,
|
||||
conditions=(lambda: False,),
|
||||
)
|
||||
assert spec.should_execute() is False
|
||||
|
||||
# conditions 返回 True,命令存在,应执行
|
||||
spec = TaskSpec(
|
||||
"test",
|
||||
cmd=["python", "--version"],
|
||||
skip_if_missing=True,
|
||||
conditions=(lambda: True,),
|
||||
)
|
||||
assert spec.should_execute() is True
|
||||
|
||||
# conditions 返回 True,命令不存在,应跳过
|
||||
spec = TaskSpec(
|
||||
"test",
|
||||
cmd=["definitely_not_installed_app_xyz"],
|
||||
skip_if_missing=True,
|
||||
conditions=(lambda: True,),
|
||||
)
|
||||
assert spec.should_execute() is False
|
||||
|
||||
|
||||
def test_skip_if_missing_skips_task_in_run():
|
||||
"""skip_if_missing=True 时,命令不存在的任务在 run 中应被跳过."""
|
||||
spec = TaskSpec("missing_cmd", cmd=["definitely_not_installed_app_xyz"], skip_if_missing=True)
|
||||
graph = px.Graph.from_specs([spec])
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success is True
|
||||
result = report.result_of("missing_cmd")
|
||||
assert result.status == px.TaskStatus.SKIPPED
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -159,7 +160,7 @@ def test_app_installed_conditions():
|
||||
px.TaskSpec(
|
||||
"python_check",
|
||||
cmd=python_cmd,
|
||||
conditions=(BuiltinConditions.HAS_APP_INSTALLED("python"),),
|
||||
conditions=(BuiltinConditions.HAS_INSTALLED("python"),),
|
||||
),
|
||||
]
|
||||
)
|
||||
@@ -357,27 +358,21 @@ class TestTaskSpecVerbose:
|
||||
|
||||
def test_verbose_default_is_false(self) -> None:
|
||||
"""verbose 默认应为 False."""
|
||||
spec: px.TaskSpec[object] = px.TaskSpec("a", cmd=[*ECHO_CMD, "hi"])
|
||||
spec: px.TaskSpec[Any] = px.TaskSpec[Any]("a", cmd=[*ECHO_CMD, "hi"])
|
||||
assert spec.verbose is False
|
||||
|
||||
def test_verbose_true_prints_command(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
def test_verbose_true_prints_command(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose=True 时应打印执行的命令."""
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("echo", cmd=[*ECHO_CMD, "verbose-output"], verbose=True)]
|
||||
)
|
||||
px.run(graph, strategy="sequential")
|
||||
graph = px.Graph.from_specs([px.TaskSpec("echo", cmd=[*ECHO_CMD, "verbose-output"], verbose=True)])
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
captured = capsys.readouterr()
|
||||
assert "执行命令" in captured.out
|
||||
assert "返回码" in captured.out
|
||||
|
||||
def test_verbose_false_silent(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose=False 时不应打印命令信息."""
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("echo", cmd=[*ECHO_CMD, "silent"], verbose=False)]
|
||||
)
|
||||
px.run(graph, strategy="sequential")
|
||||
graph = px.Graph.from_specs([px.TaskSpec[Any]("echo", cmd=[*ECHO_CMD, "silent"], verbose=False)])
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
captured = capsys.readouterr()
|
||||
assert "执行命令" not in captured.out
|
||||
assert "返回码" not in captured.out
|
||||
@@ -390,7 +385,7 @@ class TestTaskSpecVerbose:
|
||||
shell_cmd = "echo 'shell-verbose'"
|
||||
|
||||
graph = px.Graph.from_specs([px.TaskSpec("shell", cmd=shell_cmd, verbose=True)])
|
||||
px.run(graph, strategy="sequential")
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
captured = capsys.readouterr()
|
||||
assert "执行 Shell" in captured.out
|
||||
|
||||
@@ -399,16 +394,12 @@ class TestTaskSpecVerbose:
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("ls", cmd=ECHO_CMD, cwd=Path(tmpdir), verbose=True)]
|
||||
)
|
||||
px.run(graph, strategy="sequential")
|
||||
graph = px.Graph.from_specs([px.TaskSpec[Any]("ls", cmd=ECHO_CMD, cwd=Path(tmpdir), verbose=True)])
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
captured = capsys.readouterr()
|
||||
assert "工作目录" in captured.out
|
||||
|
||||
def test_verbose_failure_includes_returncode(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
def test_verbose_failure_includes_returncode(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""verbose=True 时失败也应打印返回码."""
|
||||
from pyflowx.errors import TaskFailedError
|
||||
|
||||
@@ -422,7 +413,7 @@ class TestTaskSpecVerbose:
|
||||
]
|
||||
)
|
||||
with pytest.raises(TaskFailedError):
|
||||
px.run(graph, strategy="sequential")
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
captured = capsys.readouterr()
|
||||
assert "返回码" in captured.out
|
||||
|
||||
@@ -438,15 +429,12 @@ class TestTaskSpecCmdErrors:
|
||||
from pyflowx.errors import TaskFailedError
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("missing", cmd=["this-command-does-not-exist-xyz"])]
|
||||
[px.TaskSpec("missing", cmd=["this-command-does-not-exist-xyz"], skip_if_missing=False)],
|
||||
)
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
px.run(graph, strategy="sequential")
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
# 错误信息应包含命令未找到
|
||||
assert (
|
||||
"命令未找到" in str(exc_info.value.cause)
|
||||
or "not found" in str(exc_info.value.cause).lower()
|
||||
)
|
||||
assert "命令未找到" in str(exc_info.value.cause) or "not found" in str(exc_info.value.cause).lower()
|
||||
|
||||
def test_cmd_list_failure_includes_stderr(self) -> None:
|
||||
"""命令失败时错误信息应包含 stderr."""
|
||||
@@ -465,7 +453,7 @@ class TestTaskSpecCmdErrors:
|
||||
]
|
||||
)
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
px.run(graph, strategy="sequential")
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
# 非 verbose 模式下, stderr 应包含在错误信息中
|
||||
assert "error-msg" in str(exc_info.value.cause)
|
||||
|
||||
@@ -473,19 +461,15 @@ class TestTaskSpecCmdErrors:
|
||||
"""shell 命令不存在时应抛出 RuntimeError."""
|
||||
from pyflowx.errors import TaskFailedError
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("missing", cmd="this-command-does-not-exist-xyz-123")]
|
||||
)
|
||||
graph = px.Graph.from_specs([px.TaskSpec("missing", cmd="this-command-does-not-exist-xyz-123")])
|
||||
with pytest.raises(TaskFailedError):
|
||||
px.run(graph, strategy="sequential")
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
|
||||
def test_cmd_string_failure(self) -> None:
|
||||
"""shell 命令失败时应抛出 RuntimeError."""
|
||||
from pyflowx.errors import TaskFailedError
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("fail", cmd='python -c "import sys; sys.exit(1)"')]
|
||||
)
|
||||
graph = px.Graph.from_specs([px.TaskSpec("fail", cmd='python -c "import sys; sys.exit(1)"')])
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
assert "Shell 命令执行失败" in str(exc_info.value.cause)
|
||||
@@ -513,32 +497,12 @@ class TestTaskSpecCmdErrors:
|
||||
"""shell 命令超时应抛出 RuntimeError."""
|
||||
from pyflowx.errors import TaskFailedError
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"slow", cmd='python -c "import time; time.sleep(5)"', timeout=0.1
|
||||
)
|
||||
]
|
||||
)
|
||||
graph = px.Graph.from_specs([px.TaskSpec("slow", cmd='python -c "import time; time.sleep(5)"', timeout=0.1)])
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
assert "超时" in str(exc_info.value.cause)
|
||||
|
||||
def test_unsupported_cmd_type_raises(self) -> None:
|
||||
"""不支持的 cmd 类型应在执行时抛出 TypeError."""
|
||||
from pyflowx.errors import TaskFailedError
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("bad", cmd=123)] # type: ignore[arg-type]
|
||||
)
|
||||
with pytest.raises((TypeError, TaskFailedError)):
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
|
||||
def test_no_fn_no_cmd_raises(self) -> None:
|
||||
"""没有 fn 和 cmd 时应抛出 ValueError."""
|
||||
with pytest.raises(ValueError, match="必须提供 fn 或 cmd"):
|
||||
px.TaskSpec("empty")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
_ = px.TaskSpec("empty")
|
||||
|
||||
Reference in New Issue
Block a user