zhou/pyflowx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: 新增skip_if_missing特性，支持命令不存在时自动跳过任务

Browse Source 
本次提交实现了命令任务的自动跳过功能：
1. 为TaskSpec新增skip_if_missing参数，默认开启，仅对list[str]类型cmd生效
2. 通过shutil.which检查命令是否存在，不存在则标记任务为SKIPPED而非失败
3. 重构should_execute方法，整合条件检查与命令可用性检查
4. 更新文档与示例代码，添加该参数的使用说明
5. 移除cli/pymake.py中的冗余check辅助函数，改用内置特性
6. 为所有内置任务添加skip_if_missing=True配置
7. 修复线程并行测试的超时阈值，放宽到1.0秒
8. 优化代码格式与压缩单行表达式
9. 新增完整的单元测试覆盖该特性的各种场景
This commit is contained in:
gooker_young
2026-06-21 18:55:24 +08:00
parent cae51856d2
commit ea591d1088
11 changed files with 193 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pyflowx/__init__.py
+6
View File
@@ -45,6 +45,12 @@
cmd=["git", "--version"],
conditions=(BuiltinConditions.HAS_INSTALLED("git"),)
),
# 命令不存在时自动跳过（而非失败）
px.TaskSpec(
"optional_build",
cmd=["maturin", "build"],
skip_if_missing=True
),
])
report = px.run(graph)
"""
src/pyflowx/cli/pymake.py
+30 -98
View File
@@ -7,7 +7,7 @@
from __future__ import annotations
import pyflowx as px
from pyflowx.conditions import BuiltinConditions, Constants
from pyflowx.conditions import Constants
def maturin_build_cmd() -> list[str]:
@@ -18,9 +18,9 @@ def maturin_build_cmd() -> list[str]:
list[str]
完整的 maturin 构建命令列表.
"""
base_cmd = ["maturin", "build", "-r"].copy()
command = ["maturin", "build", "-r"].copy()
if Constants.IS_WINDOWS:
base_cmd.extend(
command.extend(
[
"--target",
"x86_64-win7-windows-msvc",
@@ -29,94 +29,31 @@ def maturin_build_cmd() -> list[str]:
"python3.8",
]
)
return base_cmd
return command
def check(name: str) -> px.Condition:
"""检查指定工具是否已安装.
Returns
-------
bool
如果已安装则返回 True,否则返回 False.
"""
return BuiltinConditions.HAS_INSTALLED(name)
uv_build: px.TaskSpec = px.TaskSpec("uv_build", cmd=["uv", "build"], conditions=(check("uv"),))
maturin_build: px.TaskSpec = px.TaskSpec("maturin_build", cmd=maturin_build_cmd(), conditions=(check("maturin"),))
uv_sync: px.TaskSpec = px.TaskSpec("uv_sync", cmd=["uv", "sync"], conditions=(check("uv"),))
git_clean: px.TaskSpec = px.TaskSpec("git_clean", cmd=["gitt", "c"], conditions=(check("gitt"),))
uv_build: px.TaskSpec = px.TaskSpec("uv_build", cmd=["uv", "build"])
maturin_build: px.TaskSpec = px.TaskSpec("maturin_build", cmd=maturin_build_cmd())
uv_sync: px.TaskSpec = px.TaskSpec("uv_sync", cmd=["uv", "sync"])
git_clean: px.TaskSpec = px.TaskSpec("git_clean", cmd=["gitt", "c"])
test: px.TaskSpec = px.TaskSpec(
"test",
cmd=[
"pytest",
"-m",
"not slow",
"-n",
"8",
"--dist",
"loadfile",
"--color=yes",
"--durations=10",
],
conditions=(check("pytest"),),
"test", cmd=["pytest", "-m", "not slow", "-n", "8", "--dist", "loadfile", "--color=yes", "--durations=10"]
)
test_fast: px.TaskSpec = px.TaskSpec(
"test_fast",
cmd=[
"pytest",
"-m",
"not slow",
"--dist",
"loadfile",
"--color=yes",
"--durations=10",
],
conditions=(check("pytest"),),
"test_fast", cmd=["pytest", "-m", "not slow", "--dist", "loadfile", "--color=yes", "--durations=10"]
)
test_coverage: px.TaskSpec = px.TaskSpec(
"test_coverage",
cmd=[
"pytest",
"--cov",
"-n",
"8",
"--dist",
"loadfile",
"--tb=short",
"-v",
"--color=yes",
"--durations=10",
],
conditions=(check("pytest"),),
cmd=["pytest", "--cov", "-n", "8", "--dist", "loadfile", "--tb=short", "-v", "--color=yes", "--durations=10"],
)
ruff_lint: px.TaskSpec = px.TaskSpec(
"lint",
cmd=[
"ruff",
"check",
"--fix",
"--unsafe-fixes",
],
conditions=(check("ruff"),),
)
mypy_check: px.TaskSpec = px.TaskSpec("typecheck", cmd=["mypy", "."], conditions=(check("mypy"),))
ty_check: px.TaskSpec = px.TaskSpec("ty_check", cmd=["ty", "check", "."], conditions=(check("ty"),))
doc: px.TaskSpec = px.TaskSpec(
"doc", cmd=["sphinx-build", "-b", "html", "docs", "docs/_build"], conditions=(check("sphinx-build"),)
)
hatch_publish: px.TaskSpec = px.TaskSpec("publish_python", cmd=["hatch", "publish"], conditions=(check("hatch"),))
twine_publish: px.TaskSpec = px.TaskSpec(
"twine_publish",
cmd=[
"twine",
"upload",
"--disable-progress-bar",
],
conditions=(check("twine"),),
)
tox: px.TaskSpec = px.TaskSpec("tox", cmd=["tox", "-p", "auto"], conditions=(check("tox"),))
ruff_lint: px.TaskSpec = px.TaskSpec("lint", cmd=["ruff", "check", "--fix", "--unsafe-fixes"])
ruff_format: px.TaskSpec = px.TaskSpec("format", cmd=["ruff", "format", "--check", "."], depends_on=("lint",))
mypy_check: px.TaskSpec = px.TaskSpec("typecheck", cmd=["mypy", "."])
ty_check: px.TaskSpec = px.TaskSpec("ty_check", cmd=["ty", "check", "."])
doc: px.TaskSpec = px.TaskSpec("doc", cmd=["sphinx-build", "-b", "html", "docs", "docs/_build"])
hatch_publish: px.TaskSpec = px.TaskSpec("publish_python", cmd=["hatch", "publish"])
twine_publish: px.TaskSpec = px.TaskSpec("twine_publish", cmd=["twine", "upload", "--disable-progress-bar"])
tox: px.TaskSpec = px.TaskSpec("tox", cmd=["tox", "-p", "auto"])
def main():
@@ -128,13 +65,13 @@ def main():
🔨 构建命令:
pymake b - 构建 Python 主包 (uv build)
pymake bc - 构建 Rust 核心模块 (maturin build)
pymake ba - 构建所有包 (先 Rust 后 Python)
pymake ba - 构建所有包 (先 Python 后 Rust)
📦 安装命令 (开发模式):
pymake sync - 安装依赖包 (uv sync)
🧹 清理命令:
pymake c - 清理所有构建产物
pymake c - 清理所有构建产物 (gitt c)
🛠️ 开发工具:
pymake t - 运行测试 (pytest)
@@ -145,29 +82,24 @@ def main():
pymake doc - 构建文档 (sphinx)
🔬 多版本测试:
pymake tox - 多版本 Python 测试 (3.8-3.14)
pymake tox_install - 安装所有 Python 版本 (仅安装不测试)
pymake tox - 多版本 Python 测试 (tox -p auto)
📦 发布命令:
pymake pb - 发布到 PyPI (hatch publish)
pymake pba - 发布所有包 (先 Rust 后 Python)
pymake pbc - 发布 Rust 核心模块 (maturin publish)
pymake pb - 发布到 PyPI (twine + hatch)
💡 常用工作流:
1. 初始化开发环境: pymake ia
2. 日常开发: pymake lint && pymake t
3. 构建发布包: pymake ba
4. 多版本兼容性测试: pymake tox
5. 发布到 PyPI: pymake pb
6. 清理重新开始: pymake ca && pymake ia
1. 日常开发: pymake lint && pymake t
2. 构建发布包: pymake ba
3. 多版本兼容性测试: pymake tox
4. 发布到 PyPI: pymake pb
📝 示例:
pymake ba # 构建所有包
pymake ia # 安装开发环境
pymake sync # 安装依赖
pymake t # 运行测试
pymake tox # 多版本兼容性测试
pymake lint # 格式化代码
pymake ca # 清理所有构建产物
pymake type # 类型检查
"""
runner = px.CliRunner(
strategy="sequential",
@@ -185,7 +117,7 @@ def main():
"t": px.Graph.from_specs([test]),
"tc": px.Graph.from_specs([test, test_coverage]),
"tf": px.Graph.from_specs([test_fast]),
"lint": px.Graph.from_specs([ruff_lint]),
"lint": px.Graph.from_specs([ruff_lint, ruff_format]),
"type": px.Graph.from_specs([mypy_check, ty_check]),
"doc": px.Graph.from_specs([doc]),
"pb": px.Graph.from_specs([twine_publish, hatch_publish]),
src/pyflowx/context.py
+1 -3
View File
@@ -82,9 +82,7 @@ def build_call_args(
)
# 与本任务相关的上下文子集。
dep_context: dict[str, Any] = {
name: context[name] for name in spec.depends_on if name in context
}
dep_context: dict[str, Any] = {name: context[name] for name in spec.depends_on if name in context}
# 检测静态 kwargs 与依赖名的冲突。
collisions = set(spec.kwargs) & set(dep_context)
src/pyflowx/executors.py
+2 -2
View File
@@ -96,7 +96,7 @@ def _run_sync_with_retry(
result: TaskResult[Any] = TaskResult(spec=spec)
# 检查条件是否满足
if spec.conditions and not spec.should_execute():
if not spec.should_execute():
result.status = TaskStatus.SKIPPED
result.finished_at = datetime.now()
logger.info("task %r skipped (条件不满足)", spec.name)
@@ -131,7 +131,7 @@ async def _run_async_with_retry(
result: TaskResult[Any] = TaskResult[Any](spec=spec)
# 检查条件是否满足
if spec.conditions and not spec.should_execute():
if not spec.should_execute():
result.status = TaskStatus.SKIPPED
result.finished_at = datetime.now()
logger.info("task %r skipped (条件不满足)", spec.name)
src/pyflowx/storage.py
+1 -3
View File
@@ -112,9 +112,7 @@ class JSONBackend(StateBackend):
try:
_ = json.dumps(value)
except (TypeError, ValueError) as exc:
raise StorageError(
f"result of task {name!r} is not JSON-serialisable", exc
) from exc
raise StorageError(f"result of task {name!r} is not JSON-serialisable", exc) from exc
self._store[name] = value
self._flush()
src/pyflowx/task.py
+31 -3
View File
@@ -112,6 +112,12 @@ class TaskSpec(Generic[T]):
是否在命令执行时显示详细输出。``True`` 时会打印执行的命令
及其标准输出/标准错误。仅在使用 ``cmd`` 参数时有效。
``False`` 时静默捕获输出（失败时仍会包含在错误信息中）。
skip_if_missing:
仅对 ``cmd`` 为 ``list[str]`` 的任务有效。``True`` 时自动检查
命令是否存在（通过 :func:`shutil.which`），不存在则跳过任务
（标记为 SKIPPED）而非失败。适用于构建工具场景，避免因未安装
某些工具（如 maturin、tox）而导致整个图执行失败。
对于 ``str`` (shell) 和 ``Callable`` 类型的 ``cmd``，此参数无效。
"""
name: str
@@ -126,6 +132,7 @@ class TaskSpec(Generic[T]):
conditions: Tuple[Condition, ...] = ()
cwd: Optional[Path] = None
verbose: bool = False
skip_if_missing: bool = True
def __post_init__(self) -> None:
if not self.name:
@@ -257,10 +264,31 @@ class TaskSpec(Generic[T]):
Returns
-------
bool
若所有条件都返回 ``True``，则返回 ``True``；
否则返回 ``False``。
若所有条件都返回 ``True``，且 ``skip_if_missing`` 检查通过，
则返回 ``True``；否则返回 ``False``。
"""
return all(condition() for condition in self.conditions)
if not all(condition() for condition in self.conditions):
return False
return not (self.skip_if_missing and not self._is_cmd_available())
def _is_cmd_available(self) -> bool:
"""检查 ``cmd`` 是否可用.
仅对 ``list[str]`` 类型的 ``cmd`` 进行检查（通过 :func:`shutil.which`）。
对于 ``str`` (shell) 和 ``Callable`` 类型，始终返回 ``True``。
Returns
-------
bool
命令可用返回 ``True``，否则返回 ``False``。
"""
import shutil
cmd = self.cmd
if isinstance(cmd, list) and cmd:
return shutil.which(cmd[0]) is not None
return True
@dataclass