diff --git a/README.md b/README.md index b6803d2..c3b41bd 100644 --- a/README.md +++ b/README.md @@ -555,19 +555,16 @@ uv run ruff format --check src tests | `storage.py` | 状态后端：`MemoryBackend` / `JSONBackend`（batch flush） | | `runner.py` | CLI 运行器：`CliRunner` | | `report.py` | 运行结果：`RunReport` / `TaskResult` | -| `yaml_loader.py` | YAML 任务编排：GitHub Actions 风格 schema 解析（`load_yaml` / `parse_yaml_string` / `run_cli`） | -| `registry.py` | 函数注册中心：`register_fn` / `get_fn` / `has_fn`（YAML 的 `fn:` 引用） | +| `tools.py` | `@px.tool` 装饰器：`ToolSpec` / `run_tool` / `list_tools` / `list_subcommands` | | `profiling.py` | 性能分析：`Profiler` 任务耗时统计 | | `errors.py` | 错误家族：`PyFlowXError` 子类 | -| `ops/` | 工具函数（dev/files/llm/media/system），被 YAML 的 `fn:` 引用 | +| `ops/` | CLI 工具模块集合（每个子模块用 `@px.tool` 注册一个工具） | ### CLI 工具 | 模块 | 职责 | |------|------| -| `cli/pf.py` | 统一入口：`pf [command]`，自动发现 `configs/*.yaml` 并路由 | -| `configs/` | YAML 工具配置（clr/taskkill/which/msdownload/sglang/dockercmd/envdev 等） | -| `cli/yamlrun.py` | YAML pipeline 执行器，`pf yamlrun pipeline.yaml` 调用 | +| `cli/pf.py` | 统一入口：`pf [command]`，按需导入 `ops/.py` 触发 `@px.tool` 注册 | | `cli/profiler.py` | 性能分析 CLI | | `cli/emlmanager.py` | 邮件管理 CLI | diff --git a/docs/api.rst b/docs/api.rst index 030f39a..b0fd250 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -50,21 +50,17 @@ API 参考 :members: :undoc-members: -YAML 编排 ---------- +@px.tool 工具 +------------- -.. autofunction:: pyflowx.load_yaml -.. autofunction:: pyflowx.parse_yaml_string -.. autofunction:: pyflowx.run_yaml -.. autofunction:: pyflowx.run_cli -.. autofunction:: pyflowx.build_cli_parser +.. autoclass:: pyflowx.ToolSpec + :members: + :undoc-members: -函数注册 --------- - -.. autofunction:: pyflowx.register_fn -.. autofunction:: pyflowx.get_fn -.. autofunction:: pyflowx.has_fn +.. autofunction:: pyflowx.tool +.. autofunction:: pyflowx.run_tool +.. autofunction:: pyflowx.list_tools +.. autofunction:: pyflowx.list_subcommands 命令执行 -------- diff --git a/pyproject.toml b/pyproject.toml index 9720f94..79180bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ classifiers = [ ] dependencies = [ "graphlib_backport >= 1.0.0; python_version < '3.9'", - "pyyaml>=6.0.1", "typing-extensions>=4.13.2; python_version < '3.13'", ] description = "Lightweight, type-safe DAG task scheduler with multi-strategy execution." @@ -25,10 +24,9 @@ requires-python = ">=3.8" version = "0.4.7" [project.scripts] -emlman = "pyflowx.cli.emlmanager:main" -pf = "pyflowx.cli.pf:main" -pxp = "pyflowx.cli.profiler:main" -yamlrun = "pyflowx.cli.yamlrun:main" +emlman = "pyflowx.cli.emlmanager:main" +pf = "pyflowx.cli.pf:main" +pxp = "pyflowx.cli.profiler:main" [project.optional-dependencies] dev = [ @@ -45,7 +43,6 @@ dev = [ "ruff>=0.8.0", "tox-uv>=1.13.1", "tox>=4.25.0", - "types-PyYAML>=6.0.12", ] docs = ["myst-parser>=3.0", "sphinx-rtd-theme>=2.0", "sphinx>=7.0"] office = [ @@ -134,7 +131,7 @@ select = [ ] [tool.ruff.lint.per-file-ignores] -"**/tests/**" = ["ARG001", "ARG002"] +"**/tests/**" = ["ARG001", "ARG002"] "src/pyflowx/tools.py" = ["PLR0911"] [tool.pyrefly] diff --git a/src/pyflowx/__init__.py b/src/pyflowx/__init__.py index 55ce985..a470add 100644 --- a/src/pyflowx/__init__.py +++ b/src/pyflowx/__init__.py @@ -83,7 +83,6 @@ from .errors import ( from .executors import Strategy, run from .graph import Graph, GraphDefaults from .profiling import ProfileReport, TaskProfile -from .registry import FnRegistry, get_fn, has_fn, register_fn from .report import RunReport from .runner import CliExitCode, CliRunner from .storage import JSONBackend, MemoryBackend, StateBackend @@ -101,7 +100,6 @@ from .task import ( task_template, ) from .tools import ToolSpec, list_subcommands, list_tools, run_tool, tool -from .yaml_loader import YamlLoadError, build_cli_parser, load_yaml, parse_yaml_string, run_cli, run_yaml __version__ = "0.4.7" @@ -119,7 +117,6 @@ __all__ = [ "Context", "CycleError", "DuplicateTaskError", - "FnRegistry", "Graph", "GraphComposer", "GraphDefaults", @@ -144,24 +141,15 @@ __all__ = [ "TaskStatus", "TaskTimeoutError", "ToolSpec", - "YamlLoadError", "build_call_args", - "build_cli_parser", "cmd", "compose", "describe_injection", - "get_fn", - "has_fn", "list_subcommands", "list_tools", - "load_yaml", - "parse_yaml_string", - "register_fn", "run", - "run_cli", "run_command", "run_tool", - "run_yaml", "task", "task_template", "tool", diff --git a/src/pyflowx/cli/pf.py b/src/pyflowx/cli/pf.py index b0926e5..6afb7c2 100644 --- a/src/pyflowx/cli/pf.py +++ b/src/pyflowx/cli/pf.py @@ -1,7 +1,7 @@ """PyFlowX 统一 CLI 入口. 通过 ``pf [command] [options]`` 调用所有工具, -工具定义在 ``configs/`` 目录下的 YAML 文件中. +工具定义在 ``pyflowx.ops`` 子包中, 每个模块用 ``@px.tool`` 装饰器注册. 用法 ---- @@ -16,21 +16,18 @@ from __future__ import annotations import contextlib import importlib import sys -from pathlib import Path from typing import Sequence -import pyflowx as px +from pyflowx.tools import _TOOL_REGISTRY, run_tool class PfApp: """pf 统一入口应用. - 路由 ``pf [command]`` 到 YAML 配置工具或传统 Python 工具. + 路由 ``pf [command]`` 到 ``@px.tool`` 注册的工具或传统 Python 工具. """ - _CONFIGS_DIR = Path(__file__).parent.parent / "configs" - - # 工具名到 YAML 配置文件的映射 (支持短别名) + # 工具名到 ops 模块名的映射 (支持短别名) _TOOL_ALIASES: dict[str, str] = { "autofmt": "autofmt", "af": "autofmt", @@ -91,12 +88,11 @@ class PfApp: "which": "which", } - # 传统工具: 有自己的 main() 函数 (无法 YAML 化的复杂逻辑) + # 传统工具: 有自己的 main() 函数 (无法 @px.tool 化的复杂逻辑) _LEGACY_TOOLS: dict[str, str] = { "emlman": "pyflowx.cli.emlmanager:main", "profiler": "pyflowx.cli.profiler:main", "pxp": "pyflowx.cli.profiler:main", - "yamlrun": "pyflowx.cli.yamlrun:main", } def __init__(self, argv: Sequence[str] | None = None) -> None: @@ -120,16 +116,20 @@ class PfApp: tool_type, target = resolved if tool_type == "legacy": return self._run_legacy(target, rest_argv) - return self._run_tool_or_yaml(target, rest_argv) + return self._run_tool(target, rest_argv) def _list_tools(self) -> None: """列出所有可用工具.""" print("PyFlowX 工具列表:") print() - print("YAML 配置工具:") + print("@px.tool 工具:") yaml_tools = sorted(set(self._TOOL_ALIASES.values())) for tool in yaml_tools: - print(f" pf {tool:<15} - {self._tool_description(tool)}") + description = self._tool_description(tool) + if description: + print(f" pf {tool:<15} - {description}") + else: + print(f" pf {tool:<15}") print() print("传统工具:") for tool in sorted(self._LEGACY_TOOLS): @@ -140,28 +140,33 @@ class PfApp: print(" pf pymake b") def _tool_description(self, tool_name: str) -> str: - """获取工具描述 (从 YAML cli.description).""" - config_path = self._CONFIGS_DIR / f"{tool_name}.yaml" - if not config_path.exists(): - return "" - try: - import yaml + """获取工具描述 (从 _TOOL_REGISTRY 中已注册 ToolSpec 的 description/help).""" + # 触发 @px.tool 注册 + with contextlib.suppress(ImportError): + importlib.import_module(f"pyflowx.ops.{tool_name}") - data = yaml.safe_load(config_path.read_text(encoding="utf-8")) - if isinstance(data, dict) and isinstance(data.get("cli"), dict): - return str(data["cli"].get("description", "")) - except Exception: - pass + if tool_name not in _TOOL_REGISTRY: + return "" + + subs = _TOOL_REGISTRY[tool_name] + # 优先取 description, 否则取任一 subcommand 的 help + for spec in subs.values(): + if spec.description: + return spec.description + # 取第一个非 hidden subcommand 的 help + for spec in subs.values(): + if not spec.hidden and spec.help: + return spec.help return "" def _resolve_tool(self, name: str) -> tuple[str, str] | None: """解析工具名, 返回 (类型, 目标). - 类型: "yaml" 或 "legacy" - 目标: YAML 文件名 (不含 .yaml) 或 legacy 模块路径 + 类型: "tool" 或 "legacy" + 目标: ops 模块名或 legacy 模块路径 """ if name in self._TOOL_ALIASES: - return ("yaml", self._TOOL_ALIASES[name]) + return ("tool", self._TOOL_ALIASES[name]) if name in self._LEGACY_TOOLS: return ("legacy", self._LEGACY_TOOLS[name]) return None @@ -182,30 +187,20 @@ class PfApp: finally: sys.argv = original_argv - def _run_tool_or_yaml(self, target: str, argv: list[str]) -> int: - """运行工具: 优先 @px.tool, 回退 YAML. + def _run_tool(self, target: str, argv: list[str]) -> int: + """运行 @px.tool 工具. - 尝试导入 ``pyflowx.ops.`` 触发 ``@px.tool`` 注册; - 若注册表中已有该工具, 走 ``run_tool``; 否则回退到 YAML 配置. + 导入 ``pyflowx.ops.`` 触发 ``@px.tool`` 注册, 然后调用 ``run_tool``. """ - # 尝试加载 ops/.py 触发 @px.tool 注册 with contextlib.suppress(ImportError): importlib.import_module(f"pyflowx.ops.{target}") - # 若 @px.tool 已注册该工具, 走 run_tool - from pyflowx.tools import _TOOL_REGISTRY, run_tool - - if target in _TOOL_REGISTRY: - return run_tool(target, argv) - - # 回退到 YAML - config_path = self._CONFIGS_DIR / f"{target}.yaml" - if not config_path.exists(): + if target not in _TOOL_REGISTRY: print(f"错误: 未找到工具 '{target}'", file=sys.stderr) print("运行 'pf' 查看可用工具列表", file=sys.stderr) return 1 - return px.run_cli(config_path, argv) + return run_tool(target, argv) def main() -> None: diff --git a/src/pyflowx/cli/yamlrun.py b/src/pyflowx/cli/yamlrun.py deleted file mode 100644 index 17411a6..0000000 --- a/src/pyflowx/cli/yamlrun.py +++ /dev/null @@ -1,109 +0,0 @@ -"""YAML 任务编排执行工具. - -从 YAML 文件加载 GitHub Actions 风格的任务图并执行. -支持串并行编排、矩阵扇出、条件执行等 CI/CD 核心概念. - -用法 ----- - yamlrun pipeline.yaml # 执行 YAML 任务图 - yamlrun pipeline.yaml --strategy thread # 指定执行策略 - yamlrun pipeline.yaml --dry-run # 仅打印任务分层, 不执行 - yamlrun pipeline.yaml --list # 列出所有任务名 - yamlrun pipeline.yaml --quiet # 静默模式 - -示例 YAML ----------- -:: - - strategy: thread - jobs: - setup: - cmd: ["git", "clone", "https://github.com/foo/bar"] - build: - needs: [setup] - cmd: ["python", "-m", "build"] - test: - needs: [build] - cmd: ["pytest"] - strategy: - matrix: - python: ["3.8", "3.9"] -""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path -from typing import cast - -import pyflowx as px -from pyflowx.executors import Strategy - - -def main() -> None: - """YAML 任务编排执行工具主函数.""" - parser = argparse.ArgumentParser( - description="YamlRun - 从 YAML 文件加载并执行任务图", - usage="yamlrun [--strategy STRATEGY] [--dry-run] [--list] [--quiet]", - ) - parser.add_argument("file", type=str, help="YAML 任务图文件路径") - parser.add_argument( - "--strategy", - type=str, - default=None, - help="执行策略: sequential/thread/async/dependency (默认: YAML 中指定的策略或 dependency)", - ) - parser.add_argument("--dry-run", action="store_true", help="仅打印任务分层, 不执行") - parser.add_argument("--list", action="store_true", help="列出所有任务名后退出") - parser.add_argument("--quiet", action="store_true", help="静默模式, 不打印详细输出") - args = parser.parse_args() - - file_path = Path(args.file) - if not file_path.exists(): - print(f"错误: 文件不存在: {file_path}", file=sys.stderr) - sys.exit(1) - - try: - graph = px.Graph.from_yaml(file_path) - except px.YamlLoadError as e: - print(f"错误: YAML 加载失败: {e}", file=sys.stderr) - sys.exit(1) - - if args.list: - print("任务列表:") - for name in graph.names: - spec = graph.spec(name) - deps = ", ".join(spec.depends_on) if spec.depends_on else "(无依赖)" - print(f" - {name} (依赖: {deps})") - sys.exit(0) - - layers = graph.layers() - print(f"任务分层 ({len(layers)} 层):") - for i, layer in enumerate(layers): - print(f" 层 {i + 1}: {layer}") - - if args.dry_run: - print("\n[dry-run] 跳过执行") - sys.exit(0) - - strategy = args.strategy or graph.defaults.strategy or "dependency" - print(f"\n执行策略: {strategy}") - print(f"任务总数: {len(graph.names)}") - print("-" * 40) - - report = px.run(graph, strategy=cast(Strategy, strategy), verbose=not args.quiet) - - print("-" * 40) - succeeded = report.succeeded_tasks() - failed = report.failed_tasks() - skipped = report.skipped_tasks() - print(f"完成: {len(succeeded)} 成功 / {len(failed)} 失败 / {len(skipped)} 跳过 (共 {len(graph.names)})") - - if failed: - print(f"失败任务: {failed}") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/src/pyflowx/configs/envdev.yaml b/src/pyflowx/configs/envdev.yaml deleted file mode 100644 index c2906b9..0000000 --- a/src/pyflowx/configs/envdev.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# envdev - 开发环境镜像源配置工具 -# 用法: -# pf envdev -# pf envdev --python-mirror aliyun --conda-mirror ustc --rust-mirror ustc --rust-version nightly -# 说明 -# 配置 Python / Conda / Rust 镜像源 (Linux 还会安装 Qt 库、中文字体、Docker). -# 所有镜像源参数互不影响, 可单独使用. -# Linux 专用操作 (系统镜像/Qt/字体/Docker) 在非 Linux 平台上由函数内部跳过. -strategy: thread -variables: - PYTHON_MIRROR: tsinghua - CONDA_MIRROR: tsinghua - RUST_MIRROR: tsinghua - RUST_VERSION: stable -cli: - description: "EnvDev - 开发环境镜像源配置工具" - usage: "pf envdev [options]" - options: - - name: PYTHON_MIRROR - flag: "--python-mirror" - type: str - default: tsinghua - help: "Python 镜像源: tsinghua/aliyun/huaweicloud/ustc/zju (默认: tsinghua)" - - name: CONDA_MIRROR - flag: "--conda-mirror" - type: str - default: tsinghua - help: "Conda 镜像源: tsinghua/ustc/bsfu/aliyun (默认: tsinghua)" - - name: RUST_MIRROR - flag: "--rust-mirror" - type: str - default: tsinghua - help: "Rust 镜像源: tsinghua/ustc/aliyun (默认: tsinghua)" - - name: RUST_VERSION - flag: "--rust-version" - type: str - default: stable - help: "Rust 版本: stable/nightly/beta (默认: stable)" -jobs: - # Linux 系统镜像配置 (函数内部判断平台与已配置状态, 非自动跳过) - setup_linux_mirror: - fn: setup_linux_system_mirror - # 安装 Qt 依赖 (仅 Linux, 函数内部判断) - install_qt_libs: - fn: install_linux_qt_libs - needs: [setup_linux_mirror] - allow-upstream-skip: true - # 安装中文字体 (仅 Linux, 函数内部判断) - install_fonts: - fn: install_linux_fonts - needs: [setup_linux_mirror] - allow-upstream-skip: true - # 安装 Docker (仅 Linux, 函数内部判断) - install_docker: - fn: install_linux_docker - needs: [setup_linux_mirror] - allow-upstream-skip: true - # 配置 Python 镜像源 (跨平台) - setup_python: - fn: setup_python_mirror - args: ["${PYTHON_MIRROR}"] - # 配置 Conda 镜像源 (跨平台) - setup_conda: - fn: setup_conda_mirror - args: ["${CONDA_MIRROR}"] - # 配置 Rust 镜像源 (跨平台) - setup_rust: - fn: setup_rust_mirror - args: ["${RUST_MIRROR}", "${RUST_VERSION}"] - # 下载 Rustup 安装脚本 (跨平台, 已安装时由函数内部跳过) - download_rustup: - fn: download_rustup_script - # 安装 Rust 工具链 (rustup 未安装时由函数内部跳过) - install_rust: - fn: install_rust_toolchain - args: ["${RUST_VERSION}"] - needs: [setup_rust, download_rustup] - allow-upstream-skip: true diff --git a/src/pyflowx/configs/gittool.yaml b/src/pyflowx/configs/gittool.yaml deleted file mode 100644 index 970608e..0000000 --- a/src/pyflowx/configs/gittool.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# gittool - Git 执行工具 -# 用法: -# pf gittool a -# pf gittool c -# pf gittool i -# pf gittool isub -# pf gittool p -# pf gittool pl -strategy: thread -variables: - # git clean -e 参数列表 (展开为 cmd 数组元素) - CLEAN_EXCLUDES: ["-e", ".venv", "-e", ".tox", "-e", ".pytest_cache", - "-e", ".ruff_cache", "-e", "node_modules", - "-e", ".idea", "-e", ".vscode", - "-e", ".trae", "-e", ".qoder", - "-e", ".editorconfig", "-e", "idea.config", - "-e", "idea_modules.xml", "-e", "vcs.xml"] -cli: - description: "GitTool - Git 执行工具" - usage: "pf gittool " - subcommands: - a: - help: "添加并提交" - c: - help: "清理并查看状态" - i: - help: "初始化并提交" - isub: - help: "初始化子目录" - p: - help: "推送" - pl: - help: "拉取" -jobs: - a: - fn: git_add_commit - args: ["chore: update"] - clean: - cmd: ["git", "clean", "-xfd", "${CLEAN_EXCLUDES}"] - c: - needs: [clean] - cmd: ["git", "status", "--porcelain"] - i: - fn: git_init_add_commit - args: ["init commit"] - isub: - fn: init_sub_dirs - p: - cmd: ["git", "push"] - pl: - cmd: ["git", "pull"] diff --git a/src/pyflowx/configs/msdownload.yaml b/src/pyflowx/configs/msdownload.yaml deleted file mode 100644 index 795fcc3..0000000 --- a/src/pyflowx/configs/msdownload.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# msdownload - ModelScope 下载工具 -# 用法: -# pf msdownload Qwen/Qwen2.5-Coder-32B-Instruct -# pf msdownload AI-ModelScope/MNIST --type dataset --dir ./data -strategy: thread -variables: - NAME: "" - TYPE: model - DIR: null -cli: - description: "MSDownload - ModelScope 模型/数据集下载工具" - usage: "pf msdownload [--type TYPE] [--dir DIR]" - positional: - - name: NAME - type: str - help: "目标名称 (如: Qwen/Qwen2.5-Coder-32B-Instruct)" - options: - - name: TYPE - flag: "--type" - type: str - default: model - help: "目标类型: model / dataset / space (默认: model)" - - name: DIR - flag: "--dir" - type: str - default: null - help: "下载目录 (默认: ~/.models/)" -jobs: - download: - fn: msdownload_run - args: ["${NAME}"] - kwargs: - target_type: ${TYPE} - download_dir: ${DIR} diff --git a/src/pyflowx/configs/pymake.yaml b/src/pyflowx/configs/pymake.yaml deleted file mode 100644 index aa93279..0000000 --- a/src/pyflowx/configs/pymake.yaml +++ /dev/null @@ -1,125 +0,0 @@ -# pymake - 项目构建工具 -# 用法 -# pf pymake -# 命令 -# b: 构建 Python 主包 (uv build) -# ba: 构建所有包 (Python + Rust) -# bc: 构建 Rust 核心模块 (maturin build) -# bump: 升级版本号 (清理 + 检查 + add + bumpversion) -# bumpmi: 升级次版本号 (bumpversion minor) -# c: 清理构建产物 (调用 gitt c) -# cov: 测试并生成覆盖率 -# doc: 构建 Sphinx 文档 -# lint: 代码格式化与检查 (ruff) -# p: 推送代码 (清理 + push + push tags) -# pb: 发布到 PyPI (twine + hatch) -# sync: 同步依赖 (uv sync) -# t: 运行测试 -# tc: 类型检查 (pyrefly + ruff) -# tf: 快速测试 (无 slow) -# tox: 多版本测试 (tox) -strategy: thread -variables: - CWD: "." -cli: - description: "PyMake - 项目构建工具" - usage: "pf pymake " - options: - - name: CWD - flag: "--cwd" - type: path - required: false - default: "." - help: "工作目录 (默认: 当前目录)" - subcommands: - b: {help: "构建 Python 主包 (uv build)"} - ba: {help: "构建所有包 (Python + Rust)"} - bc: {help: "构建 Rust 核心模块 (maturin build)"} - bump: {help: "升级版本号 (清理 + 检查 + add + bumpversion)"} - bumpmi: {help: "升级次版本号 (bumpversion minor)"} - c: {help: "清理构建产物 (调用 gitt c)"} - cov: {help: "测试并生成覆盖率"} - doc: {help: "构建 Sphinx 文档"} - lint: {help: "代码格式化与检查 (ruff)"} - p: {help: "推送代码 (清理 + push + push tags)"} - pb: {help: "发布到 PyPI (twine + hatch)"} - sync: {help: "同步依赖 (uv sync)"} - t: {help: "运行测试"} - tc: {help: "类型检查 (pyrefly + ruff)"} - tf: {help: "快速测试 (无 slow)"} - tox: {help: "多版本测试 (tox)"} -jobs: - # 单任务别名 - b: - cmd: ["uv", "build"] - cwd: ${CWD} - bc: - cmd: ["maturin", "build", "-r"] - cwd: ${CWD} - sync: - cmd: ["uv", "sync"] - cwd: ${CWD} - c: - cmd: ["pf", "gitt", "c"] - cwd: ${CWD} - t: - cmd: ["pytest", "-m", "not slow", "-n", "8", "--dist", "loadfile", "--color=yes", "--durations=10"] - cwd: ${CWD} - tf: - cmd: ["pytest", "-m", "not slow", "--dist", "loadfile", "--color=yes", "--durations=10"] - cwd: ${CWD} - bumpversion: - cmd: ["pf", "bumpversion", "patch"] - needs: [git_add_all] - cwd: ${CWD} - bumpmi: - cmd: ["pf", "bumpversion", "minor"] - cwd: ${CWD} - doc: - cmd: ["sphinx-build", "-b", "html", "docs", "docs/_build"] - cwd: ${CWD} - lint: - cmd: ["ruff", "check", "--fix", "--unsafe-fixes"] - cwd: ${CWD} - tox: - cmd: ["tox", "-p", "auto"] - cwd: ${CWD} - - # 内部 job (不暴露为 subcommand) - test_coverage: - cmd: ["pytest", "--cov", "-n", "8", "--dist", "loadfile", "--tb=short", "-v", "--color=yes", "--durations=10"] - needs: [c] - cwd: ${CWD} - pyrefly_check: - cmd: ["pyrefly", "check", "."] - cwd: ${CWD} - git_add_all: - cmd: ["git", "add", "-A"] - needs: [tc] - cwd: ${CWD} - git_push: - cmd: ["git", "push"] - cwd: ${CWD} - git_push_tags: - cmd: ["git", "push", "--tags"] - cwd: ${CWD} - twine_publish: - cmd: ["twine", "upload", "--disable-progress-bar"] - cwd: ${CWD} - publish_python: - cmd: ["hatch", "publish"] - cwd: ${CWD} - - # 聚合 job (方向 B: 有 needs 无 cmd/fn) - ba: - needs: [b, bc] - bump: - needs: [bumpversion] - cov: - needs: [test_coverage] - tc: - needs: [c, pyrefly_check, lint] - p: - needs: [c, git_push, git_push_tags] - pb: - needs: [twine_publish, publish_python] diff --git a/src/pyflowx/configs/sglang.yaml b/src/pyflowx/configs/sglang.yaml deleted file mode 100644 index 916d965..0000000 --- a/src/pyflowx/configs/sglang.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# sglang - SGLang 本地模型服务 -# 用法: -# pf sglang -# pf sglang --model ~/.models/Qwen2.5-Coder-32B-Instruct-AWQ -# pf sglang --port 9000 --mem 0.8 -strategy: sequential -variables: - MODEL: "~/.models/Qwen2.5-Coder-32B-Instruct-AWQ" - PORT: 8000 - CTX_LEN: 32768 - MEM: 0.75 - HOST: "0.0.0.0" - LOG_LEVEL: "info" -cli: - description: "SGLang - 本地模型服务启动工具" - usage: "pf sglang [options]" - options: - - name: MODEL - flag: "--model" - type: str - default: "~/.models/Qwen2.5-Coder-32B-Instruct-AWQ" - help: "模型路径" - - name: PORT - flag: "--port" - type: int - default: 8000 - help: "服务端口 (默认: 8000)" - - name: CTX_LEN - flag: "--ctx-len" - type: int - default: 32768 - help: "最大上下文长度 (默认: 32768)" - - name: MEM - flag: "--mem" - type: float - default: 0.75 - help: "显存占比 0-1 (默认: 0.75)" - - name: HOST - flag: "--host" - type: str - default: "0.0.0.0" - help: "主机地址 (默认: 0.0.0.0)" - - name: LOG_LEVEL - flag: "--log-level" - type: str - default: "info" - help: "日志级别 (默认: info)" -jobs: - install: - fn: install_sglang - run: - fn: run_sglang - needs: [install] - kwargs: - model: ${MODEL} - port: ${PORT} - ctx_len: ${CTX_LEN} - mem_fraction: ${MEM} - host: ${HOST} - log_level: ${LOG_LEVEL} diff --git a/src/pyflowx/graph.py b/src/pyflowx/graph.py index eb2f377..31aab47 100644 --- a/src/pyflowx/graph.py +++ b/src/pyflowx/graph.py @@ -20,7 +20,6 @@ __all__ = [ import inspect import sys from dataclasses import dataclass, field, replace -from pathlib import Path from typing import Any, Callable, Iterable, Mapping, Sequence from .errors import CycleError, DuplicateTaskError, MissingDependencyError @@ -235,38 +234,6 @@ class Graph: graph.validate() return graph - @classmethod - def from_yaml( - cls, - path: str | Path, - variables: Mapping[str, Any] | None = None, - ) -> Graph: - """从 YAML 文件构建任务图。 - - 参考 GitHub Actions 风格 schema, 支持 jobs/needs/strategy.matrix/if - 等 CI/CD 概念。详见 :mod:`pyflowx.yaml_loader`。 - - Parameters - ---------- - path : str | Path - YAML 文件路径 - variables : Mapping[str, Any] | None - 运行时变量, 用于替换 ``${VAR}`` 占位符 - - Returns - ------- - Graph - 构建好的任务图 - - Raises - ------ - YamlLoadError - 文件不存在、YAML 格式错误、schema 校验失败、循环依赖等 - """ - from .yaml_loader import load_yaml - - return load_yaml(path, variables=variables) - def add_subgraph(self, sub: Graph, *, namespace: str | None = None) -> Graph: """将子图合并到当前图，任务名加命名空间前缀避免冲突。 diff --git a/src/pyflowx/ops/__init__.py b/src/pyflowx/ops/__init__.py index f960bbf..d7c5f92 100644 --- a/src/pyflowx/ops/__init__.py +++ b/src/pyflowx/ops/__init__.py @@ -1,20 +1,34 @@ -"""工具函数模块. +"""ops 子包 - CLI 工具模块集合. -按类别组织 CLI 工具中可复用的函数, 每个子模块使用 ``@px.register_fn`` 注册函数, -供 YAML 任务编排通过 ``fn`` 字段引用. +每个子模块使用 ``@px.tool`` 装饰器注册 CLI 工具, 由 ``pf`` 入口按需 +``importlib.import_module`` 触发注册. 子模块按工具名组织 (一文件一工具). 子模块 ------ -- :mod:`files` —— 文件日期/等级/备份/压缩相关函数 -- :mod:`dev` —— 开发工具 (ruff/pip/git/envdev/dockercmd) 相关函数 -- :mod:`bumpversion` —— 版本号管理相关函数 -- :mod:`media` —— PDF/截图相关函数 -- :mod:`system` —— LS-DYNA/SSH/打包/清屏/进程终止相关函数 -- :mod:`llm` —— ModelScope 下载/SGLang 服务相关函数 +- :mod:`autofmt` —— 代码格式化 (ruff) +- :mod:`bumpversion` —— 版本号管理 +- :mod:`clr` —— 清屏 +- :mod:`dockercmd` —— Docker 镜像登录 +- :mod:`envdev` —— 开发环境镜像源配置 +- :mod:`filedate` —— 文件日期修改 +- :mod:`filelevel` —— 文件等级 +- :mod:`folderback` —— 文件夹备份 +- :mod:`folderzip` —— 文件夹压缩 +- :mod:`gittool` —— Git 工具 +- :mod:`lscalc` —— LS-DYNA 计算 +- :mod:`msdownload` —— ModelScope 下载 +- :mod:`packtool` —— 打包工具 +- :mod:`pdftool` —— PDF 工具 +- :mod:`piptool` —— pip 工具 +- :mod:`pymake` —— Python 项目构建 +- :mod:`reseticoncache` —— 重置图标缓存 +- :mod:`screenshot` —— 截图 +- :mod:`sglang` —— SGLang 服务 +- :mod:`sshcopyid` —— SSH 公钥拷贝 +- :mod:`taskkill` —— 进程终止 +- :mod:`which` —— 命令查找 """ from __future__ import annotations -from . import bumpversion, dev, files, llm, media, system - -__all__ = ["bumpversion", "dev", "files", "llm", "media", "system"] +__all__: list[str] = [] diff --git a/src/pyflowx/ops/bumpversion.py b/src/pyflowx/ops/bumpversion.py index a2ab448..8266743 100644 --- a/src/pyflowx/ops/bumpversion.py +++ b/src/pyflowx/ops/bumpversion.py @@ -1,8 +1,8 @@ """版本号管理模块. 提供单文件版本号更新 (``bump_file_version``) 与项目级批量版本号同步 -(``bump_project_version``) 能力. 所有公共函数通过 ``@px.register_fn`` 注册, -供 YAML 任务编排引用. +(``bump_project_version``) 能力. ``bump_project_version`` 通过 ``@px.tool`` +注册为 CLI 工具, 可通过 ``pf bumpversion`` 调用. 设计要点 -------- @@ -131,7 +131,6 @@ def _write_version_to_file(file_path: Path, new_version: str) -> bool: # ============================================================================ -@px.register_fn def bump_file_version(file_path: Path, part: BumpVersionType = "patch") -> str | None: """更新单个文件中的版本号. @@ -163,7 +162,6 @@ def bump_file_version(file_path: Path, part: BumpVersionType = "patch") -> str | return new_version -@px.register_fn @px.tool("bumpversion", help="版本号自动管理") def bump_project_version(part: BumpVersionType = "patch", no_tag: bool = False) -> str | None: """批量同步项目所有版本号文件并提交. diff --git a/src/pyflowx/ops/dev.py b/src/pyflowx/ops/dev.py deleted file mode 100644 index 208322e..0000000 --- a/src/pyflowx/ops/dev.py +++ /dev/null @@ -1,823 +0,0 @@ -"""开发工具类函数模块. - -聚合自动格式化 (autofmt)、pip 包管理 (piptool)、git 工具 (gittool)、 -开发环境配置 (envdev)、docker 镜像登录 (dockercmd) 的可复用函数. -版本号管理已抽离到 :mod:`pyflowx.ops.bumpversion`. 所有公共函数通过 -``@px.register_fn`` 注册, 供 YAML 任务编排引用. -""" - -from __future__ import annotations - -import ast -import fnmatch -import getpass -import os -import shutil -import subprocess -from pathlib import Path -from typing import Literal - -import pyflowx as px -from pyflowx.conditions import Constants - -__all__ = [ - "IGNORE_PATTERNS", - "PACKAGE_DIR", - "REQUIREMENTS_FILE", - "_PROTECTED_PACKAGES", - "add_docstring", - "auto_add_docstrings", - "docker_login_tencent", - "download_rustup_script", - "format_all", - "format_with_ruff", - "generate_module_docstring", - "git_add_commit", - "git_init_add_commit", - "has_files", - "init_sub_dirs", - "install_linux_docker", - "install_linux_fonts", - "install_linux_qt_libs", - "install_rust_toolchain", - "lint_with_ruff", - "not_has_git_repo", - "pip_download", - "pip_freeze", - "pip_reinstall", - "pip_uninstall", - "setup_conda_mirror", - "setup_linux_system_mirror", - "setup_python_mirror", - "setup_rust_mirror", - "sync_pyproject_config", -] - -# ============================================================================ -# autofmt 配置 -# ============================================================================ - -IGNORE_PATTERNS = [ - "__pycache__", - "*.pyc", - "*.pyo", - ".git", - ".venv", - ".idea", - ".vscode", - "*.egg-info", - "dist", - "build", - ".pytest_cache", - ".tox", - ".mypy_cache", -] - -# ============================================================================ -# piptool 配置 -# ============================================================================ - -PACKAGE_DIR = "packages" -REQUIREMENTS_FILE = "requirements.txt" - -_PROTECTED_PACKAGES: frozenset[str] = frozenset( - { - "pyflowx", - "bitool", - } -) - - -# ============================================================================ -# autofmt 私有辅助函数 -# ============================================================================ - - -# ============================================================================ -# autofmt 函数 -# ============================================================================ - - -@px.register_fn -def format_with_ruff(target: Path, fix: bool = True) -> None: - """使用 ruff 格式化代码. - - Parameters - ---------- - target : Path - 目标路径 - fix : bool - 是否自动修复 - """ - cmd = ["ruff", "format", str(target)] - if fix: - cmd.append("--fix") - - subprocess.run(cmd, check=True) - print(f"ruff format 完成: {target}") - - -@px.register_fn -def lint_with_ruff(target: Path, fix: bool = True) -> None: - """使用 ruff 检查代码. - - Parameters - ---------- - target : Path - 目标路径 - fix : bool - 是否自动修复 - """ - cmd = ["ruff", "check", str(target)] - if fix: - cmd.extend(["--fix", "--unsafe-fixes"]) - - subprocess.run(cmd, check=True) - print(f"ruff check 完成: {target}") - - -@px.register_fn -def add_docstring(file_path: Path, docstring: str) -> bool: - """为文件添加 docstring. - - Parameters - ---------- - file_path : Path - 文件路径 - docstring : str - docstring 内容 - - Returns - ------- - bool - 是否成功添加 - """ - try: - content = file_path.read_text(encoding="utf-8") - tree = ast.parse(content) - - first_node = tree.body[0] if tree.body else None - if first_node and isinstance(first_node, ast.Expr) and isinstance(first_node.value, ast.Constant): - return False - - lines = content.splitlines() - doc_lines = docstring.splitlines() - doc_lines.append("") - new_content = "\n".join(doc_lines + lines) - - file_path.write_text(new_content, encoding="utf-8") - print(f"添加 docstring: {file_path}") - return True - - except (OSError, UnicodeDecodeError, SyntaxError) as e: - print(f"处理失败: {file_path} - {e}") - return False - - -@px.register_fn -def generate_module_docstring(file_path: Path) -> str: - """生成模块 docstring. - - Parameters - ---------- - file_path : Path - 文件路径 - - Returns - ------- - str - 生成的 docstring - """ - stem = file_path.stem - parent = file_path.parent.name - - keywords = { - "cli": f"Command-line interface for {parent}", - "gui": f"Graphical user interface for {parent}", - "core": f"Core functionality for {parent}", - "util": f"Utility functions for {parent}", - "model": f"Data models for {parent}", - "test": f"Tests for {parent}", - } - - for key, desc in keywords.items(): - if key in stem.lower(): - return f'"""{desc}."""' - - return f'"""{stem.replace("_", " ").title()} module."""' - - -@px.register_fn -def auto_add_docstrings(root_dir: Path) -> int: - """自动为所有 Python 文件添加 docstring. - - Parameters - ---------- - root_dir : Path - 根目录 - - Returns - ------- - int - 添加的 docstring 数量 - """ - count = 0 - for py_file in root_dir.rglob("*.py"): - if any(pattern in str(py_file) for pattern in IGNORE_PATTERNS): - continue - - docstring = generate_module_docstring(py_file) - if add_docstring(py_file, docstring): - count += 1 - - print(f"共添加 {count} 个 docstring") - return count - - -@px.register_fn -def sync_pyproject_config(root_dir: Path) -> None: - """同步 pyproject.toml 配置到子项目. - - Parameters - ---------- - root_dir : Path - 根目录 - """ - main_toml = root_dir / "pyproject.toml" - if not main_toml.exists(): - print(f"主项目配置文件不存在: {main_toml}") - return - - sub_tomls = [p for p in root_dir.rglob("pyproject.toml") if p != main_toml and ".venv" not in str(p)] - - if not sub_tomls: - print("没有找到子项目的 pyproject.toml") - return - - print(f"找到 {len(sub_tomls)} 个子项目配置文件") - - for sub_toml in sub_tomls: - subprocess.run(["ruff", "format", str(sub_toml)], check=False) - - print("配置同步完成") - - -@px.register_fn -def format_all(root_dir: Path) -> None: - """格式化所有 Python 文件. - - Parameters - ---------- - root_dir : Path - 根目录 - """ - subprocess.run(["ruff", "format", str(root_dir)], check=True) - subprocess.run(["ruff", "check", "--fix", "--unsafe-fixes", str(root_dir)], check=True) - print(f"格式化完成: {root_dir}") - - -# ============================================================================ -# piptool 私有辅助函数 -# ============================================================================ - - -def _get_installed_packages() -> list[str]: - """获取当前环境中所有已安装的包名.""" - try: - result = subprocess.run( - ["pip", "list", "--format=freeze"], - capture_output=True, - text=True, - check=True, - ) - packages: list[str] = [] - for line in result.stdout.strip().split("\n"): - if line and "==" in line: - pkg_name = line.split("==")[0].strip() - packages.append(pkg_name) - except (subprocess.SubprocessError, OSError): - return [] - return packages - - -def _expand_wildcard_packages(pattern: str) -> list[str]: - """展开通配符模式为实际的包名列表.""" - if not any(char in pattern for char in ["*", "?", "[", "]"]): - return [pattern] - - installed_packages = _get_installed_packages() - matched = [pkg for pkg in installed_packages if fnmatch.fnmatchcase(pkg.lower(), pattern.lower())] - return matched - - -def _filter_protected_packages(packages: list[str]) -> list[str]: - """过滤掉受保护的包名.""" - safe = [p for p in packages if p.lower() not in {p.lower() for p in _PROTECTED_PACKAGES}] - filtered = [p for p in packages if p.lower() in {p.lower() for p in _PROTECTED_PACKAGES}] - if filtered: - print(f"跳过受保护的包: {', '.join(filtered)}") - return safe - - -# ============================================================================ -# piptool 函数 -# ============================================================================ - - -@px.register_fn -def pip_uninstall(pkg_names: list[str]) -> None: - """卸载包.""" - packages_to_uninstall: list[str] = [] - for pattern in pkg_names: - packages_to_uninstall.extend(_expand_wildcard_packages(pattern)) - - packages_to_uninstall = _filter_protected_packages(packages_to_uninstall) - - if not packages_to_uninstall: - return - - subprocess.run(["pip", "uninstall", "-y", *packages_to_uninstall], check=True) - - -@px.register_fn -def pip_reinstall(pkg_names: list[str], offline: bool = False) -> None: - """重新安装包.""" - safe_ps = _filter_protected_packages(pkg_names) - if not safe_ps: - print("所有指定的包均为受保护包, 跳过重装") - return - - subprocess.run(["pip", "uninstall", "-y", *safe_ps], check=True) - - options = ["--no-index", "--find-links", "."] if offline else [] - subprocess.run(["pip", "install", *options, *safe_ps], check=True) - - -@px.register_fn -def pip_download(pkg_names: list[str], offline: bool = False) -> None: - """下载包.""" - options = ["--no-index", "--find-links", "."] if offline else [] - subprocess.run( - ["pip", "download", *pkg_names, *options, "-d", PACKAGE_DIR], - check=True, - ) - - -@px.register_fn -def pip_freeze() -> None: - """冻结依赖.""" - result = subprocess.run( - ["pip", "freeze", "--exclude-editable"], - capture_output=True, - text=True, - check=True, - ) - Path(REQUIREMENTS_FILE).write_text(result.stdout) - - -# ============================================================================ -# gittool 函数 -# ============================================================================ - - -@px.register_fn -def init_sub_dirs() -> None: - """初始化子目录的 Git 仓库.""" - sub_dirs = [subdir for subdir in Path.cwd().iterdir() if subdir.is_dir()] - for subdir in sub_dirs: - px.run( - px.Graph().chain( - px.cmd(["git", "init"], conditions=(lambda _: not_has_git_repo(),), cwd=subdir), - px.cmd(["git", "add", "."]), - px.cmd(["git", "commit", "-m", "init commit"]), - ), - ) - - -@px.register_fn -def not_has_git_repo() -> bool: - """检查当前目录没有 Git 仓库.""" - return not Path.cwd().exists() or not (Path.cwd() / ".git").is_dir() - - -@px.register_fn -def has_files() -> bool: - """检查当前 Git 仓库是否有未提交的更改.""" - try: - result = subprocess.run( - ["git", "status", "--porcelain"], - capture_output=True, - text=True, - check=False, - ) - return bool(result.stdout.strip()) - except (subprocess.SubprocessError, OSError): - return False - - -@px.register_fn -def git_add_commit(message: str = "chore: update") -> None: - """执行 git add + git commit (仅当有未提交更改时). - - Parameters - ---------- - message : str - 提交信息 - """ - if not has_files(): - print("没有文件需要提交") - return - subprocess.run(["git", "add", "."], check=True) - subprocess.run(["git", "commit", "-m", message], check=True) - - -@px.register_fn -def git_init_add_commit(message: str = "init commit") -> None: - """执行 git init (若需) + git add + git commit (若有更改). - - Parameters - ---------- - message : str - 提交信息 - """ - if not_has_git_repo(): - subprocess.run(["git", "init"], check=True) - if has_files(): - subprocess.run(["git", "add", "."], check=True) - subprocess.run(["git", "commit", "-m", message], check=True) - else: - print("没有文件需要提交") - - -# ============================================================================ -# envdev 配置 (Python / Conda / Rust 镜像源) -# ============================================================================ - -PyMirrorType = Literal["tsinghua", "aliyun", "huaweicloud", "ustc", "zju"] -CondaMirrorType = Literal["tsinghua", "ustc", "bsfu", "aliyun"] -RustMirrorType = Literal["tsinghua", "ustc", "aliyun"] - -_PIP_INDEX_URLS: dict[str, str] = { - "tsinghua": "https://pypi.tuna.tsinghua.edu.cn/simple", - "aliyun": "https://mirrors.aliyun.com/pypi/simple/", - "huaweicloud": "https://mirrors.huaweicloud.com/repository/pypi/simple/", - "ustc": "https://pypi.mirrors.ustc.edu.cn/simple/", - "zju": "https://mirrors.zju.edu.cn/pypi/simple/", -} - -_PIP_TRUSTED_HOSTS: dict[str, str] = { - "tsinghua": "pypi.tuna.tsinghua.edu.cn", - "aliyun": "mirrors.aliyun.com", - "huaweicloud": "mirrors.huaweicloud.com", - "ustc": "pypi.mirrors.ustc.edu.cn", - "zju": "mirrors.zju.edu.cn", -} - -_UV_PYTHON_INSTALL_MIRROR: str = "https://registry.npmmirror.com/-/binary/python-build-standalone" - -_CONDA_MIRROR_URLS: dict[str, list[str]] = { - "tsinghua": [ - "https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/", - "https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/", - "https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r/", - "https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2/", - "https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/pro/", - "https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/", - "https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/bioconda/", - "https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/menpo/", - "https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/pytorch/", - ], - "ustc": [ - "https://mirrors.ustc.edu.cn/anaconda/pkgs/main/", - "https://mirrors.ustc.edu.cn/anaconda/pkgs/free/", - "https://mirrors.ustc.edu.cn/anaconda/pkgs/r/", - "https://mirrors.ustc.edu.cn/anaconda/pkgs/msys2/", - "https://mirrors.ustc.edu.cn/anaconda/pkgs/pro/", - "https://mirrors.ustc.edu.cn/anaconda/pkgs/dev/", - "https://mirrors.ustc.edu.cn/anaconda/cloud/conda-forge/", - "https://mirrors.ustc.edu.cn/anaconda/cloud/bioconda/", - "https://mirrors.ustc.edu.cn/anaconda/cloud/menpo/", - "https://mirrors.ustc.edu.cn/anaconda/cloud/pytorch/", - ], - "bsfu": [ - "https://mirrors.bsfu.edu.cn/anaconda/pkgs/main/", - "https://mirrors.bsfu.edu.cn/anaconda/pkgs/free/", - "https://mirrors.bsfu.edu.cn/anaconda/pkgs/r/", - "https://mirrors.bsfu.edu.cn/anaconda/pkgs/msys2/", - "https://mirrors.bsfu.edu.cn/anaconda/pkgs/pro/", - "https://mirrors.bsfu.edu.cn/anaconda/pkgs/dev/", - "https://mirrors.bsfu.edu.cn/anaconda/cloud/conda-forge/", - "https://mirrors.bsfu.edu.cn/anaconda/cloud/bioconda/", - "https://mirrors.bsfu.edu.cn/anaconda/cloud/menpo/", - "https://mirrors.bsfu.edu.cn/anaconda/cloud/pytorch/", - ], - "aliyun": [ - "https://mirrors.aliyun.com/anaconda/pkgs/main/", - "https://mirrors.aliyun.com/anaconda/pkgs/free/", - "https://mirrors.aliyun.com/anaconda/pkgs/r/", - "https://mirrors.aliyun.com/anaconda/pkgs/msys2/", - "https://mirrors.aliyun.com/anaconda/pkgs/pro/", - "https://mirrors.aliyun.com/anaconda/pkgs/dev/", - "https://mirrors.aliyun.com/anaconda/cloud/conda-forge/", - "https://mirrors.aliyun.com/anaconda/cloud/bioconda/", - "https://mirrors.aliyun.com/anaconda/cloud/menpo/", - "https://mirrors.aliyun.com/anaconda/cloud/pytorch/", - ], -} - -_RUSTUP_MIRRORS: dict[str, dict[str, str]] = { - "tsinghua": { - "RUSTUP_DIST_SERVER": "https://mirrors.tuna.tsinghua.edu.cn/rustup", - "RUSTUP_UPDATE_ROOT": "https://mirrors.tuna.tsinghua.edu.cn/rustup/rustup", - "TOML_REGISTRY": "https://mirrors.tuna.tsinghua.edu.cn/crates.io-index/", - }, - "aliyun": { - "RUSTUP_DIST_SERVER": "https://mirrors.aliyun.com/rustup", - "RUSTUP_UPDATE_ROOT": "https://mirrors.aliyun.com/rustup/rustup", - "TOML_REGISTRY": "https://mirrors.aliyun.com/crates.io-index/", - }, - "ustc": { - "RUSTUP_DIST_SERVER": "https://mirrors.ustc.edu.cn/rust-static", - "RUSTUP_UPDATE_ROOT": "https://mirrors.ustc.edu.cn/rust-static/rustup", - "TOML_REGISTRY": "https://mirrors.ustc.edu.cn/crates.io-index/", - }, -} - -_RUST_SCCACHE_DIR: Path = Path.home() / ".cargo" / "sccache" -_RUST_SCCACHE_CACHE_SIZE: str = "20G" - - -def _pip_config_path() -> Path: - """返回当前平台的 pip 配置文件路径.""" - if Constants.IS_LINUX: - return Path.home() / ".pip" / "pip.conf" - return Path.home() / "pip" / "pip.ini" - - -@px.register_fn -def setup_python_mirror(mirror: str) -> None: - """配置 Python 镜像源 (设置环境变量 + 写入 pip 配置文件). - - 设置 ``PIP_INDEX_URL`` / ``PIP_TRUSTED_HOSTS`` / ``UV_INDEX_URL`` / - ``UV_PYTHON_INSTALL_MIRROR`` 等环境变量, 并写入 pip 配置文件. - - Parameters - ---------- - mirror : str - 镜像源名称, 见 :data:`_PIP_INDEX_URLS` - """ - if mirror not in _PIP_INDEX_URLS: - print(f"未知 Python 镜像源: {mirror}") - return - - index_url = _PIP_INDEX_URLS[mirror] - trusted_host = _PIP_TRUSTED_HOSTS[mirror] - - os.environ["PIP_INDEX_URL"] = index_url - os.environ["PIP_TRUSTED_HOSTS"] = trusted_host - os.environ["UV_INDEX_URL"] = index_url - os.environ["UV_PYTHON_INSTALL_MIRROR"] = _UV_PYTHON_INSTALL_MIRROR - os.environ["UV_HTTP_TIMEOUT"] = "600" - os.environ["UV_LINK_MODE"] = "copy" - - config_path = _pip_config_path() - config_path.parent.mkdir(parents=True, exist_ok=True) - content = f"[global]\nindex-url = {index_url}\ntrusted-host = {trusted_host}\n" - config_path.write_text(content, encoding="utf-8") - print(f"Python 镜像源已配置: {mirror} -> {config_path}") - - -@px.register_fn -def setup_conda_mirror(mirror: str) -> None: - """配置 Conda 镜像源 (写入 ~/.condarc). - - Parameters - ---------- - mirror : str - 镜像源名称, 见 :data:`_CONDA_MIRROR_URLS` - """ - if mirror not in _CONDA_MIRROR_URLS: - print(f"未知 Conda 镜像源: {mirror}") - return - - urls = _CONDA_MIRROR_URLS[mirror] - config_path = Path.home() / ".condarc" - config_path.parent.mkdir(parents=True, exist_ok=True) - content = "show_channel_urls: true\nchannels:\n - " + "\n - ".join(urls) + "\n - defaults\n" - config_path.write_text(content, encoding="utf-8") - print(f"Conda 镜像源已配置: {mirror} -> {config_path}") - - -@px.register_fn -def setup_rust_mirror(mirror: str, version: str = "stable") -> None: - """配置 Rust 镜像源 (设置环境变量 + 写入 cargo config + 创建 sccache 目录). - - 设置 ``RUSTUP_DIST_SERVER`` / ``RUSTUP_UPDATE_ROOT`` / ``RUST_SCCACHE_DIR`` - 等环境变量, 写入 ``~/.cargo/config.toml``, 并创建 sccache 缓存目录. - - Parameters - ---------- - mirror : str - 镜像源名称, 见 :data:`_RUSTUP_MIRRORS` - version : str - Rust 版本 (未使用, 保留以与原 envdev 参数对齐) - """ - del version # 兼容旧参数, 实际安装由独立 job 处理 - - if mirror not in _RUSTUP_MIRRORS: - print(f"未知 Rust 镜像源: {mirror}") - return - - mirrors = _RUSTUP_MIRRORS[mirror] - os.environ["RUSTUP_DIST_SERVER"] = mirrors["RUSTUP_DIST_SERVER"] - os.environ["RUSTUP_UPDATE_ROOT"] = mirrors["RUSTUP_UPDATE_ROOT"] - os.environ["RUST_SCCACHE_DIR"] = str(_RUST_SCCACHE_DIR) - os.environ["RUST_SCCACHE_CACHE_SIZE"] = _RUST_SCCACHE_CACHE_SIZE - - _RUST_SCCACHE_DIR.mkdir(parents=True, exist_ok=True) - - config_path = Path.home() / ".cargo" / "config.toml" - config_path.parent.mkdir(parents=True, exist_ok=True) - registry = mirrors["TOML_REGISTRY"] - content = ( - f"\n[source.crates-io]\nreplace-with = '{mirror}'\n\n" - f'[source.{mirror}]\nregistry = "sparse+{registry}"\n\n' - f'[registries.{mirror}]\nindex = "sparse+{registry}"\n' - ) - config_path.write_text(content, encoding="utf-8") - print(f"Rust 镜像源已配置: {mirror} -> {config_path}") - - -# ============================================================================ -# dockercmd 函数 -# ============================================================================ - -_DOCKER_MIRROR_TENCENT: str = "ccr.ccs.tencentyun.com" - - -@px.register_fn -def docker_login_tencent(username: str = "") -> None: - """登录腾讯云 Docker 镜像仓库. - - Parameters - ---------- - username : str - Docker 用户名 (为空时由 docker 交互式提示输入) - """ - user = username or getpass.getuser() - subprocess.run(["docker", "login", "--username", user, _DOCKER_MIRROR_TENCENT], check=False) - print(f"已尝试登录腾讯云镜像仓库 (用户: {user})") - - -# ============================================================================ -# envdev Linux 专用函数 -# ============================================================================ - -_QT_LIBS: list[str] = [ - "build-essential", - "libgl1", - "libegl1", - "libglib2.0-0", - "libfontconfig1", - "libfreetype6", - "libxkbcommon0", - "libdbus-1-3", - "libxcb-xinerama0", - "libxcb-icccm4", - "libxcb-image0", - "libxcb-keysyms1", - "libxcb-randr0", - "libxcb-render-util0", - "libxcb-shape0", - "libxcb-xfixes0", - "libxcb-cursor0", -] - -_CHINESE_FONTS: list[str] = [ - "fonts-noto-cjk", - "fonts-wqy-microhei", - "fonts-wqy-zenhei", - "fonts-noto-color-emoji", -] - -_DOWNLOAD_MIRROR_SCRIPT: str = "curl -sSL https://linuxmirrors.cn/main.sh -o /tmp/linuxmirrors.sh" -_INSTALL_MIRROR_SCRIPT: str = "sudo bash /tmp/linuxmirrors.sh" - -_RUSTUP_DOWNLOAD_URL_LINUX: str = "https://mirrors.aliyun.com/repo/rust/rustup-init.sh" -_RUSTUP_DOWNLOAD_URL_WINDOWS: str = "https://static.rust-lang.org/rustup/dist/x86_64-pc-windows-msvc/rustup-init.exe" - - -@px.register_fn -def setup_linux_system_mirror() -> None: - """下载并安装 Linux 系统镜像源 (仅 Linux, 已配置国内镜像时跳过). - - 检查 ``/etc/apt/sources.list`` 与 ``/etc/apt/sources.list.d/ubuntu.sources`` - 是否已配置国内镜像, 已配置则跳过; 未配置则下载并执行 linuxmirrors 脚本. - """ - if not Constants.IS_LINUX: - print("setup_linux_system_mirror: 仅在 Linux 上执行") - return - - apt_files = ["/etc/apt/sources.list", "/etc/apt/sources.list.d/ubuntu.sources"] - mirror_keys = list(_PIP_INDEX_URLS.keys()) - already_configured = False - for apt_file in apt_files: - try: - content = Path(apt_file).read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - continue - if any(mirror in content for mirror in mirror_keys): - already_configured = True - break - - if already_configured: - print("已配置国内镜像源, 跳过系统镜像配置") - return - - print("下载 linuxmirrors 脚本...") - subprocess.run(_DOWNLOAD_MIRROR_SCRIPT, shell=True, check=False) - print("安装 linuxmirrors...") - subprocess.run(_INSTALL_MIRROR_SCRIPT, shell=True, check=False) - - -@px.register_fn -def install_linux_qt_libs() -> None: - """安装 Qt 依赖库 (仅 Linux).""" - if not Constants.IS_LINUX: - print("install_linux_qt_libs: 仅在 Linux 上执行") - return - - subprocess.run(["sudo", "apt", "install", "-y", *_QT_LIBS], check=False) - print("Qt 依赖库安装完成") - - -@px.register_fn -def install_linux_fonts() -> None: - """安装中文字体 (仅 Linux).""" - if not Constants.IS_LINUX: - print("install_linux_fonts: 仅在 Linux 上执行") - return - - subprocess.run(["sudo", "apt", "install", "-y", *_CHINESE_FONTS], check=False) - print("中文字体安装完成") - - -@px.register_fn -def install_linux_docker() -> None: - """安装 Docker (仅 Linux).""" - if not Constants.IS_LINUX: - print("install_linux_docker: 仅在 Linux 上执行") - return - - subprocess.run(["sudo", "apt", "install", "-y", "docker-compose-v2"], check=False) - subprocess.run(["sudo", "usermod", "-aG", "docker", getpass.getuser()], check=False) - print("Docker 安装完成 (需重新登录以生效 docker 用户组)") - - -@px.register_fn -def download_rustup_script() -> None: - """下载 Rustup 安装脚本 (跨平台, 已安装 rustup 时跳过). - - Linux 下载 ``rustup-init.sh``, Windows 下载 ``rustup-init.exe``. - """ - if shutil.which("rustup") is not None: - print("rustup 已安装, 跳过下载") - return - - if Constants.IS_WINDOWS: - print("下载 rustup-init.exe...") - subprocess.run( - [ - "powershell", - "-Command", - "Invoke-WebRequest", - "-Uri", - _RUSTUP_DOWNLOAD_URL_WINDOWS, - "-OutFile", - "rustup-init.exe", - ], - check=False, - ) - else: - print("下载 rustup-init.sh...") - subprocess.run( - ["curl", "-fsSL", _RUSTUP_DOWNLOAD_URL_LINUX, "-o", "rustup-init.sh"], - check=False, - ) - - -@px.register_fn -def install_rust_toolchain(version: str = "stable") -> None: - """安装 Rust 工具链 (rustup 未安装时跳过). - - Parameters - ---------- - version : str - Rust 版本: ``stable`` / ``nightly`` / ``beta`` (默认: ``stable``) - """ - if shutil.which("rustup") is None: - print("rustup 未安装, 跳过工具链安装") - return - - subprocess.run(["rustup", "toolchain", "install", version], check=False) - print(f"Rust 工具链 {version} 安装完成") diff --git a/src/pyflowx/ops/files.py b/src/pyflowx/ops/files.py deleted file mode 100644 index 5c3b169..0000000 --- a/src/pyflowx/ops/files.py +++ /dev/null @@ -1,327 +0,0 @@ -"""文件类函数模块. - -聚合文件日期处理、文件等级重命名、文件夹备份、文件夹压缩工具的可复用函数. -所有公共函数通过 ``@px.register_fn`` 注册, 供 YAML 任务编排引用. -""" - -from __future__ import annotations - -import re -import shutil -import time -import zipfile -from pathlib import Path - -import pyflowx as px - -__all__ = [ - "BRACKETS", - "DATE_PATTERN", - "IGNORE_DIRS", - "IGNORE_EXT", - "IGNORE_FILES", - "LEVELS", - "SEP", - "add_date_prefix", - "archive_folder", - "backup_folder", - "folderback_default", - "folderzip_default", - "get_file_timestamp", - "process_file_date", - "process_file_level", - "process_files_date", - "process_files_level", - "remove_date_prefix", - "remove_dump", - "remove_marks", - "zip_folders", - "zip_target", -] - -# ============================================================================ -# filedate 配置 -# ============================================================================ - -DATE_PATTERN = re.compile(r"(20|19)\d{2}[-_#.~]?((0[1-9])|(1[012]))[-_#.~]?((0[1-9])|([12]\d)|(3[01]))[-_#.~]?") -SEP = "_" - -# ============================================================================ -# filelevel 配置 -# ============================================================================ - -LEVELS: dict[str, str] = { - "0": "", - "1": "PUB,NOR", - "2": "INT", - "3": "CON", - "4": "CLA", -} - -BRACKETS: tuple[str, str] = (" ([_(【-", " )]_）】") - -# ============================================================================ -# folderzip 配置 -# ============================================================================ - -IGNORE_DIRS: list[str] = [".git", ".idea", ".vscode", "__pycache__"] -IGNORE_FILES: list[str] = [".gitignore"] -IGNORE: list[str] = [*IGNORE_DIRS, *IGNORE_FILES] -IGNORE_EXT: list[str] = [".zip", ".rar", ".7z", ".tar", ".gz"] - - -# ============================================================================ -# filedate 函数 -# ============================================================================ - - -@px.register_fn -def get_file_timestamp(filepath: Path) -> str: - """获取文件时间戳.""" - modified_time = filepath.stat().st_mtime - created_time = filepath.stat().st_ctime - return time.strftime("%Y%m%d", time.localtime(max((modified_time, created_time)))) - - -@px.register_fn -def remove_date_prefix(filepath: Path) -> Path: - """移除文件日期前缀.""" - stem = filepath.stem - new_stem = DATE_PATTERN.sub("", stem) - if new_stem != stem: - new_path = filepath.with_name(new_stem + filepath.suffix) - filepath.rename(new_path) - return new_path - return filepath - - -@px.register_fn -def add_date_prefix(filepath: Path) -> Path: - """添加文件日期前缀.""" - timestamp = get_file_timestamp(filepath) - stem = filepath.stem - new_stem = f"{timestamp}{SEP}{stem}" - new_path = filepath.with_name(new_stem + filepath.suffix) - if new_path != filepath: - filepath.rename(new_path) - return new_path - return filepath - - -@px.register_fn -def process_file_date(filepath: Path, clear: bool = False) -> None: - """处理单个文件的日期前缀. - - Parameters - ---------- - filepath : Path - 文件路径 - clear : bool - 是否清除日期前缀 - """ - if clear: - remove_date_prefix(filepath) - else: - new_path = remove_date_prefix(filepath) - add_date_prefix(new_path) - - -@px.register_fn -def process_files_date(targets: list[Path], clear: bool = False) -> None: - """批量处理文件日期前缀. - - Parameters - ---------- - targets : list[Path] - 文件路径列表 - clear : bool - 是否清除日期前缀 - """ - for target in targets: - if target.exists() and not target.name.startswith("."): - process_file_date(target, clear) - - -# ============================================================================ -# filelevel 函数 -# ============================================================================ - - -@px.register_fn -def remove_marks(stem: str, marks: list[str]) -> str: - """从文件名主干中移除所有标记.""" - left_brackets, right_brackets = BRACKETS - for mark in marks: - pos = 0 - while True: - pos = stem.find(mark, pos) - if pos == -1: - break - b, e = pos - 1, pos + len(mark) - if b >= 0 and e < len(stem) and stem[b] in left_brackets and stem[e] in right_brackets: - stem = stem[:b] + stem[e + 1 :] - else: - pos = e - return stem - - -@px.register_fn -def process_file_level(filepath: Path, level: int = 0) -> None: - """处理单个文件的等级标记. - - Parameters - ---------- - filepath : Path - 文件路径 - level : int - 文件等级 (0-4), 0 用于清除等级 - """ - if not (0 <= level < len(LEVELS)): - print(f"无效的等级 {level}, 必须在 0 和 {len(LEVELS) - 1} 之间") - return - - if not filepath.exists(): - print(f"文件不存在: {filepath}") - return - - filestem = filepath.stem - original_stem = filestem - - for level_names in LEVELS.values(): - if level_names: - filestem = remove_marks(filestem, level_names.split(",")) - - for digit in map(str, range(1, 10)): - filestem = remove_marks(filestem, [digit]) - - if level > 0: - levelstr = LEVELS.get(str(level), "").split(",")[0] - if levelstr: - filestem = f"{filestem}({levelstr})" - - if filestem != original_stem: - new_path = filepath.with_name(filestem + filepath.suffix) - filepath.rename(new_path) - print(f"重命名: {filepath} -> {new_path}") - - -@px.register_fn -def process_files_level(targets: list[Path], level: int = 0) -> None: - """批量处理文件等级标记. - - Parameters - ---------- - targets : list[Path] - 文件路径列表 - level : int - 文件等级 (0-4) - """ - for target in targets: - process_file_level(target, level) - - -# ============================================================================ -# folderback 函数 -# ============================================================================ - - -@px.register_fn -def remove_dump(src: Path, dst: Path, max_zip: int) -> None: - """递归删除旧的备份 zip 文件.""" - zip_paths = [filepath for filepath in dst.rglob("*.zip") if src.stem in str(filepath)] - zip_files = sorted(zip_paths, key=lambda fn: str(fn)[-19:-4]) - if len(zip_files) > max_zip: - zip_files[0].unlink() - remove_dump(src, dst, max_zip) - - -@px.register_fn -def zip_target(src: Path, dst: Path, max_zip: int) -> None: - """将单个文件或文件夹压缩为 zip 文件.""" - files = [str(_) for _ in src.rglob("*")] - timestamp = time.strftime("_%Y%m%d_%H%M%S") - target_path = dst / (src.stem + timestamp + ".zip") - - with zipfile.ZipFile(target_path, "w") as zip_file: - for file in files: - zip_file.write(file, arcname=file.replace(str(src.parent), "")) - - remove_dump(src, dst, max_zip) - print(f"备份完成: {target_path}") - - -@px.register_fn -def backup_folder(src: str, dst: str, max_zip: int = 5) -> None: - """备份文件夹. - - Parameters - ---------- - src : str - 源文件夹路径 - dst : str - 目标文件夹路径 - max_zip : int - 最大备份数量 - """ - src_path = Path(src) - dst_path = Path(dst) - - if not src_path.exists(): - print(f"源文件夹不存在: {src_path}") - return - - if not dst_path.exists(): - dst_path.mkdir(parents=True, exist_ok=True) - print(f"创建目标文件夹: {dst_path}") - - zip_target(src_path, dst_path, max_zip) - - -@px.register_fn("folderback_default") -def folderback_default() -> None: - """备份当前目录到 ./backup.""" - backup_folder(".", "./backup", 5) - - -# ============================================================================ -# folderzip 函数 -# ============================================================================ - - -@px.register_fn -def archive_folder(folder: Path) -> None: - """压缩单个文件夹.""" - shutil.make_archive( - str(folder.with_name(folder.name)), - format="zip", - base_dir=folder, - ) - print(f"压缩完成: {folder.name}.zip") - - -@px.register_fn -def zip_folders(cwd: str = ".") -> None: - """压缩目录下的所有文件夹. - - Parameters - ---------- - cwd : str - 工作目录 - """ - cwd_path = Path(cwd) - if not cwd_path.exists(): - print(f"目录不存在: {cwd_path}") - return - - dirs: list[Path] = [ - e for e in cwd_path.iterdir() if e.is_dir() and e.name not in IGNORE_DIRS and e.suffix not in IGNORE_EXT - ] - - for dir_path in dirs: - archive_folder(dir_path) - - -@px.register_fn("folderzip_default") -def folderzip_default() -> None: - """压缩当前目录下的所有文件夹.""" - zip_folders(".") diff --git a/src/pyflowx/ops/llm.py b/src/pyflowx/ops/llm.py deleted file mode 100644 index d123f15..0000000 --- a/src/pyflowx/ops/llm.py +++ /dev/null @@ -1,117 +0,0 @@ -"""LLM 工具类函数模块. - -聚合 ModelScope 下载 (msdownload) 与 SGLang 本地模型服务 (sglang) 的可复用函数. -所有公共函数通过 ``@px.register_fn`` 注册, 供 YAML 任务编排引用. -""" - -from __future__ import annotations - -import shutil -import subprocess -from pathlib import Path - -import pyflowx as px -from pyflowx.conditions import Constants - -__all__ = [ - "install_sglang", - "msdownload_run", - "run_sglang", -] - - -@px.register_fn -def msdownload_run(name: str, target_type: str = "model", download_dir: str | None = None) -> None: - """从 ModelScope 下载模型/数据集/空间. - - Parameters - ---------- - name : str - 目标名称 (如: ``Qwen/Qwen2.5-Coder-32B-Instruct``) - target_type : str - 目标类型: ``model`` / ``dataset`` / ``space`` (默认: ``model``) - download_dir : str | None - 下载目录; 为 None 时默认 ``~/.models/`` - """ - if not name: - print("msdownload: name 不能为空") - return - - if download_dir: - out_dir = Path(download_dir) - else: - out_dir = Path.home() / ".models" / name.rsplit("/", 1)[-1] - out_dir.mkdir(parents=True, exist_ok=True) - - cmd = ["uvx", "modelscope", "download", f"--{target_type}", name, "--local_dir", str(out_dir)] - print(f"下载 {target_type}: {name} -> {out_dir}") - subprocess.run(cmd, check=False) - - -@px.register_fn -def install_sglang() -> None: - """安装 sglang (若未安装). - - 通过 ``shutil.which`` 检测 sglang 是否已安装, 未安装时执行 ``uv install sglang[all]``. - """ - if shutil.which("sglang") is not None: - print("sglang 已安装, 跳过安装步骤") - return - - print("正在安装 sglang[all]...") - subprocess.run(["uv", "install", "sglang[all]"], check=False) - - -@px.register_fn -def run_sglang( - model: str = "~/.models/Qwen2.5-Coder-32B-Instruct-AWQ", - port: int = 8000, - ctx_len: int = 32768, - mem_fraction: float = 0.75, - host: str = "0.0.0.0", - log_level: str = "info", -) -> None: - """启动 SGLang 本地模型服务. - - Parameters - ---------- - model : str - 模型路径 (默认: ``~/.models/Qwen2.5-Coder-32B-Instruct-AWQ``) - port : int - 服务端口 (默认: 8000) - ctx_len : int - 最大上下文长度 (默认: 32768) - mem_fraction : float - 显存占比 0-1 (默认: 0.75) - host : str - 主机地址 (默认: 0.0.0.0) - log_level : str - 日志级别 (默认: info) - """ - model_dir = Path(model).expanduser() - if not model_dir.exists(): - print(f"模型目录不存在: {model_dir}") - return - - python_bin = "python" if Constants.IS_WINDOWS else "python3" - cmd = [ - python_bin, - "-m", - "sglang.launch_server", - "--model-path", - str(model_dir), - "--host", - host, - "--port", - str(port), - "--mem-fraction-static", - str(mem_fraction), - "--context-length", - str(ctx_len), - "--tool-call-parser", - "qwen", - "--log-level", - log_level, - ] - print(f"启动 SGLang: {model_dir} (port={port}, ctx={ctx_len}, mem={mem_fraction})") - subprocess.run(cmd, check=False) diff --git a/src/pyflowx/ops/media.py b/src/pyflowx/ops/media.py deleted file mode 100644 index d063599..0000000 --- a/src/pyflowx/ops/media.py +++ /dev/null @@ -1,498 +0,0 @@ -"""媒体类函数模块. - -聚合 PDF 工具 (pdftool) 和截图工具 (screenshot) 的可复用函数. -所有公共函数通过 ``@px.register_fn`` 注册, 供 YAML 任务编排引用. -""" - -from __future__ import annotations - -import subprocess -from datetime import datetime -from pathlib import Path - -import pyflowx as px -from pyflowx.conditions import Constants - -__all__ = [ - "DEFAULT_PASSWORD", - "DEFAULT_QUALITY", - "PDF_SUFFIX", - "get_screenshot_path", - "pdf_add_watermark", - "pdf_compress", - "pdf_crop", - "pdf_decrypt", - "pdf_encrypt", - "pdf_extract_images", - "pdf_extract_text", - "pdf_info", - "pdf_merge", - "pdf_ocr", - "pdf_reorder", - "pdf_repair", - "pdf_rotate", - "pdf_split", - "pdf_to_images", - "take_screenshot_area", - "take_screenshot_full", -] - -try: - import fitz # PyMuPDF - - HAS_PYMUPDF = True -except ImportError: - HAS_PYMUPDF = False - -try: - import pypdf - - HAS_PYPDF = True -except ImportError: - HAS_PYPDF = False - - -# ============================================================================ -# 配置 -# ============================================================================ - -PDF_SUFFIX = ".pdf" -DEFAULT_QUALITY = 75 -DEFAULT_PASSWORD = "" - - -# ============================================================================ -# PDF 函数 -# ============================================================================ - - -@px.register_fn -def pdf_merge(input_paths: list[Path], output_path: Path) -> None: - """合并多个 PDF 文件.""" - if not HAS_PYPDF: - print("未安装 pypdf 库, 请安装: pip install pypdf") - return - - writer = pypdf.PdfWriter() - for input_path in input_paths: - if input_path.exists(): - reader = pypdf.PdfReader(str(input_path)) - for page in reader.pages: - writer.add_page(page) - - output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, "wb") as f: - writer.write(f) - - print(f"合并完成: {output_path}") - - -@px.register_fn -def pdf_split(input_path: Path, output_dir: Path) -> None: - """拆分 PDF 文件为单页.""" - if not HAS_PYPDF: - print("未安装 pypdf 库, 请安装: pip install pypdf") - return - - reader = pypdf.PdfReader(str(input_path)) - output_dir.mkdir(parents=True, exist_ok=True) - - for i, page in enumerate(reader.pages): - writer = pypdf.PdfWriter() - writer.add_page(page) - output_file = output_dir / f"{input_path.stem}_page_{i + 1}.pdf" - with open(output_file, "wb") as f: - writer.write(f) - - print(f"拆分完成: {output_dir}") - - -@px.register_fn -def pdf_compress(input_path: Path, output_path: Path) -> None: - """压缩 PDF 文件.""" - if not HAS_PYMUPDF: - print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF") - return - - doc = fitz.open(str(input_path)) - output_path.parent.mkdir(parents=True, exist_ok=True) - doc.save(str(output_path), garbage=4, deflate=True, clean=True) - doc.close() - - original_size = input_path.stat().st_size - new_size = output_path.stat().st_size - ratio = (1 - new_size / original_size) * 100 - print(f"压缩完成: {output_path} (缩小 {ratio:.1f}%)") - - -@px.register_fn -def pdf_encrypt(input_path: Path, output_path: Path, password: str) -> None: - """加密 PDF 文件.""" - if not HAS_PYPDF: - print("未安装 pypdf 库, 请安装: pip install pypdf") - return - - reader = pypdf.PdfReader(str(input_path)) - writer = pypdf.PdfWriter() - - for page in reader.pages: - writer.add_page(page) - - writer.encrypt(user_password=password, owner_password=password, use_128bit=True) - output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, "wb") as f: - writer.write(f) - - print(f"加密完成: {output_path}") - - -@px.register_fn -def pdf_decrypt(input_path: Path, output_path: Path, password: str) -> None: - """解密 PDF 文件.""" - if not HAS_PYPDF: - print("未安装 pypdf 库, 请安装: pip install pypdf") - return - - reader = pypdf.PdfReader(str(input_path)) - if reader.is_encrypted: - reader.decrypt(password) - - writer = pypdf.PdfWriter() - for page in reader.pages: - writer.add_page(page) - - output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, "wb") as f: - writer.write(f) - - print(f"解密完成: {output_path}") - - -@px.register_fn -def pdf_extract_text(input_path: Path, output_path: Path) -> None: - """提取 PDF 文本.""" - if not HAS_PYMUPDF: - print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF") - return - - doc = fitz.open(str(input_path)) - text = "" - for page in doc: - text += str(page.get_text()) + "\n\n" - doc.close() - - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(text, encoding="utf-8") - print(f"文本提取完成: {output_path}") - - -@px.register_fn -def pdf_extract_images(input_path: Path, output_dir: Path) -> None: - """提取 PDF 图片.""" - if not HAS_PYMUPDF: - print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF") - return - - doc = fitz.open(str(input_path)) - output_dir.mkdir(parents=True, exist_ok=True) - - image_count = 0 - # pyrefly: ignore [bad-argument-type] - for page_num, page in enumerate(doc): - images = page.get_images(full=True) - for img_idx, img in enumerate(images): - xref = img[0] - base_image = doc.extract_image(xref) - image_data = base_image["image"] - image_ext = base_image["ext"] - image_path = output_dir / f"page_{page_num + 1}_img_{img_idx + 1}.{image_ext}" - image_path.write_bytes(image_data) - image_count += 1 - - doc.close() - print(f"图片提取完成: {output_dir} (共 {image_count} 张)") - - -@px.register_fn -def pdf_add_watermark(input_path: Path, output_path: Path, text: str = "CONFIDENTIAL") -> None: - """添加 PDF 水印.""" - if not HAS_PYMUPDF: - print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF") - return - - doc = fitz.open(str(input_path)) - for page in doc: - rect = page.rect - text_width = fitz.get_text_length(text, fontsize=48) - x = (rect.width - text_width) / 2 - y = rect.height / 2 - page.insert_text((x, y), text, fontsize=48, rotate=45, color=(0, 0, 0)) - - output_path.parent.mkdir(parents=True, exist_ok=True) - doc.save(str(output_path)) - doc.close() - print(f"水印添加完成: {output_path}") - - -@px.register_fn -def pdf_rotate(input_path: Path, output_path: Path, rotation: int = 90) -> None: - """旋转 PDF 页面.""" - if not HAS_PYMUPDF: - print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF") - return - - doc = fitz.open(str(input_path)) - for page in doc: - page.set_rotation(rotation) - - output_path.parent.mkdir(parents=True, exist_ok=True) - doc.save(str(output_path)) - doc.close() - print(f"旋转完成: {output_path}") - - -@px.register_fn -def pdf_crop(input_path: Path, output_path: Path, margins: tuple[int, int, int, int]) -> None: - """裁剪 PDF 页面.""" - if not HAS_PYMUPDF: - print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF") - return - - doc = fitz.open(str(input_path)) - left, top, right, bottom = margins - - for page in doc: - rect = page.rect - new_rect = fitz.Rect( - rect.x0 + left, - rect.y0 + top, - rect.x1 - right, - rect.y1 - bottom, - ) - page.set_cropbox(new_rect) - - output_path.parent.mkdir(parents=True, exist_ok=True) - doc.save(str(output_path)) - doc.close() - print(f"裁剪完成: {output_path}") - - -@px.register_fn -def pdf_info(input_path: Path) -> None: - """显示 PDF 信息.""" - if not HAS_PYMUPDF: - print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF") - return - - doc = fitz.open(str(input_path)) - print(f"文件: {input_path}") - print(f"页数: {doc.page_count}") - # pyrefly: ignore [missing-attribute] - print(f"标题: {doc.metadata.get('title', 'N/A')}") - # pyrefly: ignore [missing-attribute] - print(f"作者: {doc.metadata.get('author', 'N/A')}") - # pyrefly: ignore [missing-attribute] - print(f"创建日期: {doc.metadata.get('creationDate', 'N/A')}") - # pyrefly: ignore [missing-attribute] - print(f"修改日期: {doc.metadata.get('modDate', 'N/A')}") - print(f"文件大小: {input_path.stat().st_size / 1024:.1f} KB") - doc.close() - - -@px.register_fn -def pdf_ocr(input_path: Path, output_path: Path, lang: str = "chi_sim+eng") -> None: - """PDF OCR 识别.""" - try: - import pytesseract - from PIL import Image - except ImportError: - print("未安装 OCR 相关库, 请安装: pip install pytesseract pillow") - return - - if not HAS_PYMUPDF: - print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF") - return - - doc = fitz.open(str(input_path)) - new_doc = fitz.open() - - for page in doc: - pix = page.get_pixmap() - img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples) - ocr_text = pytesseract.image_to_string(img, lang=lang) - - new_page = new_doc.new_page(width=page.rect.width, height=page.rect.height) - new_page.insert_image(new_page.rect, pixmap=pix) - text_rect = fitz.Rect(0, 0, page.rect.width, page.rect.height) - # pyrefly: ignore [bad-argument-type] - new_page.insert_textbox(text_rect, ocr_text, fontname="china-ss", fontsize=11) - - output_path.parent.mkdir(parents=True, exist_ok=True) - new_doc.save(str(output_path)) - new_doc.close() - doc.close() - print(f"OCR 识别完成: {output_path}") - - -@px.register_fn -def pdf_reorder(input_path: Path, output_path: Path, order: list[int]) -> None: - """重排 PDF 页面顺序.""" - if not HAS_PYPDF: - print("未安装 pypdf 库, 请安装: pip install pypdf") - return - - reader = pypdf.PdfReader(str(input_path)) - writer = pypdf.PdfWriter() - - for page_num in order: - if 0 <= page_num < len(reader.pages): - writer.add_page(reader.pages[page_num]) - - output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, "wb") as f: - writer.write(f) - - print(f"重排完成: {output_path}") - - -@px.register_fn -def pdf_to_images(input_path: Path, output_dir: Path, dpi: int = 300) -> None: - """PDF 转图片.""" - if not HAS_PYMUPDF: - print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF") - return - - doc = fitz.open(str(input_path)) - output_dir.mkdir(parents=True, exist_ok=True) - - # pyrefly: ignore [bad-argument-type] - for page_num, page in enumerate(doc): - pix = page.get_pixmap(dpi=dpi) - image_path = output_dir / f"{input_path.stem}_page_{page_num + 1}.png" - pix.save(str(image_path)) - - doc.close() - print(f"转换完成: {output_dir}") - - -@px.register_fn -def pdf_repair(input_path: Path, output_path: Path) -> None: - """修复 PDF 文件.""" - if not HAS_PYMUPDF: - print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF") - return - - doc = fitz.open(str(input_path)) - output_path.parent.mkdir(parents=True, exist_ok=True) - doc.save(str(output_path), garbage=4, deflate=True, clean=True) - doc.close() - print(f"修复完成: {output_path}") - - -# ============================================================================ -# screenshot 函数 -# ============================================================================ - - -@px.register_fn -def get_screenshot_path(filename: str | None = None) -> Path: - """获取截图保存路径. - - Parameters - ---------- - filename : str | None - 文件名, 如果为 None 则自动生成 - - Returns - ------- - Path - 截图保存路径 - """ - if filename is None: - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - filename = f"screenshot_{timestamp}.png" - - screenshots_dir = Path.home() / "Pictures" / "screenshots" - screenshots_dir.mkdir(parents=True, exist_ok=True) - return screenshots_dir / filename - - -@px.register_fn -def take_screenshot_full(filename: str | None = None) -> None: - """全屏截图. - - Parameters - ---------- - filename : str | None - 文件名 - """ - output_path = get_screenshot_path(filename) - - if Constants.IS_WINDOWS: - ps_script = f""" -Add-Type -AssemblyName System.Windows.Forms -Add-Type -AssemblyName System.Drawing -$screen = [System.Windows.Forms.Screen]::PrimaryScreen -$bounds = $screen.Bounds -$bitmap = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height -$graphics = [System.Drawing.Graphics]::FromImage($bitmap) -$graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size) -$bitmap.Save('{output_path.as_posix()}') -$graphics.Dispose() -$bitmap.Dispose() -""" - subprocess.run(["powershell", "-Command", ps_script], check=True) - elif Constants.IS_MACOS: - subprocess.run(["screencapture", "-x", str(output_path)], check=True) - else: - try: - subprocess.run(["gnome-screenshot", "-f", str(output_path)], check=True) - except FileNotFoundError: - subprocess.run(["scrot", str(output_path)], check=True) - - print(f"截图已保存: {output_path}") - - -@px.register_fn -def take_screenshot_area(filename: str | None = None) -> None: - """区域截图. - - Parameters - ---------- - filename : str | None - 文件名 - """ - output_path = get_screenshot_path(filename) - - if Constants.IS_WINDOWS: - ps_script = f""" -Add-Type -AssemblyName System.Windows.Forms -Add-Type -AssemblyName System.Drawing -$form = New-Object System.Windows.Forms.Form -$form.WindowState = 'Maximized' -$form.FormBorderStyle = 'None' -$form.BackColor = [System.Drawing.Color]::FromArgb(1, 0, 0) -$form.Opacity = 0.5 -$form.TopMost = $true -$form.Show() -Start-Sleep -Milliseconds 100 -$screen = [System.Windows.Forms.Screen]::PrimaryScreen -$bounds = $screen.Bounds -$bitmap = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height -$graphics = [System.Drawing.Graphics]::FromImage($bitmap) -$graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size) -$form.Close() -$bitmap.Save('{output_path.as_posix()}') -$graphics.Dispose() -$bitmap.Dispose() -""" - subprocess.run(["powershell", "-Command", ps_script], check=True) - elif Constants.IS_MACOS: - subprocess.run(["screencapture", "-i", str(output_path)], check=True) - else: - try: - subprocess.run(["gnome-screenshot", "-a", "-f", str(output_path)], check=True) - except FileNotFoundError: - subprocess.run(["scrot", "-s", str(output_path)], check=True) - - print(f"截图已保存: {output_path}") diff --git a/src/pyflowx/ops/system.py b/src/pyflowx/ops/system.py deleted file mode 100644 index 6389dc9..0000000 --- a/src/pyflowx/ops/system.py +++ /dev/null @@ -1,568 +0,0 @@ -"""系统类函数模块. - -聚合 LS-DYNA 计算 (lscalc)、SSH 密钥部署 (sshcopyid)、Python 打包 (packtool)、 -重置图标缓存 (reset_icon_cache) 的可复用函数. 所有公共函数通过 ``@px.register_fn`` -注册, 供 YAML 任务编排引用. -""" - -from __future__ import annotations - -import os -import platform -import shutil -import subprocess -import sys -import urllib.request -import zipfile -from pathlib import Path - -import pyflowx as px -from pyflowx.conditions import Constants - -__all__ = [ - "DEFAULT_BUILD_DIR", - "DEFAULT_CACHE_DIR", - "DEFAULT_DIST_DIR", - "DEFAULT_INPUT_FILE", - "DEFAULT_LIB_DIR", - "DEFAULT_NCPU", - "IGNORE_PATTERNS", - "LS_DYNA_COMMANDS", - "check_ls_dyna_status", - "clean_build_dir", - "clear_screen_run", - "create_zip_package", - "get_ls_dyna_command", - "install_embed_python", - "pack_dependencies", - "pack_source", - "pack_wheel", - "reset_icon_cache_run", - "run_ls_dyna", - "run_ls_dyna_mpi", - "ssh_copy_id", - "taskkill_run", - "which_run", -] - -# ============================================================================ -# lscalc 配置 -# ============================================================================ - -LS_DYNA_COMMANDS: dict[str, list[str]] = { - "windows": ["ls-dyna_mpp", "i=input.k", "ncpu=4"], - "linux": ["ls-dyna_mpp", "i=input.k", "ncpu=8"], - "macos": ["ls-dyna_mpp", "i=input.k", "ncpu=4"], -} - -DEFAULT_INPUT_FILE: str = "input.k" -DEFAULT_NCPU: int = 4 - -# ============================================================================ -# packtool 配置 -# ============================================================================ - -DEFAULT_BUILD_DIR = ".pypack" -DEFAULT_DIST_DIR = "dist" -DEFAULT_LIB_DIR = "libs" -DEFAULT_CACHE_DIR = ".cache/pypack" - -IGNORE_PATTERNS = [ - "__pycache__", - "*.pyc", - "*.pyo", - ".git", - ".venv", - ".idea", - ".vscode", - "*.egg-info", - "dist", - "build", - ".pytest_cache", - ".tox", - ".mypy_cache", -] - - -# ============================================================================ -# lscalc 函数 -# ============================================================================ - - -@px.register_fn -def get_ls_dyna_command(input_file: str, ncpu: int) -> list[str]: - """获取 LS-DYNA 命令. - - Parameters - ---------- - input_file : str - 输入文件路径 - ncpu : int - CPU 核心数 - - Returns - ------- - list[str] - LS-DYNA 命令列表 - """ - if Constants.IS_WINDOWS or Constants.IS_MACOS: - return ["ls-dyna_mpp", f"i={input_file}", f"ncpu={ncpu}"] - else: - return ["ls-dyna_mpp", f"i={input_file}", f"ncpu={ncpu}"] - - -@px.register_fn -def run_ls_dyna(input_file: str, ncpu: int = DEFAULT_NCPU) -> None: - """运行 LS-DYNA 计算. - - Parameters - ---------- - input_file : str - 输入文件路径 - ncpu : int - CPU 核心数 - """ - input_path = Path(input_file) - if not input_path.exists(): - print(f"输入文件不存在: {input_path}") - return - - cmd = get_ls_dyna_command(input_file, ncpu) - try: - subprocess.run(cmd, check=True) - print(f"LS-DYNA 计算完成: {input_file}") - except FileNotFoundError: - print("未找到 ls-dyna_mpp 命令") - except subprocess.CalledProcessError as e: - print(f"LS-DYNA 计算失败: {e}") - - -@px.register_fn -def run_ls_dyna_mpi(input_file: str, ncpu: int = DEFAULT_NCPU) -> None: - """运行 LS-DYNA MPI 计算. - - Parameters - ---------- - input_file : str - 输入文件路径 - ncpu : int - CPU 核心数 - """ - input_path = Path(input_file) - if not input_path.exists(): - print(f"输入文件不存在: {input_path}") - return - - cmd = ["mpirun", "-np", str(ncpu), "ls-dyna_mpp", f"i={input_file}"] - try: - subprocess.run(cmd, check=True) - print(f"LS-DYNA MPI 计算完成: {input_file}") - except FileNotFoundError: - print("未找到 mpirun 或 ls-dyna_mpp 命令") - except subprocess.CalledProcessError as e: - print(f"LS-DYNA MPI 计算失败: {e}") - - -@px.register_fn -def check_ls_dyna_status() -> None: - """检查 LS-DYNA 进程状态.""" - try: - if Constants.IS_WINDOWS: - result = subprocess.run( - ["tasklist", "/fi", "imagename eq ls-dyna_mpp.exe"], - capture_output=True, - text=True, - check=True, - ) - print(result.stdout) - else: - result = subprocess.run( - ["pgrep", "-f", "ls-dyna"], - capture_output=True, - text=True, - check=False, - ) - if result.stdout.strip(): - print(f"运行中的 LS-DYNA 进程 PID: {result.stdout.strip()}") - else: - print("没有运行中的 LS-DYNA 进程") - except subprocess.CalledProcessError as e: - print(f"检查进程状态失败: {e}") - - -# ============================================================================ -# sshcopyid 函数 -# ============================================================================ - - -@px.register_fn -def ssh_copy_id( - hostname: str, - username: str, - password: str, - port: int = 22, - keypath: str = "~/.ssh/id_rsa.pub", - timeout: int = 30, -) -> None: - """将 SSH 公钥部署到远程服务器. - - Parameters - ---------- - hostname : str - 远程服务器主机名或 IP 地址 - username : str - 远程服务器用户名 - password : str - 远程服务器密码 - port : int - SSH 端口, 默认 22 - keypath : str - 公钥文件路径, 默认 ~/.ssh/id_rsa.pub - timeout : int - SSH 操作超时秒数, 默认 30 - """ - pub_key_path = Path(keypath).expanduser() - if not pub_key_path.exists(): - print(f"公钥文件不存在: {pub_key_path}") - sys.exit(1) - - pub_key = pub_key_path.read_text().strip() - - script = f"""mkdir -p ~/.ssh && chmod 700 ~/.ssh -cd ~/.ssh && touch authorized_keys && chmod 600 authorized_keys -grep -qF '{pub_key.split()[1]}' authorized_keys 2>/dev/null || echo '{pub_key}' >> authorized_keys""" - - try: - subprocess.run( - [ - "sshpass", - "-p", - password, - "ssh", - "-p", - str(port), - "-o", - "StrictHostKeyChecking=no", - "-o", - "UserKnownHostsFile=/dev/null", - "-o", - f"ConnectTimeout={timeout}", - f"{username}@{hostname}", - script, - ], - check=True, - timeout=timeout, - ) - print(f"SSH 密钥已部署到 {username}@{hostname}:{port}") - except FileNotFoundError: - print(f"未找到 sshpass 工具, 请手动执行: ssh-copy-id -p {port} {username}@{hostname}") - sys.exit(1) - except subprocess.TimeoutExpired: - print("SSH 连接超时") - sys.exit(1) - except subprocess.CalledProcessError as e: - print(f"SSH 执行失败: {e}") - sys.exit(1) - - -# ============================================================================ -# packtool 函数 -# ============================================================================ - - -@px.register_fn -def pack_source(project_dir: Path, output_dir: Path) -> None: - """打包项目源码. - - Parameters - ---------- - project_dir : Path - 项目目录 - output_dir : Path - 输出目录 - """ - output_dir.mkdir(parents=True, exist_ok=True) - - pyproject_file = project_dir / "pyproject.toml" - project_name = project_dir.name - - if pyproject_file.exists(): - try: - import tomllib - - content = pyproject_file.read_text(encoding="utf-8") - data = tomllib.loads(content) - project_name = data.get("project", {}).get("name", project_name) - except ImportError: - pass - - source_dir = output_dir / "src" / project_name - source_dir.mkdir(parents=True, exist_ok=True) - - src_subdir = project_dir / "src" - if src_subdir.exists(): - shutil.copytree( - src_subdir, - source_dir / "src", - ignore=shutil.ignore_patterns(*IGNORE_PATTERNS), - dirs_exist_ok=True, - ) - else: - for item in project_dir.iterdir(): - if item.name in IGNORE_PATTERNS or item.name.startswith("."): - continue - dst_item = source_dir / item.name - if item.is_dir(): - shutil.copytree( - item, - dst_item, - ignore=shutil.ignore_patterns(*IGNORE_PATTERNS), - dirs_exist_ok=True, - ) - else: - shutil.copy2(item, dst_item) - - print(f"源码打包完成: {source_dir}") - - -@px.register_fn -def pack_dependencies(lib_dir: Path, dependencies: list[str]) -> None: - """打包项目依赖. - - Parameters - ---------- - lib_dir : Path - 依赖库目录 - dependencies : list[str] - 依赖列表 - """ - lib_dir.mkdir(parents=True, exist_ok=True) - - if not dependencies: - print("没有依赖需要打包") - return - - cmd = [ - "pip", - "install", - "--target", - str(lib_dir), - "--no-compile", - "--no-warn-script-location", - ] - cmd.extend(dependencies) - - subprocess.run(cmd, check=True) - print(f"依赖打包完成: {lib_dir}") - - -@px.register_fn -def pack_wheel(project_dir: Path, output_dir: Path) -> None: - """打包项目为 wheel 文件. - - Parameters - ---------- - project_dir : Path - 项目目录 - output_dir : Path - 输出目录 - """ - output_dir.mkdir(parents=True, exist_ok=True) - - cmd = [ - "pip", - "wheel", - "--no-deps", - "--wheel-dir", - str(output_dir), - str(project_dir), - ] - - subprocess.run(cmd, check=True) - print(f"Wheel 打包完成: {output_dir}") - - -@px.register_fn -def install_embed_python(version: str, output_dir: Path) -> None: - """安装嵌入式 Python. - - Parameters - ---------- - version : str - Python 版本 (如: 3.10, 3.11) - output_dir : Path - 输出目录 - """ - output_dir.mkdir(parents=True, exist_ok=True) - - arch = platform.machine().lower() - if arch in ["x86_64", "amd64"]: - arch = "amd64" - elif arch in ["arm64", "aarch64"]: - arch = "arm64" - - version_map = { - "3.8": "3.8.10", - "3.9": "3.9.13", - "3.10": "3.10.11", - "3.11": "3.11.9", - "3.12": "3.12.4", - } - full_version = version_map.get(version, f"{version}.0") - - url = f"https://www.python.org/ftp/python/{full_version}/python-{full_version}-embed-{arch}.zip" - - cache_file = Path(DEFAULT_CACHE_DIR) / f"python-{full_version}-embed-{arch}.zip" - cache_file.parent.mkdir(parents=True, exist_ok=True) - - if not cache_file.exists(): - print(f"正在下载嵌入式 Python {full_version}...") - urllib.request.urlretrieve(url, cache_file) - print(f"下载完成: {cache_file}") - - with zipfile.ZipFile(cache_file, "r") as zf: - zf.extractall(output_dir) - - print(f"嵌入式 Python 安装完成: {output_dir}") - - -@px.register_fn -def create_zip_package(source_dir: Path, output_file: Path) -> None: - """创建 ZIP 打包文件. - - Parameters - ---------- - source_dir : Path - 源目录 - output_file : Path - 输出文件 - """ - output_file.parent.mkdir(parents=True, exist_ok=True) - - with zipfile.ZipFile(output_file, "w", zipfile.ZIP_DEFLATED) as zf: - for file in source_dir.rglob("*"): - if file.is_file(): - arcname = file.relative_to(source_dir) - zf.write(file, arcname) - - print(f"ZIP 打包完成: {output_file}") - - -@px.register_fn -def clean_build_dir(build_dir: Path) -> None: - """清理构建目录. - - Parameters - ---------- - build_dir : Path - 构建目录 - """ - if build_dir.exists(): - shutil.rmtree(build_dir) - print(f"清理完成: {build_dir}") - else: - print(f"目录不存在: {build_dir}") - - -# ============================================================================ -# reseticoncache 函数 -# ============================================================================ - - -@px.register_fn -def reset_icon_cache_run() -> None: - """重置 Windows 图标缓存. - - 执行流程: 杀掉 explorer → 删除 IconCache.db → 删除 iconcache* → 重启 explorer. - 仅在 Windows 上执行, 非 Windows 平台打印提示并跳过. - """ - if not Constants.IS_WINDOWS: - print("reset_icon_cache: 仅在 Windows 上支持") - return - - local_app_data = os.environ.get("LOCALAPPDATA", "") - if not local_app_data: - print("reset_icon_cache: LOCALAPPDATA 环境变量未设置") - return - - icon_cache_db = Path(local_app_data) / "IconCache.db" - explorer_cache_dir = Path(local_app_data) / "Microsoft" / "Windows" / "Explorer" - - print("正在终止 explorer 进程...") - subprocess.run(["taskkill", "/f", "/im", "explorer.exe"], check=False) - - if icon_cache_db.exists(): - print(f"删除图标缓存: {icon_cache_db}") - subprocess.run(["cmd", "/c", "del", "/a", "/q", str(icon_cache_db)], check=False) - - if explorer_cache_dir.exists(): - print(f"清理 Explorer 缓存: {explorer_cache_dir}") - subprocess.run( - ["cmd", "/c", "del", "/a", "/q", str(explorer_cache_dir / "iconcache*")], - check=False, - ) - - print("重启 explorer...") - subprocess.run(["cmd", "/c", "start", "explorer.exe"], check=False) - print("图标缓存已重置") - - -# ============================================================================ -# clearscreen / taskkill / which 函数 -# ============================================================================ - - -@px.register_fn -def clear_screen_run() -> None: - """清屏 (跨平台). - - Windows 调用 ``cls``, Linux/macOS 调用 ``clear``. - """ - cmd = ["cls"] if Constants.IS_WINDOWS else ["clear"] - subprocess.run(cmd, check=False) - - -@px.register_fn -def taskkill_run(process_names: list[str]) -> None: - """按名称终止进程 (跨平台). - - Windows 使用 ``taskkill /f /im *``, - Linux/macOS 使用 ``pkill -f *``. - - Parameters - ---------- - process_names : list[str] - 进程名称列表 (如: ``["chrome.exe", "python"]``) - """ - if Constants.IS_WINDOWS: - cmd_prefix: list[str] = ["taskkill", "/f", "/im"] - else: - cmd_prefix = ["pkill", "-f"] - - for name in process_names: - print(f"终止进程: {name}") - subprocess.run([*cmd_prefix, f"{name}*"], check=False) - - -@px.register_fn -def which_run(commands: list[str]) -> None: - """查找可执行命令路径 (跨平台). - - Windows 使用 ``where``, Linux/macOS 使用 ``which``. - 对每个命令打印 `` -> `` 或 `` -> 未找到``. - - Parameters - ---------- - commands : list[str] - 要查找的命令名称列表 - """ - which_cmd = "where" if Constants.IS_WINDOWS else "which" - - for cmd in commands: - result = subprocess.run([which_cmd, cmd], capture_output=True, text=True, check=False) - if result.returncode == 0: - # Windows 的 where 可能返回多行, 取第一个 - path = result.stdout.strip().split("\n")[0].strip() - print(f"{cmd} -> {path}") - else: - print(f"{cmd} -> 未找到") diff --git a/src/pyflowx/registry.py b/src/pyflowx/registry.py deleted file mode 100644 index aea0301..0000000 --- a/src/pyflowx/registry.py +++ /dev/null @@ -1,159 +0,0 @@ -"""函数注册表. - -提供全局函数注册机制, 供 YAML 任务编排通过 ``fn`` 字段引用 Python 函数. - -使用方式 --------- - import pyflowx as px - - @px.register_fn("pack_source") - def pack_source(project_dir, output_dir): - ... - - # YAML 中引用: - # jobs: - # pack: - # fn: pack_source - # args: ["./project", "./dist"] -""" - -from __future__ import annotations - -import sys -from typing import Any, Callable, TypeVar, overload - -if sys.version_info >= (3, 10): - from typing import ParamSpec -else: - from typing_extensions import ParamSpec # pragma: no cover - -__all__ = ["FnRegistry", "get_fn", "has_fn", "register_fn"] - -P = ParamSpec("P") -T = TypeVar("T") - -_REGISTRY: dict[str, Callable[..., Any]] = {} - - -@overload -def register_fn(name: Callable[P, T]) -> Callable[P, T]: ... - - -@overload -def register_fn(name: str | None = None) -> Callable[[Callable[P, T]], Callable[P, T]]: ... - - -def register_fn(name: str | Callable[..., Any] | None = None) -> Callable[..., Any]: - """装饰器：将函数注册到全局 registry. - - 支持两种用法:: - - @register_fn # 使用函数 __name__ 作为注册名 - def my_func(): ... - - @register_fn("custom") # 显式指定注册名 - def my_func(): ... - - Parameters - ---------- - name : str | Callable | None - 注册名或被装饰函数; 为 None 时使用函数 ``__name__`` - - Returns - ------- - Callable - 装饰器函数或被装饰函数 - - Raises - ------ - ValueError - 名称已注册或无法推断函数名 - """ - if callable(name): - fn = name - key = getattr(fn, "__name__", None) - if key is None: - raise ValueError("无法推断函数名, 请显式提供 name 参数") - if key in _REGISTRY: - raise ValueError(f"函数 {key!r} 已注册") - _REGISTRY[key] = fn - return fn - - def decorator(fn: Callable[P, T]) -> Callable[P, T]: - key = name if name is not None else getattr(fn, "__name__", None) - if key is None: - raise ValueError("无法推断函数名, 请显式提供 name 参数") - if key in _REGISTRY: - raise ValueError(f"函数 {key!r} 已注册") - _REGISTRY[key] = fn - return fn - - return decorator - - -def get_fn(name: str) -> Callable[..., Any]: - """按名称获取已注册的函数. - - Parameters - ---------- - name : str - 函数名 - - Returns - ------- - Callable - 已注册的函数 - - Raises - ------ - KeyError - 函数未注册 - """ - if name not in _REGISTRY: - raise KeyError(f"函数 {name!r} 未注册") - return _REGISTRY[name] - - -def has_fn(name: str) -> bool: - """检查函数是否已注册. - - Parameters - ---------- - name : str - 函数名 - - Returns - ------- - bool - 是否已注册 - """ - return name in _REGISTRY - - -class FnRegistry: - """函数注册表的面向对象访问接口.""" - - @staticmethod - def register(name: str | None = None) -> Callable[[Callable[..., T]], Callable[..., T]]: - """注册装饰器, 等价于 :func:`register_fn`.""" - return register_fn(name) - - @staticmethod - def get(name: str) -> Callable[..., Any]: - """获取已注册函数, 等价于 :func:`get_fn`.""" - return get_fn(name) - - @staticmethod - def has(name: str) -> bool: - """检查是否已注册, 等价于 :func:`has_fn`.""" - return has_fn(name) - - @staticmethod - def clear() -> None: - """清空注册表.""" - _REGISTRY.clear() - - @staticmethod - def names() -> list[str]: - """返回所有已注册函数名.""" - return list(_REGISTRY.keys()) diff --git a/src/pyflowx/yaml_loader.py b/src/pyflowx/yaml_loader.py deleted file mode 100644 index 69ce000..0000000 --- a/src/pyflowx/yaml_loader.py +++ /dev/null @@ -1,1095 +0,0 @@ -"""YAML 任务编排加载器. - -参考 GitHub Actions 风格, 从 YAML 文件构建任务图. -支持 jobs/needs/strategy.matrix/if 等核心 CI/CD 概念, 实现声明式串并行任务编排. - -Schema 概览 ------------ -:: - - strategy: thread # 图级默认策略 - defaults: # 图级默认值 - retry: {max_attempts: 3} - verbose: true - - jobs: - setup: # job_id 即任务名 - run: "git clone https://github.com/foo/bar" - - build: - needs: [setup] # 依赖列表 - cmd: ["python", "-m", "build"] - strategy: # 矩阵扇出 - matrix: - python: ["3.8", "3.9"] - if: "success()" # 条件 - timeout: 300 - retry: {max_attempts: 2, delay: 1.0} - cwd: ./build - env: {DEBUG: "1"} - verbose: true - continue-on-error: false - tags: [ci] - runs-on: linux - priority: 10 - concurrency-key: deploy_lock - - deploy: - needs: [build] - run: "twine upload dist/*" - if: "env.DEPLOY_TOKEN != ''" - allow-upstream-skip: true - -字段映射 -------- -- ``cmd`` / ``run``: ``cmd`` 为命令列表, ``run`` 为 shell 字符串 -- ``fn``: 引用 ``register_fn`` 注册的 Python 函数名, 配合 ``args``/``kwargs`` 传参 -- ``needs``: 对应 ``depends_on`` -- ``if``: 条件表达式, 支持 ``success()`` / ``always()`` / ``env.VAR`` / ``env.VAR == 'x'`` -- ``strategy.matrix``: 笛卡尔积展开为多个任务, 任务名追加 ``_{key}-{val}`` 后缀 -- ``runs-on``: 追加到 ``tags`` -- 其余字段名连字符转下划线后透传给 :class:`~pyflowx.task.TaskSpec` - -矩阵占位符 ----------- -矩阵值通过 ``${{ matrix.key }}`` 占位符注入 ``cmd`` / ``run`` / ``cwd`` / ``env`` / ``args`` / ``kwargs`` 中:: - - jobs: - test: - cmd: ["python{{ matrix.version }}", "-m", "pytest"] - strategy: - matrix: - version: ["3.8", "3.9"] - -变量占位符 ----------- -运行时变量通过 ``${VAR}`` 占位符注入, 变量由 ``load_yaml(path, variables={...})`` -或 ``parse_yaml_string(content, variables={...})`` 传入:: - - # Python - graph = px.load_yaml("pipeline.yaml", variables={"PROJECT_DIR": "./project"}) - - # YAML - jobs: - pack: - fn: pack_source - args: ["${PROJECT_DIR}", "./dist"] - -函数引用 ----------- -``fn`` 字段引用 ``@px.register_fn`` 注册的函数, 配合 ``args``/``kwargs`` 传参:: - - # Python - @px.register_fn("pack_source") - def pack_source(project_dir, output_dir): - ... - - # YAML - jobs: - pack: - fn: pack_source - args: ["./project", "./dist"] - needs: [clean] -""" - -from __future__ import annotations - -__all__ = ["YamlLoadError", "build_cli_parser", "load_yaml", "parse_yaml_string", "run_cli", "run_yaml"] - -import itertools -import re -import sys -from pathlib import Path -from typing import TYPE_CHECKING, Any, Iterable, Mapping, Sequence, cast - -import pyflowx as px -from pyflowx.conditions import BuiltinConditions -from pyflowx.executors import Strategy -from pyflowx.registry import get_fn as _get_fn -from pyflowx.task import Condition, RetryPolicy, TaskSpec - -if TYPE_CHECKING: - from pyflowx.executors import EventCallback, RunReport - from pyflowx.storage import StateBackend - -# 导入 ops 模块以注册 CLI 工具函数, 使 YAML fn 字段可引用 -# 使用 try/except 避免最小安装场景下的导入失败 -try: - from pyflowx.ops import dev as _dev # noqa: F401 - from pyflowx.ops import files as _files # noqa: F401 - from pyflowx.ops import media as _media # noqa: F401 - from pyflowx.ops import system as _system # noqa: F401 -except ImportError: # pragma: no cover - pass - -# 矩阵占位符: ${{ matrix.key }} -_MATRIX_PLACEHOLDER = re.compile(r"\$\{\{\s*matrix\.(\w+)\s*\}\}") - -# 变量占位符: ${VAR} -_VARIABLE_PLACEHOLDER = re.compile(r"\$\{(\w+)\}") - -# 变量全匹配: 字符串完全等于 ${VAR} (用于返回变量实际值而非字符串化) -_VARIABLE_FULL_MATCH = re.compile(r"^\$\{(\w+)\}$") - -# 条件表达式 -_ENV_VAR_EXISTS_RE = re.compile(r"^env\.(\w+)$") -_ENV_VAR_EQUALS_RE = re.compile(r"^env\.(\w+)\s*==\s*['\"]([^'\"]*)['\"]$") -_ENV_VAR_NOT_EQUALS_RE = re.compile(r"^env\.(\w+)\s*!=\s*['\"]([^'\"]*)['\"]$") - -# 支持的 job 字段名（连字符形式） -_JOB_FIELDS_HYPHEN = frozenset( - { - "continue-on-error", - "skip-if-missing", - "allow-upstream-skip", - "concurrency-key", - "runs-on", - } -) - - -class YamlLoadError(px.PyFlowXError): - """YAML 加载错误.""" - - -def _aggregate_noop() -> None: - """聚合 job 的占位函数. - - 方向 B: 有 ``needs`` 无 ``cmd/run/fn`` 的 job 仅作为依赖聚合点, - 执行时调用此 no-op 函数, 不产生副作用, 返回 ``None``. - """ - return None - - -def load_yaml(path: str | Path, variables: Mapping[str, Any] | None = None) -> px.Graph: - """从 YAML 文件加载任务图. - - Parameters - ---------- - path : str | Path - YAML 文件路径 - variables : Mapping[str, Any] | None - 运行时变量, 用于替换 ``${VAR}`` 占位符 - - Returns - ------- - px.Graph - 构建好的任务图 - - Raises - ------ - YamlLoadError - 文件不存在、YAML 格式错误、schema 校验失败、循环依赖等 - """ - p = Path(path) - if not p.exists(): - raise YamlLoadError(f"YAML 文件不存在: {p}") - try: - content = p.read_text(encoding="utf-8") - except OSError as e: - raise YamlLoadError(f"读取文件失败: {e}") from e - return parse_yaml_string(content, variables=variables) - - -def parse_yaml_string(content: str, variables: Mapping[str, Any] | None = None) -> px.Graph: - """从 YAML 字符串构建任务图. - - Parameters - ---------- - content : str - YAML 文本 - variables : Mapping[str, Any] | None - 运行时变量, 用于替换 ``${VAR}`` 占位符 - - Returns - ------- - px.Graph - 构建好的任务图 - """ - data = _parse_yaml(content) - if not isinstance(data, Mapping): - raise YamlLoadError("YAML 根节点必须是 mapping (对象)") - return _build_graph(data, variables=variables) - - -def run_yaml( - path: str | Path, - jobs: str | Sequence[str] | None = None, - variables: Mapping[str, Any] | None = None, - strategy: str | None = None, - *, - max_workers: int | None = None, - dry_run: bool = False, - verbose: bool = False, - on_event: EventCallback | None = None, - state: StateBackend | None = None, - concurrency_limits: Mapping[str, int] | None = None, -) -> RunReport: - """加载 YAML 配置并执行任务图. - - 便捷函数, 组合 :func:`load_yaml` + (可选) 子图选择 + :func:`pyflowx.run`. - 指定 ``jobs`` 时自动包含其传递依赖, 适用于从多 job 配置中执行单个命令. - - Parameters - ---------- - path : str | Path - YAML 配置文件路径 - jobs : str | Sequence[str] | None - 要执行的 job 名. None 表示执行全部. str 表示单个 job. - 指定 job 时自动包含其传递依赖. - variables : Mapping[str, Any] | None - 运行时变量, 用于替换 ``${VAR}`` 占位符 - strategy : str | None - 执行策略, None 使用 YAML 中的策略或默认 ``"dependency"`` - max_workers : int | None - ``"thread"`` 策略的线程池大小 - dry_run : bool - 仅打印执行计划, 不执行 - verbose : bool - 打印任务生命周期 - on_event : EventCallback | None - 状态转换回调 - state : StateBackend | None - 断点续跑后端 - concurrency_limits : Mapping[str, int] | None - ``{concurrency_key: max_concurrent}`` 映射 - - Returns - ------- - RunReport - 执行报告 - - Raises - ------ - YamlLoadError - YAML 加载或 schema 校验失败 - KeyError - ``jobs`` 指定了不存在的 job 名 - """ - graph = load_yaml(path, variables=variables) - - if jobs is not None: - job_names = [jobs] if isinstance(jobs, str) else list(jobs) - all_names = _collect_with_deps(graph, job_names) - graph = graph.subgraph_by_names(all_names) - - effective_strategy = strategy if strategy is not None else (graph.defaults.strategy or "dependency") - return px.run( - graph, - strategy=cast(Strategy, effective_strategy), - max_workers=max_workers, - dry_run=dry_run, - verbose=verbose, - on_event=on_event, - state=state, - concurrency_limits=concurrency_limits, - ) - - -def _collect_with_deps(graph: px.Graph, names: Iterable[str]) -> set[str]: - """收集 ``names`` 及其所有传递依赖, 用于子图选择.""" - result: set[str] = set() - queue: list[str] = list(names) - while queue: - name = queue.pop() - if name in result: - continue - result.add(name) - spec = graph.spec(name) - queue.extend(spec.depends_on) - return result - - -def _parse_yaml(content: str) -> Any: - """解析 YAML 文本, 失败抛出 YamlLoadError.""" - try: - import yaml - except ImportError as e: # pragma: no cover - raise YamlLoadError("缺少 PyYAML 依赖, 请执行 `pip install pyyaml`") from e - - try: - return yaml.safe_load(content) - except yaml.YAMLError as e: - raise YamlLoadError(f"YAML 解析失败: {e}") from e - - -def _merge_variables( - yaml_vars: Any, - caller_vars: Mapping[str, Any] | None, -) -> dict[str, Any]: - """合并 YAML 中的 variables (默认值) 与调用方传入的 variables (后者优先).""" - if isinstance(yaml_vars, Mapping): - return {**yaml_vars, **(caller_vars or {})} - return dict(caller_vars) if caller_vars else {} - - -def _build_graph(data: Mapping[str, Any], variables: Mapping[str, Any] | None = None) -> px.Graph: - """从解析后的 mapping 构建 Graph.""" - jobs = data.get("jobs") - if not isinstance(jobs, Mapping): - raise YamlLoadError("YAML 必须包含 jobs 字段, 且 jobs 必须是 mapping") - - if not jobs: - raise YamlLoadError("jobs 不能为空") - - merged_vars = _merge_variables(data.get("variables"), variables) - defaults = _parse_defaults(data.get("defaults") or {}) - - # 顶层 strategy 覆盖 defaults.strategy - top_strategy = data.get("strategy") - if top_strategy is not None: - if not isinstance(top_strategy, str): - raise YamlLoadError("顶层 strategy 必须是字符串") - defaults = px.GraphDefaults( - retry=defaults.retry, - timeout=defaults.timeout, - strategy=top_strategy, - tags=defaults.tags, - env=defaults.env, - cwd=defaults.cwd, - priority=defaults.priority, - continue_on_error=defaults.continue_on_error, - concurrency_key=defaults.concurrency_key, - verbose=defaults.verbose, - ) - - # 第一阶段: 预计算每个 job 的展开名 (用于矩阵依赖展开) - job_expansions: dict[str, list[str]] = {} - for job_id, job_config in jobs.items(): - if not isinstance(job_id, str) or not job_id: - raise YamlLoadError(f"job id 必须是非空字符串, 收到: {job_id!r}") - if not isinstance(job_config, Mapping): - raise YamlLoadError(f"job {job_id!r} 必须是 mapping") - matrix_combos = _expand_matrix(job_config.get("strategy")) - if any(combo is not None for combo in matrix_combos): - job_expansions[job_id] = [_matrix_name(job_id, c) for c in matrix_combos] - else: - job_expansions[job_id] = [job_id] - - # 第二阶段: 解析 job, 展开矩阵依赖 - specs: list[TaskSpec[Any]] = [] - seen_names: set[str] = set() - for job_id, job_config in jobs.items(): - for spec in _parse_job(job_id, job_config, job_expansions, variables=merged_vars): - if spec.name in seen_names: - raise YamlLoadError(f"重复任务名: {spec.name!r}") - seen_names.add(spec.name) - specs.append(spec) - - return px.Graph.from_specs(specs, defaults=defaults) - - -def _parse_defaults(data: Mapping[str, Any]) -> px.GraphDefaults: - """解析 defaults 字段为 GraphDefaults.""" - if not data: - return px.GraphDefaults() - - retry = _parse_retry(data["retry"]) if "retry" in data else None - return px.GraphDefaults( - retry=retry, - timeout=_as_float(data.get("timeout")), - strategy=_as_str(data.get("strategy")), - tags=_as_str_tuple(data.get("tags")), - env=_as_str_mapping(data.get("env")), - cwd=_as_path(data.get("cwd")), - priority=_as_int(data.get("priority")) or 0, - continue_on_error=bool(data.get("continue-on-error") or data.get("continue_on_error") or False), - concurrency_key=_as_str(data.get("concurrency-key") or data.get("concurrency_key")), - verbose=bool(data.get("verbose") or False), - ) - - -def _parse_job( - job_id: str, - config: Mapping[str, Any], - job_expansions: Mapping[str, list[str]], - variables: Mapping[str, Any] | None = None, -) -> list[TaskSpec[Any]]: - """解析单个 job, 返回 spec 列表 (矩阵展开可能产生多个). - - Parameters - ---------- - job_id : str - job 标识符 - config : Mapping[str, Any] - job 配置 - job_expansions : Mapping[str, list[str]] - 每个 job_id 到其展开名列表的映射, 用于展开矩阵依赖 - variables : Mapping[str, Any] | None - 运行时变量, 用于替换 ``${VAR}`` 占位符 - """ - cmd_data, run_data, fn_data, args_data, kwargs_data = _parse_task_type_fields(job_id, config) - - matrix_combos = _expand_matrix(config.get("strategy")) - if not matrix_combos: - matrix_combos = [None] - - needs_tuple = _expand_needs(_as_str_tuple(config.get("needs")), job_expansions) - - if_data = config.get("if") - timeout = _as_float(config.get("timeout")) - retry = _parse_retry(config["retry"]) if "retry" in config else None - cwd = _as_path(config.get("cwd")) - env = _as_str_mapping(config.get("env")) - verbose = bool(config.get("verbose")) - continue_on_error = bool(config.get("continue-on-error") or config.get("continue_on_error") or False) - skip_if_missing = bool(config.get("skip-if-missing") or config.get("skip_if_missing") or False) - allow_upstream_skip = bool(config.get("allow-upstream-skip") or config.get("allow_upstream_skip") or False) - priority = _as_int(config.get("priority")) or 0 - concurrency_key = _as_str(config.get("concurrency-key") or config.get("concurrency_key")) - tags = _as_str_tuple(config.get("tags")) - runs_on = _as_str(config.get("runs-on") or config.get("runs_on")) - if runs_on: - tags = (*tags, runs_on) - - fn = _resolve_fn(job_id, fn_data) - # 方向 B: 聚合 job (有 needs 无 cmd/run/fn) 注入 no-op fn, 仅作依赖聚合点 - is_aggregate = fn is None and cmd_data is None and run_data is None and bool(needs_tuple) - if is_aggregate: - fn = _aggregate_noop - - specs: list[TaskSpec[Any]] = [] - for combo in matrix_combos: - name = _matrix_name(job_id, combo) - cmd = _build_cmd(cmd_data, run_data, combo, variables) if fn_data is None and not is_aggregate else None - cwd_resolved = Path(_replace_in_str(str(cwd), combo, variables)) if cwd is not None else None - env_resolved = _replace_in_mapping(env, combo, variables) if env else None - conditions = _parse_condition(if_data) if if_data is not None else () - - # fn 任务的 args/kwargs 应用占位符替换 - args_resolved = _replace_in_value(args_data, combo, variables) if args_data is not None else () - kwargs_resolved = _replace_in_value(kwargs_data, combo, variables) if kwargs_data is not None else {} - - spec = px.TaskSpec( - name=name, - fn=fn, - cmd=cmd, - args=tuple(args_resolved), - kwargs=dict(kwargs_resolved), - depends_on=needs_tuple, - retry=retry if retry is not None else RetryPolicy(), - timeout=timeout, - tags=tags, - conditions=conditions, - cwd=cwd_resolved, - env=env_resolved, - verbose=verbose, - skip_if_missing=skip_if_missing, - allow_upstream_skip=allow_upstream_skip, - priority=priority, - concurrency_key=concurrency_key, - continue_on_error=continue_on_error, - ) - specs.append(spec) - return specs - - -def _parse_task_type_fields( - job_id: str, - config: Mapping[str, Any], -) -> tuple[Any, Any, Any, Any, Any]: - """解析并校验任务类型字段 (cmd/run/fn) 及参数 (args/kwargs). - - Returns - ------- - tuple - (cmd_data, run_data, fn_data, args_data, kwargs_data) - """ - cmd_data = config.get("cmd") - run_data = config.get("run") - fn_data = config.get("fn") - args_data = config.get("args") - kwargs_data = config.get("kwargs") - - # 校验互斥: cmd/run/fn 三选一 - provided = [x for x in (cmd_data, run_data, fn_data) if x is not None] - if len(provided) > 1: - raise YamlLoadError(f"job {job_id!r} 不能同时提供 cmd/run/fn") - # 方向 B: 允许有 needs 无 cmd/run/fn 的聚合 job (仅作为依赖聚合点) - if not provided and not config.get("needs"): - raise YamlLoadError(f"job {job_id!r} 必须提供 cmd/run/fn 之一, 或提供 needs 作为聚合 job") - - # fn 必须是字符串 - if fn_data is not None and not isinstance(fn_data, str): - raise YamlLoadError(f"job {job_id!r} 的 fn 必须是字符串, 收到: {type(fn_data).__name__}") - - # args 必须是列表, kwargs 必须是 mapping - if args_data is not None and not isinstance(args_data, list): - raise YamlLoadError(f"job {job_id!r} 的 args 必须是列表, 收到: {type(args_data).__name__}") - if kwargs_data is not None and not isinstance(kwargs_data, Mapping): - raise YamlLoadError(f"job {job_id!r} 的 kwargs 必须是 mapping, 收到: {type(kwargs_data).__name__}") - - return cmd_data, run_data, fn_data, args_data, kwargs_data - - -def _expand_needs( - needs_raw: tuple[str, ...], - job_expansions: Mapping[str, list[str]], -) -> tuple[str, ...]: - """展开 needs 中的矩阵依赖.""" - needs: list[str] = [] - for dep in needs_raw: - if dep in job_expansions: - needs.extend(job_expansions[dep]) - else: - needs.append(dep) - return tuple(needs) - - -def _resolve_fn(job_id: str, fn_data: Any) -> Any: - """从 registry 查找已注册的函数.""" - if fn_data is None: - return None - try: - return _get_fn(fn_data) - except KeyError as e: - raise YamlLoadError(f"job {job_id!r} 引用的函数 {fn_data!r} 未注册") from e - - -def _expand_matrix(strategy_data: Any) -> list[Mapping[str, Any] | None]: - """展开 strategy.matrix 为笛卡尔积列表. - - Returns - ------- - list[Mapping[str, Any] | None] - 无矩阵时返回 [None], 有矩阵时返回笛卡尔积 - """ - if strategy_data is None: - return [None] - if not isinstance(strategy_data, Mapping): - raise YamlLoadError("strategy 必须是 mapping") - matrix = strategy_data.get("matrix") - if matrix is None: - return [None] - if not isinstance(matrix, Mapping): - raise YamlLoadError("strategy.matrix 必须是 mapping") - if not matrix: - return [None] - - keys = list(matrix.keys()) - value_lists = [] - for key in keys: - vals = matrix[key] - if isinstance(vals, (str, int, float, bool)): - vals = [vals] - elif not isinstance(vals, list): - raise YamlLoadError(f"matrix.{key} 必须是列表或标量") - if not vals: - raise YamlLoadError(f"matrix.{key} 不能为空列表") - value_lists.append([(key, v) for v in vals]) - - combos: list[Mapping[str, Any] | None] = [] - for combo_tuples in itertools.product(*value_lists): - combos.append(dict(combo_tuples)) - return combos - - -def _matrix_name(job_id: str, combo: Mapping[str, Any] | None) -> str: - """生成矩阵展开后的任务名.""" - if combo is None: - return job_id - parts = [job_id] - for key, val in combo.items(): - parts.append(f"{key}-{_sanitize_matrix_value(val)}") - return "_".join(parts) - - -def _sanitize_matrix_value(val: Any) -> str: - """将矩阵值转为合法任务名片段.""" - s = str(val) - # 替换任务名中不允许的字符 - return re.sub(r"[^a-zA-Z0-9._-]", "_", s) - - -def _build_cmd( - cmd_data: Any, - run_data: Any, - combo: Mapping[str, Any] | None, - variables: Mapping[str, Any] | None = None, -) -> list[str] | str: - """构建 cmd 字段, 应用占位符替换. - - 列表元素若完全匹配 ``${VAR}`` 且变量为列表时, 展开为多个元素; - 否则按字符串替换处理. - """ - if cmd_data is not None: - if isinstance(cmd_data, str): - return _replace_in_str(cmd_data, combo, variables) - if isinstance(cmd_data, list): - result: list[str] = [] - for item in cmd_data: - if variables and isinstance(item, str): - m = _VARIABLE_FULL_MATCH.match(item) - if m and m.group(1) in variables: - val = variables[m.group(1)] - if isinstance(val, list): - result.extend(str(v) for v in val) - continue - result.append(_replace_in_str(str(item), combo, variables)) - return result - raise YamlLoadError(f"cmd 必须是 list 或 str, 收到: {type(cmd_data).__name__}") - # run_data 不为 None (已在外层校验) - return _replace_in_str(str(run_data), combo, variables) - - -def _replace_in_str( - text: str, - combo: Mapping[str, Any] | None, - variables: Mapping[str, Any] | None = None, -) -> str: - """替换字符串中的 ``${VAR}`` 变量占位符和 ``${{ matrix.key }}`` 矩阵占位符.""" - # 先替换变量占位符 ${VAR} - if variables: - - def _sub_var(match: re.Match[str]) -> str: - key = match.group(1) - if key not in variables: - raise YamlLoadError(f"变量 {key!r} 未定义") - return str(variables[key]) - - text = _VARIABLE_PLACEHOLDER.sub(_sub_var, text) - - # 再替换矩阵占位符 ${{ matrix.key }} - if combo is not None: - - def _sub_matrix(match: re.Match[str]) -> str: - key = match.group(1) - if key not in combo: - raise YamlLoadError(f"矩阵占位符 matrix.{key} 未定义") - return str(combo[key]) - - text = _MATRIX_PLACEHOLDER.sub(_sub_matrix, text) - - return text - - -def _replace_in_mapping( - env: Mapping[str, str], - combo: Mapping[str, Any] | None, - variables: Mapping[str, Any] | None = None, -) -> dict[str, str]: - """替换 env 映射中的占位符.""" - return {k: _replace_in_str(v, combo, variables) for k, v in env.items()} - - -def _replace_in_value( - value: Any, - combo: Mapping[str, Any] | None, - variables: Mapping[str, Any] | None = None, -) -> Any: - """递归替换值 (list/mapping/scalar) 中的占位符. - - 当字符串完全等于 ``${VAR}`` 时, 返回变量的实际值 (非字符串化), - 适用于 ``args: ["${FILES}"]`` 这种需要传递列表/对象而非字符串的场景. - """ - if isinstance(value, str) and variables: - m = _VARIABLE_FULL_MATCH.match(value) - if m and m.group(1) in variables: - return variables[m.group(1)] - if isinstance(value, str): - return _replace_in_str(value, combo, variables) - if isinstance(value, list): - return [_replace_in_value(v, combo, variables) for v in value] - if isinstance(value, Mapping): - return {k: _replace_in_value(v, combo, variables) for k, v in value.items()} - return value - - -def _parse_condition(expr: Any) -> tuple[Condition, ...]: - """解析 if 条件表达式为 Condition 元组.""" - if not isinstance(expr, str): - raise YamlLoadError(f"if 条件必须是字符串, 收到: {type(expr).__name__}") - - expr = expr.strip() - if not expr: - raise YamlLoadError("if 条件不能为空") - - lowered = expr.lower() - if lowered in ("success()", "success"): - # 默认行为: 依赖成功才执行. 不设 condition. - return () - if lowered in ("always()", "always"): - # always() 在 PyFlowX 中无直接对应, 返回空 condition 元组, - # 由调用方通过 allow_upstream_skip=True 实现. - return () - if lowered in ("failure()", "failure"): - raise YamlLoadError("if: failure() 在 PyFlowX 中不支持 (失败会中止或跳过下游)") - - # env.VAR_EXISTS - m = _ENV_VAR_EXISTS_RE.match(expr) - if m: - return (BuiltinConditions.ENV_VAR_EXISTS(m.group(1)),) - - # env.VAR == 'value' - m = _ENV_VAR_EQUALS_RE.match(expr) - if m: - return (BuiltinConditions.ENV_VAR_EQUALS(m.group(1), m.group(2)),) - - # env.VAR != 'value' - m = _ENV_VAR_NOT_EQUALS_RE.match(expr) - if m: - return (BuiltinConditions.NOT(BuiltinConditions.ENV_VAR_EQUALS(m.group(1), m.group(2))),) - - raise YamlLoadError(f"不支持的 if 表达式: {expr!r} (支持 success()/always()/env.VAR/env.VAR=='x')") - - -def _parse_retry(data: Any) -> RetryPolicy: - """解析 retry 配置为 RetryPolicy.""" - if data is None: - return RetryPolicy() - if isinstance(data, Mapping): - return RetryPolicy( - max_attempts=int(data.get("max_attempts", data.get("max-attempts", 1))), - delay=float(data.get("delay", 0.0)), - backoff=float(data.get("backoff", 1.0)), - jitter=float(data.get("jitter", 0.0)), - retry_on=(Exception,), # 默认重试所有异常 - ) - raise YamlLoadError(f"retry 必须是 mapping, 收到: {type(data).__name__}") - - -def _as_float(value: Any) -> float | None: - """安全转换为 float.""" - if value is None: - return None - if isinstance(value, (int, float)): - return float(value) - raise YamlLoadError(f"期望数字, 收到: {value!r}") - - -def _as_int(value: Any) -> int | None: - """安全转换为 int.""" - if value is None: - return None - if isinstance(value, int) and not isinstance(value, bool): - return value - if isinstance(value, float): - return int(value) - raise YamlLoadError(f"期望整数, 收到: {value!r}") - - -def _as_str(value: Any) -> str | None: - """安全转换为 str.""" - if value is None: - return None - if isinstance(value, str): - return value - raise YamlLoadError(f"期望字符串, 收到: {value!r}") - - -def _as_str_tuple(value: Any) -> tuple[str, ...]: - """转换为字符串元组.""" - if value is None: - return () - if isinstance(value, str): - return (value,) - if isinstance(value, list): - return tuple(str(v) for v in value) - raise YamlLoadError(f"期望字符串或列表, 收到: {value!r}") - - -def _as_str_mapping(value: Any) -> dict[str, str] | None: - """转换为 {str: str} 映射.""" - if value is None: - return None - if not isinstance(value, Mapping): - raise YamlLoadError(f"期望 mapping, 收到: {value!r}") - return {str(k): str(v) for k, v in value.items()} - - -def _as_path(value: Any) -> Path | None: - """转换为 Path.""" - if value is None: - return None - return Path(str(value)) - - -# ============================================================================ -# CLI 参数解析 -# ============================================================================ - - -# 支持的 CLI 参数类型 -_CLI_TYPE_MAP: dict[str, type] = { - "str": str, - "int": int, - "float": float, - "path": Path, -} - - -def _parse_cli_type(type_str: str | None) -> type: - """解析 CLI 参数类型字符串.""" - if type_str is None: - return str - if type_str in _CLI_TYPE_MAP: - return _CLI_TYPE_MAP[type_str] - raise YamlLoadError(f"不支持的 CLI 参数类型: {type_str!r}") - - -def build_cli_parser(data: Mapping[str, Any]) -> Any: - """从 YAML 配置的 ``cli:`` 段构建 argparse 解析器. - - Parameters - ---------- - data : Mapping[str, Any] - 解析后的 YAML 配置 (包含 ``cli`` 和 ``jobs``) - - Returns - ------- - argparse.ArgumentParser - 构建好的参数解析器 - - Raises - ------ - YamlLoadError - cli 段格式错误 - """ - import argparse - - cli_raw = data.get("cli") - if cli_raw is None: - cli_config: Mapping[str, Any] = {} - elif not isinstance(cli_raw, Mapping): - raise YamlLoadError(f"cli 必须是 mapping, 收到: {type(cli_raw).__name__}") - else: - cli_config = cli_raw - - description = str(cli_config.get("description", "")) - usage = str(cli_config.get("usage", "")) if cli_config.get("usage") else None - - parser = argparse.ArgumentParser(description=description, usage=usage) - - subcommands = cli_config.get("subcommands") - if subcommands is not None: - if not isinstance(subcommands, Mapping): - raise YamlLoadError(f"cli.subcommands 必须是 mapping, 收到: {type(subcommands).__name__}") - subparsers = parser.add_subparsers(dest="command", help="可用命令") - for sub_name, sub_config in subcommands.items(): - if not isinstance(sub_config, Mapping): - raise YamlLoadError(f"cli.subcommands.{sub_name} 必须是 mapping") - sub = subparsers.add_parser(sub_name, help=str(sub_config.get("help", ""))) - _add_args_to_parser(sub, sub_config) - else: - _add_args_to_parser(parser, cli_config) - - return parser - - -def _add_args_to_parser(parser: Any, config: Mapping[str, Any]) -> None: # noqa: PLR0912 - """向 parser 添加位置参数和选项参数.""" - positional = config.get("positional", []) - if positional: - if not isinstance(positional, list): - raise YamlLoadError("positional 必须是列表") - for arg_cfg in positional: - if not isinstance(arg_cfg, Mapping): - raise YamlLoadError("positional 元素必须是 mapping") - name = str(arg_cfg["name"]) - kwargs: dict[str, Any] = {} - if "help" in arg_cfg: - kwargs["help"] = str(arg_cfg["help"]) - if "nargs" in arg_cfg: - kwargs["nargs"] = arg_cfg["nargs"] - if "type" in arg_cfg: - kwargs["type"] = _parse_cli_type(str(arg_cfg["type"])) - if "default" in arg_cfg: - kwargs["default"] = arg_cfg["default"] - parser.add_argument(name.lower().replace("-", "_"), **kwargs) - - options = config.get("options", []) - if options: - if not isinstance(options, list): - raise YamlLoadError("options 必须是列表") - for opt_cfg in options: - if not isinstance(opt_cfg, Mapping): - raise YamlLoadError("options 元素必须是 mapping") - name = str(opt_cfg["name"]) - flag = str(opt_cfg.get("flag", f"--{name.lower().replace('_', '-')}")) - kwargs = {} - if "help" in opt_cfg: - kwargs["help"] = str(opt_cfg["help"]) - if "type" in opt_cfg: - kwargs["type"] = _parse_cli_type(str(opt_cfg["type"])) - if "default" in opt_cfg: - kwargs["default"] = opt_cfg["default"] - if "required" in opt_cfg: - kwargs["required"] = bool(opt_cfg["required"]) - if "action" in opt_cfg: - kwargs["action"] = str(opt_cfg["action"]) - parser.add_argument(flag, dest=name.lower().replace("-", "_"), **kwargs) - - -class YamlCliRunner: - """从 YAML 配置构建并执行 CLI. - - 读取 YAML 的 ``cli:`` 段构建 argparse 解析器, 解析命令行参数, - 然后调用 :func:`run_yaml` 执行对应的 job. - - 全局选项 ``--dry-run`` / ``--quiet`` / ``--strategy`` / ``--list`` - 自动添加到所有工具. - """ - - def __init__(self, path: str | Path, argv: Sequence[str] | None = None) -> None: - self._path = Path(path) - self._argv = list(argv) if argv is not None else sys.argv[1:] - self._data: Mapping[str, Any] = {} - self._cli_config: Mapping[str, Any] = {} - - def run(self) -> int: # noqa: PLR0911 - """执行 CLI, 返回退出码 (0 成功, 1 失败).""" - from pyflowx.errors import TaskFailedError - - if not self._path.exists(): - print(f"错误: YAML 文件不存在: {self._path}", file=sys.stderr) - return 1 - - if not self._load_config(): - return 1 - - try: - parser = build_cli_parser(self._data) - except YamlLoadError as e: - print(f"错误: CLI 配置错误: {e}", file=sys.stderr) - return 1 - - self._add_global_options(parser) - args = parser.parse_args(self._argv) - - has_subcommands = bool(self._cli_config.get("subcommands")) - cmd = getattr(args, "command", None) if has_subcommands else None - job_name: str | None = cmd - - # --list 优先于 cmd 缺失判断 - if getattr(args, "list_jobs", False): - return self._handle_list() - - if has_subcommands and cmd is None: - parser.print_help() - return 0 - - variables = self._extract_variables(args) - job_display = job_name if job_name else "全部任务" - print(f"[{self._path.stem}] 执行: {job_display}", flush=True) - - try: - run_yaml( - self._path, - jobs=job_name, - variables=variables if variables else None, - strategy=getattr(args, "strategy", None), - dry_run=getattr(args, "dry_run", False), - verbose=getattr(args, "verbose", False), - ) - return 0 - except TaskFailedError as e: - print(f"错误: {e}", file=sys.stderr) - return 1 - except YamlLoadError as e: - print(f"错误: {e}", file=sys.stderr) - return 1 - except KeyboardInterrupt: - print("\n已中断", file=sys.stderr) - return 130 - - def _load_config(self) -> bool: - """加载并校验 YAML 配置, 返回是否成功.""" - try: - content = self._path.read_text(encoding="utf-8") - data = _parse_yaml(content) - except YamlLoadError as e: - print(f"错误: YAML 加载失败: {e}", file=sys.stderr) - return False - - if not isinstance(data, Mapping): - print("错误: YAML 根节点必须是 mapping", file=sys.stderr) - return False - - cli_raw = data.get("cli") or {} - if not isinstance(cli_raw, Mapping): - print("错误: cli 段格式错误", file=sys.stderr) - return False - - self._data = data - self._cli_config = cli_raw - return True - - def _add_global_options(self, parser: Any) -> None: - """向解析器添加全局选项. - - verbose 默认开启 (显示执行过程), 用 --quiet 关闭. - """ - parser.add_argument("--dry-run", action="store_true", help="仅打印执行计划, 不执行") - parser.add_argument("--quiet", "-q", action="store_false", dest="verbose", help="减少输出, 不显示执行过程") - parser.add_argument("--strategy", type=str, default=None, help="执行策略 (sequential/thread/dependency)") - parser.add_argument("--list", action="store_true", dest="list_jobs", help="列出所有任务名后退出") - - # 给子命令也添加全局选项 - for action in parser._actions: - if action.dest == "command" and hasattr(action, "choices"): - for sub in action.choices.values(): - sub.add_argument("--dry-run", action="store_true", help="仅打印执行计划, 不执行") - sub.add_argument("--quiet", "-q", action="store_false", dest="verbose", help="减少输出") - sub.add_argument("--strategy", type=str, default=None, help="执行策略") - sub.add_argument("--list", action="store_true", dest="list_jobs", help="列出所有任务名后退出") - - def _extract_variables(self, args: Any) -> dict[str, Any]: - """从解析后的 namespace 提取变量字典 (大写 key).""" - variables: dict[str, Any] = {} - arg_names: set[str] = set() - subcommands = self._cli_config.get("subcommands") - if subcommands is not None and isinstance(subcommands, Mapping): - cmd = getattr(args, "command", None) - if cmd and cmd in subcommands: - sub_cfg = subcommands[cmd] - if isinstance(sub_cfg, Mapping): - self._collect_arg_names(arg_names, sub_cfg) - else: - self._collect_arg_names(arg_names, self._cli_config) - - for name in arg_names: - attr_name = name.lower().replace("-", "_") - if hasattr(args, attr_name): - variables[name] = getattr(args, attr_name) - - return variables - - @staticmethod - def _collect_arg_names(names: set[str], config: Mapping[str, Any]) -> None: - """收集配置中的参数名.""" - for arg_cfg in config.get("positional", []) or []: - if isinstance(arg_cfg, Mapping) and "name" in arg_cfg: - names.add(str(arg_cfg["name"])) - for opt_cfg in config.get("options", []) or []: - if isinstance(opt_cfg, Mapping) and "name" in opt_cfg: - names.add(str(opt_cfg["name"])) - - def _handle_list(self) -> int: - """处理 --list 选项, 打印任务列表.""" - graph = load_yaml(self._path) - print("任务列表:") - for name in graph.names: - spec = graph.spec(name) - deps = ", ".join(spec.depends_on) if spec.depends_on else "(无依赖)" - print(f" - {name} (依赖: {deps})") - return 0 - - -def run_cli(path: str | Path, argv: Sequence[str] | None = None) -> int: - """从 YAML 配置构建 CLI 并执行. - - :class:`YamlCliRunner` 的便捷包装. - - Parameters - ---------- - path : str | Path - YAML 配置文件路径 - argv : Sequence[str] | None - 命令行参数, None 表示使用 sys.argv[1:] - - Returns - ------- - int - 退出码 (0 成功, 1 失败) - """ - return YamlCliRunner(path, argv).run() diff --git a/tests/cli/test_envdev.py b/tests/cli/test_envdev.py index 02c41bc..40efec1 100644 --- a/tests/cli/test_envdev.py +++ b/tests/cli/test_envdev.py @@ -1,4 +1,4 @@ -"""Tests for ops.dev 模块 envdev/dockercmd 函数 (镜像源配置/Docker 登录).""" +"""Tests for ops.envdev / ops.dockercmd 模块 (镜像源配置/Docker 登录).""" from __future__ import annotations @@ -9,8 +9,8 @@ from unittest.mock import MagicMock import pytest from pyflowx.conditions import Constants -from pyflowx.ops.dev import ( - docker_login_tencent, +from pyflowx.ops.dockercmd import docker_login_tencent +from pyflowx.ops.envdev import ( download_rustup_script, install_linux_docker, install_linux_fonts, @@ -123,10 +123,10 @@ class TestSetupRustMirror: def test_creates_sccache_dir(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """应创建 sccache 缓存目录.""" - from pyflowx.ops import dev as dev_module + from pyflowx.ops import envdev as envdev_module fake_sccache = tmp_path / ".cargo" / "sccache" - monkeypatch.setattr(dev_module, "_RUST_SCCACHE_DIR", fake_sccache) + monkeypatch.setattr(envdev_module, "_RUST_SCCACHE_DIR", fake_sccache) monkeypatch.setattr(Path, "home", lambda: tmp_path) setup_rust_mirror("tsinghua") assert fake_sccache.exists() diff --git a/tests/cli/test_folderback.py b/tests/cli/test_folderback.py index db4d03a..96d85fc 100644 --- a/tests/cli/test_folderback.py +++ b/tests/cli/test_folderback.py @@ -154,6 +154,5 @@ class TestRegisteredFunctions: """Test that folderback functions are registered.""" def test_folderback_default_spec(self) -> None: - """folderback_default should be a registered callable.""" - # folderback_default 现在是通过 @px.register_fn 注册的普通函数, 不是 TaskSpec + """folderback_default should be a callable function.""" assert callable(files.folderback_default) diff --git a/tests/cli/test_folderzip.py b/tests/cli/test_folderzip.py index dd6335a..2905773 100644 --- a/tests/cli/test_folderzip.py +++ b/tests/cli/test_folderzip.py @@ -56,6 +56,5 @@ class TestRegisteredFunctions: """Test that folderzip functions are registered.""" def test_folderzip_default_spec(self) -> None: - """folderzip_default should be a registered callable.""" - # folderzip_default 现在是通过 @px.register_fn 注册的普通函数, 不是 TaskSpec + """folderzip_default should be a callable function.""" assert callable(files.folderzip_default) diff --git a/tests/cli/test_gittool.py b/tests/cli/test_gittool.py index f512632..01f7e80 100644 --- a/tests/cli/test_gittool.py +++ b/tests/cli/test_gittool.py @@ -8,7 +8,7 @@ from unittest.mock import patch import pytest import pyflowx as px -from pyflowx.ops import dev +from pyflowx.ops import gittool as dev # ---------------------------------------------------------------------- # diff --git a/tests/cli/test_llm.py b/tests/cli/test_llm.py index bc097f0..b694fd8 100644 --- a/tests/cli/test_llm.py +++ b/tests/cli/test_llm.py @@ -1,4 +1,4 @@ -"""Tests for ops.llm 模块 (msdownload/sglang).""" +"""Tests for ops.msdownload / ops.sglang 模块 (ModelScope 下载/SGLang 服务).""" from __future__ import annotations @@ -9,7 +9,8 @@ from unittest.mock import MagicMock import pytest from pyflowx.conditions import Constants -from pyflowx.ops.llm import install_sglang, msdownload_run, run_sglang +from pyflowx.ops.msdownload import msdownload_run +from pyflowx.ops.sglang import install_sglang, run_sglang # ---------------------------------------------------------------------- # diff --git a/tests/test_registry.py b/tests/test_registry.py deleted file mode 100644 index b814356..0000000 --- a/tests/test_registry.py +++ /dev/null @@ -1,256 +0,0 @@ -"""``pyflowx.registry`` 模块的单元测试.""" - -from __future__ import annotations - -from typing import Any - -import pytest - -import pyflowx as px -from pyflowx.registry import _REGISTRY, FnRegistry, get_fn, has_fn, register_fn - - -@pytest.fixture -def clear_registry() -> Any: - """每个测试前后清空 registry, 但保留 ops 模块自动注册的函数. - - ops 模块在首次导入时注册函数, 模块不会重新执行, 因此清空后无法重新注册. - 保存原始状态并在 teardown 时恢复, 确保 TestOpsModules 等不使用此 fixture 的 - 测试仍能看到 ops 的注册. - """ - saved = dict(_REGISTRY) - _REGISTRY.clear() - yield - _REGISTRY.clear() - _REGISTRY.update(saved) - - -class TestRegisterFn: - """``register_fn`` 装饰器测试.""" - - @pytest.fixture(autouse=True) - def _clear(self, clear_registry: Any) -> None: - """每个测试前后清空 registry.""" - - def test_register_with_explicit_name(self) -> None: - """使用显式名称注册函数.""" - - @register_fn("custom_name") - def _func(x: int) -> int: - return x * 2 - - assert has_fn("custom_name") - assert get_fn("custom_name")(5) == 10 - - def test_register_without_parentheses(self) -> None: - """无括号使用 ``@register_fn`` 时以 ``__name__`` 注册.""" - - @register_fn - def my_func(x: int) -> int: - return x + 1 - - assert has_fn("my_func") - assert get_fn("my_func")(10) == 11 - - def test_register_with_none_uses_func_name(self) -> None: - """``@register_fn(None)`` 等价于使用 ``__name__``.""" - - @register_fn(None) - def auto_named(x: int) -> int: - return x - - assert has_fn("auto_named") - - def test_register_preserves_function(self) -> None: - """装饰后函数仍可直接调用.""" - - @register_fn("calc") - def _calc(a: int, b: int) -> int: - return a + b - - assert _calc(2, 3) == 5 - - def test_duplicate_registration_raises(self) -> None: - """重复注册同名函数抛出 ValueError.""" - with pytest.raises(ValueError, match="已注册"): - - @register_fn("dup") - def _first() -> None: - pass - - @register_fn("dup") - def _second() -> None: - pass - - def test_duplicate_registration_without_parentheses(self) -> None: - """无括号模式下重复注册也抛出 ValueError.""" - with pytest.raises(ValueError, match="已注册"): - - @register_fn - def shared_name() -> None: - pass - - @register_fn - def shared_name() -> None: # noqa: F811 - pass - - -class TestGetFn: - """``get_fn`` 函数测试.""" - - @pytest.fixture(autouse=True) - def _clear(self, clear_registry: Any) -> None: - """每个测试前后清空 registry.""" - - def test_get_registered_function(self) -> None: - """获取已注册的函数.""" - - @register_fn("target") - def _target(x: int) -> int: - return x * 3 - - retrieved = get_fn("target") - assert retrieved is _target - - def test_get_unregistered_raises_keyerror(self) -> None: - """获取未注册的函数抛出 KeyError.""" - with pytest.raises(KeyError, match="not_found"): - get_fn("not_found") - - -class TestHasFn: - """``has_fn`` 函数测试.""" - - @pytest.fixture(autouse=True) - def _clear(self, clear_registry: Any) -> None: - """每个测试前后清空 registry.""" - - def test_has_registered(self) -> None: - """已注册返回 True.""" - - @register_fn("exists") - def _exists() -> None: - pass - - assert has_fn("exists") is True - - def test_has_not_registered(self) -> None: - """未注册返回 False.""" - assert has_fn("nope") is False - - -class TestFnRegistry: - """``FnRegistry`` 类测试.""" - - @pytest.fixture(autouse=True) - def _clear(self, clear_registry: Any) -> None: - """每个测试前后清空 registry.""" - - def test_register_method(self) -> None: - """``FnRegistry.register`` 等价于 ``register_fn``.""" - - @FnRegistry.register("via_class") - def _func(x: int) -> int: - return x - - assert FnRegistry.has("via_class") - assert FnRegistry.get("via_class")(7) == 7 - - def test_register_method_without_name(self) -> None: - """``FnRegistry.register(None)`` 使用函数名.""" - - @FnRegistry.register() - def class_auto(x: int) -> int: - return x - - assert FnRegistry.has("class_auto") - - def test_get_method(self) -> None: - """``FnRegistry.get`` 获取已注册函数.""" - - @register_fn("klass") - def _klass() -> str: - return "hello" - - assert FnRegistry.get("klass")() == "hello" - - def test_has_method_false(self) -> None: - """``FnRegistry.has`` 未注册返回 False.""" - assert FnRegistry.has("missing") is False - - def test_clear_method(self) -> None: - """``FnRegistry.clear`` 清空注册表.""" - - @register_fn("temp") - def _temp() -> None: - pass - - assert FnRegistry.has("temp") - FnRegistry.clear() - assert not FnRegistry.has("temp") - - def test_names_method(self) -> None: - """``FnRegistry.names`` 返回所有注册名.""" - - @register_fn("alpha") - def _alpha() -> None: - pass - - @register_fn("beta") - def _beta() -> None: - pass - - names = FnRegistry.names() - assert "alpha" in names - assert "beta" in names - - def test_names_returns_copy(self) -> None: - """``names`` 返回列表副本, 修改不影响内部状态.""" - - @register_fn("orig") - def _orig() -> None: - pass - - names = FnRegistry.names() - names.append("fake") - assert not FnRegistry.has("fake") - - -class TestOpsModules: - """``ops`` 子模块导入后函数自动注册的集成测试. - - 这些测试不使用 ``_clear_registry`` fixture, 因为 ``ops`` 模块在首次导入时 - 注册函数, 清空后将无法重新注册 (模块不会再次执行). - """ - - def test_all_ops_functions_registered(self) -> None: - """导入 ``ops`` 后所有函数应已注册.""" - import inspect - - from pyflowx.ops import bumpversion, dev, files, llm, media, system - - for module in (files, dev, bumpversion, media, system, llm): - for name in module.__all__: - obj = getattr(module, name) - if not inspect.isfunction(obj): - continue - assert px.has_fn(name), f"{module.__name__}.{name} 未注册" - - def test_total_function_count(self) -> None: - """注册函数总数 = 79 (含 bumpversion/dev/files/llm/media/system 全部 register_fn).""" - from pyflowx.ops import bumpversion, dev, files, llm, media, system # noqa: F401 - - all_names = px.FnRegistry.names() - assert len(all_names) == 79 - - def test_specific_functions_callable(self) -> None: - """关键注册函数可调用.""" - from pyflowx.ops import bumpversion, dev, files, llm, media, system - - assert px.get_fn("get_file_timestamp") is files.get_file_timestamp - assert px.get_fn("bump_file_version") is bumpversion.bump_file_version - assert px.get_fn("pdf_merge") is media.pdf_merge - assert px.get_fn("ssh_copy_id") is system.ssh_copy_id - assert px.get_fn("clear_screen_run") is system.clear_screen_run - assert px.get_fn("msdownload_run") is llm.msdownload_run - assert px.get_fn("setup_python_mirror") is dev.setup_python_mirror diff --git a/tests/test_yaml_loader.py b/tests/test_yaml_loader.py deleted file mode 100644 index 4f1e103..0000000 --- a/tests/test_yaml_loader.py +++ /dev/null @@ -1,1831 +0,0 @@ -"""Tests for yaml_loader module.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -import pyflowx as px -from pyflowx.task import TaskStatus -from pyflowx.yaml_loader import YamlLoadError, load_yaml, parse_yaml_string - - -# ---------------------------------------------------------------------- # -# load_yaml (文件加载) -# ---------------------------------------------------------------------- # -class TestLoadYaml: - """测试 load_yaml 函数.""" - - def test_load_yaml_file_not_exists(self, tmp_path: Path) -> None: - """文件不存在应抛出 YamlLoadError.""" - with pytest.raises(YamlLoadError, match="文件不存在"): - load_yaml(tmp_path / "nonexistent.yaml") - - def test_load_yaml_success(self, tmp_path: Path) -> None: - """成功加载 YAML 文件.""" - yaml_file = tmp_path / "pipeline.yaml" - yaml_file.write_text( - """ -jobs: - setup: - cmd: ["echo", "hello"] -""", - encoding="utf-8", - ) - graph = load_yaml(yaml_file) - assert "setup" in graph.names - assert graph.spec("setup").cmd == ["echo", "hello"] - - def test_load_yaml_path_string(self, tmp_path: Path) -> None: - """支持 str 路径.""" - yaml_file = tmp_path / "pipeline.yaml" - yaml_file.write_text( - 'jobs:\n t:\n cmd: ["echo"]\n', - encoding="utf-8", - ) - graph = load_yaml(str(yaml_file)) - assert graph.names == ["t"] - - def test_graph_from_yaml_classmethod(self, tmp_path: Path) -> None: - """Graph.from_yaml 类方法可用.""" - yaml_file = tmp_path / "pipeline.yaml" - yaml_file.write_text( - 'jobs:\n t:\n cmd: ["echo"]\n', - encoding="utf-8", - ) - graph = px.Graph.from_yaml(yaml_file) - assert graph.names == ["t"] - - -# ---------------------------------------------------------------------- # -# parse_yaml_string (字符串解析) -# ---------------------------------------------------------------------- # -class TestParseYamlString: - """测试 parse_yaml_string 函数.""" - - def test_basic_jobs(self) -> None: - """基础 job 定义.""" - graph = parse_yaml_string( - """ -jobs: - task1: - cmd: ["echo", "hello"] -""", - ) - assert graph.names == ["task1"] - assert graph.spec("task1").cmd == ["echo", "hello"] - - def test_root_not_mapping(self) -> None: - """根节点不是 mapping 应报错.""" - with pytest.raises(YamlLoadError, match="根节点必须是 mapping"): - parse_yaml_string("- item1\n- item2\n") - - def test_root_is_scalar(self) -> None: - """根节点是标量应报错.""" - with pytest.raises(YamlLoadError, match="根节点必须是 mapping"): - parse_yaml_string("just a string\n") - - def test_empty_content(self) -> None: - """空内容应报错.""" - with pytest.raises(YamlLoadError, match="根节点必须是 mapping"): - parse_yaml_string("") - - def test_yaml_syntax_error(self) -> None: - """YAML 语法错误应报错.""" - with pytest.raises(YamlLoadError, match="YAML 解析失败"): - parse_yaml_string(":\n : invalid\n") - - def test_jobs_not_mapping(self) -> None: - """jobs 不是 mapping 应报错.""" - with pytest.raises(YamlLoadError, match="jobs 必须是 mapping"): - parse_yaml_string("jobs: [1, 2, 3]\n") - - def test_jobs_missing(self) -> None: - """缺少 jobs 字段应报错.""" - with pytest.raises(YamlLoadError, match="jobs 必须是 mapping"): - parse_yaml_string("foo: bar\n") - - def test_jobs_empty(self) -> None: - """jobs 为空应报错.""" - with pytest.raises(YamlLoadError, match="jobs 不能为空"): - parse_yaml_string("jobs: {}\n") - - def test_job_not_mapping(self) -> None: - """job 配置不是 mapping 应报错.""" - with pytest.raises(YamlLoadError, match="必须是 mapping"): - parse_yaml_string( - """ -jobs: - bad: "just a string" -""", - ) - - def test_job_id_not_string(self) -> None: - """job id 非字符串应报错 (YAML 解析为 int).""" - with pytest.raises(YamlLoadError, match="job id 必须是非空字符串"): - parse_yaml_string( - """ -jobs: - 123: - cmd: ["echo"] -""", - ) - - -# ---------------------------------------------------------------------- # -# cmd / run 字段 -# ---------------------------------------------------------------------- # -class TestCmdRunFields: - """测试 cmd 和 run 字段.""" - - def test_cmd_list_form(self) -> None: - """cmd 列表形式.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["python", "-m", "pytest"] -""", - ) - assert graph.spec("t").cmd == ["python", "-m", "pytest"] - - def test_cmd_string_form(self) -> None: - """cmd 字符串形式 (含矩阵占位符).""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: "echo hello" -""", - ) - assert graph.spec("t").cmd == "echo hello" - - def test_run_form(self) -> None: - """run 形式 (shell 字符串).""" - graph = parse_yaml_string( - """ -jobs: - shell_task: - run: "echo hello && exit 0" -""", - ) - assert graph.spec("shell_task").cmd == "echo hello && exit 0" - - def test_cmd_and_run_conflict(self) -> None: - """同时提供 cmd 和 run 应报错.""" - with pytest.raises(YamlLoadError, match="不能同时提供 cmd/run/fn"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - run: "echo" -""", - ) - - def test_missing_cmd_and_run(self) -> None: - """缺少 cmd 和 run 应报错.""" - with pytest.raises(YamlLoadError, match="必须提供 cmd/run/fn 之一"): - parse_yaml_string( - """ -jobs: - t: - verbose: true -""", - ) - - def test_aggregate_job_only_needs(self) -> None: - """方向 B: 有 needs 无 cmd/run/fn 的聚合 job 应被允许, fn 注入 no-op.""" - graph = parse_yaml_string( - """ -jobs: - a: - cmd: ["echo", "a"] - b: - cmd: ["echo", "b"] - all: - needs: [a, b] -""", - ) - spec = graph.spec("all") - assert spec.fn is not None - assert spec.cmd is None - assert spec.depends_on == ("a", "b") - # no-op fn 调用应返回 None - assert spec.fn() is None - - def test_aggregate_job_no_needs_no_cmd_raises(self) -> None: - """方向 B: 既无 needs 又无 cmd/run/fn 仍应报错.""" - with pytest.raises(YamlLoadError, match="聚合 job"): - parse_yaml_string( - """ -jobs: - empty: - verbose: true -""", - ) - - def test_cmd_invalid_type(self) -> None: - """cmd 类型错误应报错.""" - with pytest.raises(YamlLoadError, match="cmd 必须是 list 或 str"): - parse_yaml_string( - """ -jobs: - t: - cmd: 123 -""", - ) - - -# ---------------------------------------------------------------------- # -# needs 依赖 -# ---------------------------------------------------------------------- # -class TestNeedsField: - """测试 needs 依赖字段.""" - - def test_needs_list(self) -> None: - """needs 列表形式.""" - graph = parse_yaml_string( - """ -jobs: - setup: - cmd: ["echo", "setup"] - build: - needs: [setup] - cmd: ["echo", "build"] -""", - ) - assert graph.spec("build").depends_on == ("setup",) - assert graph.layers() == [["setup"], ["build"]] - - def test_needs_string(self) -> None: - """needs 字符串形式.""" - graph = parse_yaml_string( - """ -jobs: - a: - cmd: ["echo"] - b: - needs: a - cmd: ["echo"] -""", - ) - assert graph.spec("b").depends_on == ("a",) - - def test_needs_multiple(self) -> None: - """多个依赖.""" - graph = parse_yaml_string( - """ -jobs: - a: - cmd: ["echo"] - b: - cmd: ["echo"] - c: - needs: [a, b] - cmd: ["echo"] -""", - ) - assert graph.spec("c").depends_on == ("a", "b") - - def test_parallel_execution(self) -> None: - """并行任务 (同层无依赖).""" - graph = parse_yaml_string( - """ -jobs: - a: - cmd: ["echo"] - b: - cmd: ["echo"] - c: - needs: [a, b] - cmd: ["echo"] -""", - ) - assert graph.layers() == [["a", "b"], ["c"]] - - -# ---------------------------------------------------------------------- # -# 矩阵 strategy.matrix -# ---------------------------------------------------------------------- # -class TestMatrixExpansion: - """测试矩阵扇出.""" - - def test_single_key_matrix(self) -> None: - """单键矩阵.""" - graph = parse_yaml_string( - """ -jobs: - test: - cmd: ["pytest"] - strategy: - matrix: - python: ["3.8", "3.9", "3.10"] -""", - ) - assert sorted(graph.names) == ["test_python-3.10", "test_python-3.8", "test_python-3.9"] - for name in graph.names: - assert graph.spec(name).cmd == ["pytest"] - - def test_multi_key_matrix(self) -> None: - """多键矩阵 (笛卡尔积).""" - graph = parse_yaml_string( - """ -jobs: - test: - cmd: ["pytest"] - strategy: - matrix: - python: ["3.8", "3.9"] - os: ["linux", "macos"] -""", - ) - assert len(graph.names) == 4 - expected = { - "test_python-3.8_os-linux", - "test_python-3.9_os-linux", - "test_python-3.8_os-macos", - "test_python-3.9_os-macos", - } - assert set(graph.names) == expected - - def test_matrix_placeholder_in_cmd(self) -> None: - """矩阵占位符替换 cmd.""" - graph = parse_yaml_string( - """ -jobs: - test: - cmd: ["python${{ matrix.version }}", "-m", "pytest"] - strategy: - matrix: - version: ["3.8", "3.9"] -""", - ) - specs = {s.name: s for s in [graph.spec(n) for n in graph.names]} - assert specs["test_version-3.8"].cmd == ["python3.8", "-m", "pytest"] - assert specs["test_version-3.9"].cmd == ["python3.9", "-m", "pytest"] - - def test_matrix_placeholder_in_run(self) -> None: - """矩阵占位符替换 run.""" - graph = parse_yaml_string( - """ -jobs: - test: - run: "python${{ matrix.version }} -m pytest" - strategy: - matrix: - version: ["3.8", "3.9"] -""", - ) - specs = {s.name: s for s in [graph.spec(n) for n in graph.names]} - assert specs["test_version-3.8"].cmd == "python3.8 -m pytest" - - def test_matrix_placeholder_in_cwd_env(self) -> None: - """矩阵占位符替换 cwd 和 env.""" - graph = parse_yaml_string( - """ -jobs: - test: - cmd: ["pytest"] - cwd: ./build/${{ matrix.os }} - env: - TARGET: ${{ matrix.os }} - strategy: - matrix: - os: ["linux", "macos"] -""", - ) - for name in graph.names: - spec = graph.spec(name) - assert spec.cwd is not None - assert "build" in str(spec.cwd) - assert spec.cwd.name in ("linux", "macos") - - def test_matrix_scalar_value(self) -> None: - """矩阵值为标量.""" - graph = parse_yaml_string( - """ -jobs: - test: - cmd: ["echo"] - strategy: - matrix: - mode: release -""", - ) - assert graph.names == ["test_mode-release"] - - def test_matrix_placeholder_undefined(self) -> None: - """占位符引用未定义的矩阵键应报错.""" - with pytest.raises(YamlLoadError, match="未定义"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo", "${{ matrix.undefined }}"] - strategy: - matrix: - mode: release -""", - ) - - def test_matrix_empty_list(self) -> None: - """矩阵值为空列表应报错.""" - with pytest.raises(YamlLoadError, match="不能为空列表"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - strategy: - matrix: - python: [] -""", - ) - - def test_strategy_not_mapping(self) -> None: - """strategy 不是 mapping 应报错.""" - with pytest.raises(YamlLoadError, match="strategy 必须是 mapping"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - strategy: "thread" -""", - ) - - def test_matrix_not_mapping(self) -> None: - """matrix 不是 mapping 应报错.""" - with pytest.raises(YamlLoadError, match="matrix 必须是 mapping"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - strategy: - matrix: [1, 2, 3] -""", - ) - - def test_matrix_value_invalid_type(self) -> None: - """矩阵值类型错误应报错.""" - with pytest.raises(YamlLoadError, match="必须是列表或标量"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - strategy: - matrix: - k: {a: 1} -""", - ) - - def test_matrix_special_chars_in_value(self) -> None: - """矩阵值含特殊字符应被清理.""" - graph = parse_yaml_string( - """ -jobs: - test: - cmd: ["echo"] - strategy: - matrix: - version: ["3.8/extra"] -""", - ) - assert "test_version-3.8_extra" in graph.names - - def test_matrix_preserves_needs(self) -> None: - """矩阵任务保留 needs 依赖.""" - graph = parse_yaml_string( - """ -jobs: - setup: - cmd: ["echo"] - test: - needs: [setup] - cmd: ["pytest"] - strategy: - matrix: - python: ["3.8", "3.9"] -""", - ) - for name in graph.names: - if name.startswith("test_"): - assert graph.spec(name).depends_on == ("setup",) - - -# ---------------------------------------------------------------------- # -# if 条件 -# ---------------------------------------------------------------------- # -class TestConditionParsing: - """测试 if 条件解析.""" - - def test_if_success(self) -> None: - """if: success() 返回空 condition 元组.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - if: "success()" -""", - ) - assert graph.spec("t").conditions == () - - def test_if_always(self) -> None: - """if: always() 返回空 condition 元组.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - if: "always()" -""", - ) - assert graph.spec("t").conditions == () - - def test_if_failure_unsupported(self) -> None: - """if: failure() 应报错.""" - with pytest.raises(YamlLoadError, match="不支持"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - if: "failure()" -""", - ) - - def test_if_env_exists(self) -> None: - """if: env.VAR 检查环境变量存在.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - if: "env.DEPLOY_TOKEN" -""", - ) - assert len(graph.spec("t").conditions) == 1 - - def test_if_env_equals(self) -> None: - """if: env.VAR == 'value' 检查环境变量等于.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - if: "env.ENV == 'prod'" -""", - ) - assert len(graph.spec("t").conditions) == 1 - - def test_if_env_not_equals(self) -> None: - """if: env.VAR != 'value' 检查环境变量不等于.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - if: "env.ENV != 'dev'" -""", - ) - assert len(graph.spec("t").conditions) == 1 - - def test_if_unsupported_expr(self) -> None: - """不支持的 if 表达式应报错.""" - with pytest.raises(YamlLoadError, match="不支持的 if 表达式"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - if: "github.event == 'push'" -""", - ) - - def test_if_not_string(self) -> None: - """if 非字符串应报错.""" - with pytest.raises(YamlLoadError, match="if 条件必须是字符串"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - if: 123 -""", - ) - - def test_if_empty_string(self) -> None: - """if 空字符串应报错.""" - with pytest.raises(YamlLoadError, match="if 条件不能为空"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - if: "" -""", - ) - - def test_if_env_exists_runtime(self, monkeypatch: pytest.MonkeyPatch) -> None: - """if: env.VAR 运行时检查.""" - monkeypatch.setenv("TEST_YAML_TOKEN", "abc") - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - if: "env.TEST_YAML_TOKEN" -""", - ) - spec = graph.spec("t") - assert spec.conditions[0]({}) is True - - def test_if_env_equals_runtime(self, monkeypatch: pytest.MonkeyPatch) -> None: - """if: env.VAR == 'value' 运行时检查.""" - monkeypatch.setenv("TEST_YAML_ENV", "prod") - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - if: "env.TEST_YAML_ENV == 'prod'" -""", - ) - spec = graph.spec("t") - assert spec.conditions[0]({}) is True - - def test_if_env_not_equals_runtime(self, monkeypatch: pytest.MonkeyPatch) -> None: - """if: env.VAR != 'value' 运行时检查.""" - monkeypatch.setenv("TEST_YAML_ENV2", "staging") - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - if: "env.TEST_YAML_ENV2 != 'prod'" -""", - ) - spec = graph.spec("t") - assert spec.conditions[0]({}) is True - - -# ---------------------------------------------------------------------- # -# retry / timeout / cwd / env / verbose 等字段 -# ---------------------------------------------------------------------- # -class TestTaskFields: - """测试任务配置字段.""" - - def test_timeout(self) -> None: - """timeout 字段.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - timeout: 300 -""", - ) - assert graph.spec("t").timeout == 300.0 - - def test_retry_full(self) -> None: - """retry 完整配置.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - retry: - max_attempts: 3 - delay: 1.0 - backoff: 2.0 - jitter: 0.5 -""", - ) - retry = graph.spec("t").retry - assert retry.max_attempts == 3 - assert retry.delay == 1.0 - assert retry.backoff == 2.0 - assert retry.jitter == 0.5 - - def test_retry_with_hyphen(self) -> None: - """retry max-attempts 连字符形式.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - retry: {max-attempts: 5} -""", - ) - assert graph.spec("t").retry.max_attempts == 5 - - def test_retry_invalid_type(self) -> None: - """retry 类型错误应报错.""" - with pytest.raises(YamlLoadError, match="retry 必须是 mapping"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - retry: "invalid" -""", - ) - - def test_cwd(self) -> None: - """cwd 字段.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - cwd: ./build -""", - ) - assert graph.spec("t").cwd == Path("./build") - - def test_env(self) -> None: - """env 字段.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - env: - DEBUG: "1" - VERBOSE: "true" -""", - ) - env = graph.spec("t").env - assert env == {"DEBUG": "1", "VERBOSE": "true"} - - def test_verbose(self) -> None: - """verbose 字段.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - verbose: true -""", - ) - assert graph.spec("t").verbose is True - - def test_continue_on_error(self) -> None: - """continue-on-error 字段 (连字符).""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - continue-on-error: true -""", - ) - assert graph.spec("t").continue_on_error is True - - def test_continue_on_error_underscore(self) -> None: - """continue_on_error 字段 (下划线).""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - continue_on_error: true -""", - ) - assert graph.spec("t").continue_on_error is True - - def test_skip_if_missing(self) -> None: - """skip-if-missing 字段.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - skip-if-missing: true -""", - ) - assert graph.spec("t").skip_if_missing is True - - def test_allow_upstream_skip(self) -> None: - """allow-upstream-skip 字段.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - allow-upstream-skip: true -""", - ) - assert graph.spec("t").allow_upstream_skip is True - - def test_priority(self) -> None: - """priority 字段.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - priority: 10 -""", - ) - assert graph.spec("t").priority == 10 - - def test_concurrency_key(self) -> None: - """concurrency-key 字段.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - concurrency-key: deploy_lock -""", - ) - assert graph.spec("t").concurrency_key == "deploy_lock" - - def test_tags(self) -> None: - """tags 字段.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - tags: [ci, deploy] -""", - ) - assert graph.spec("t").tags == ("ci", "deploy") - - def test_runs_on_appended_to_tags(self) -> None: - """runs-on 追加到 tags.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - tags: [ci] - runs-on: linux -""", - ) - assert graph.spec("t").tags == ("ci", "linux") - - def test_runs_on_only(self) -> None: - """仅有 runs-on.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - runs-on: macos -""", - ) - assert graph.spec("t").tags == ("macos",) - - -# ---------------------------------------------------------------------- # -# 顶层 strategy 和 defaults -# ---------------------------------------------------------------------- # -class TestTopLevelConfig: - """测试顶层配置.""" - - def test_top_strategy(self) -> None: - """顶层 strategy 设置图级默认策略.""" - graph = parse_yaml_string( - """ -strategy: thread -jobs: - t: - cmd: ["echo"] -""", - ) - assert graph.defaults.strategy == "thread" - - def test_top_strategy_overrides_defaults(self) -> None: - """顶层 strategy 覆盖 defaults.strategy.""" - graph = parse_yaml_string( - """ -strategy: async -defaults: - strategy: thread -jobs: - t: - cmd: ["echo"] -""", - ) - assert graph.defaults.strategy == "async" - - def test_top_strategy_invalid_type(self) -> None: - """顶层 strategy 非字符串应报错.""" - with pytest.raises(YamlLoadError, match="顶层 strategy 必须是字符串"): - parse_yaml_string( - """ -strategy: [thread] -jobs: - t: - cmd: ["echo"] -""", - ) - - def test_defaults_retry(self) -> None: - """defaults.retry 设置图级重试策略.""" - graph = parse_yaml_string( - """ -defaults: - retry: {max_attempts: 5} -jobs: - t: - cmd: ["echo"] -""", - ) - assert graph.defaults.retry is not None - assert graph.defaults.retry.max_attempts == 5 - - def test_defaults_verbose(self) -> None: - """defaults.verbose 设置图级 verbose.""" - graph = parse_yaml_string( - """ -defaults: - verbose: true -jobs: - t: - cmd: ["echo"] -""", - ) - assert graph.defaults.verbose is True - - def test_defaults_env(self) -> None: - """defaults.env 设置图级环境变量.""" - graph = parse_yaml_string( - """ -defaults: - env: {GLOBAL: "1"} -jobs: - t: - cmd: ["echo"] -""", - ) - assert graph.defaults.env == {"GLOBAL": "1"} - - def test_defaults_timeout(self) -> None: - """defaults.timeout 设置图级超时.""" - graph = parse_yaml_string( - """ -defaults: - timeout: 100 -jobs: - t: - cmd: ["echo"] -""", - ) - assert graph.defaults.timeout == 100.0 - - def test_defaults_priority(self) -> None: - """defaults.priority 设置图级优先级.""" - graph = parse_yaml_string( - """ -defaults: - priority: 5 -jobs: - t: - cmd: ["echo"] -""", - ) - assert graph.defaults.priority == 5 - - def test_defaults_continue_on_error(self) -> None: - """defaults.continue-on-error 设置图级默认.""" - graph = parse_yaml_string( - """ -defaults: - continue-on-error: true -jobs: - t: - cmd: ["echo"] -""", - ) - assert graph.defaults.continue_on_error is True - - def test_defaults_concurrency_key(self) -> None: - """defaults.concurrency-key 设置图级并发键.""" - graph = parse_yaml_string( - """ -defaults: - concurrency-key: global_lock -jobs: - t: - cmd: ["echo"] -""", - ) - assert graph.defaults.concurrency_key == "global_lock" - - def test_defaults_tags(self) -> None: - """defaults.tags 设置图级标签.""" - graph = parse_yaml_string( - """ -defaults: - tags: [common] -jobs: - t: - cmd: ["echo"] -""", - ) - assert graph.defaults.tags == ("common",) - - def test_defaults_cwd(self) -> None: - """defaults.cwd 设置图级工作目录.""" - graph = parse_yaml_string( - """ -defaults: - cwd: ./root -jobs: - t: - cmd: ["echo"] -""", - ) - assert graph.defaults.cwd == Path("./root") - - -# ---------------------------------------------------------------------- # -# 错误处理: 重复任务名/循环依赖 -# ---------------------------------------------------------------------- # -class TestErrorHandling: - """测试错误处理.""" - - def test_duplicate_task_name(self) -> None: - """重复任务名应报错.""" - with pytest.raises(YamlLoadError, match="重复任务名"): - parse_yaml_string( - """ -jobs: - a: - cmd: ["echo"] - strategy: - matrix: - k: ["x"] - a_k-x: - cmd: ["echo"] -""", - ) - - def test_cycle_dependency(self) -> None: - """循环依赖应报错 (由 Graph 抛出).""" - with pytest.raises(px.CycleError): - parse_yaml_string( - """ -jobs: - a: - needs: [b] - cmd: ["echo"] - b: - needs: [a] - cmd: ["echo"] -""", - ) - - def test_missing_dependency(self) -> None: - """依赖不存在的任务应报错.""" - with pytest.raises(px.MissingDependencyError): - parse_yaml_string( - """ -jobs: - a: - needs: [nonexistent] - cmd: ["echo"] -""", - ) - - -# ---------------------------------------------------------------------- # -# 字段类型转换错误 -# ---------------------------------------------------------------------- # -class TestFieldTypeErrors: - """测试字段类型转换错误.""" - - def test_timeout_invalid_type(self) -> None: - """timeout 非数字应报错.""" - with pytest.raises(YamlLoadError, match="期望数字"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - timeout: "abc" -""", - ) - - def test_priority_invalid_type(self) -> None: - """priority 非整数应报错.""" - with pytest.raises(YamlLoadError, match="期望整数"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - priority: "high" -""", - ) - - def test_tags_invalid_type(self) -> None: - """tags 非列表/字符串应报错.""" - with pytest.raises(YamlLoadError, match="期望字符串或列表"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - tags: 123 -""", - ) - - def test_env_invalid_type(self) -> None: - """env 非 mapping 应报错.""" - with pytest.raises(YamlLoadError, match="期望 mapping"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - env: [1, 2, 3] -""", - ) - - def test_runs_on_invalid_type(self) -> None: - """runs-on 非字符串应报错.""" - with pytest.raises(YamlLoadError, match="期望字符串"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - runs-on: [linux] -""", - ) - - def test_concurrency_key_invalid_type(self) -> None: - """concurrency-key 非字符串应报错.""" - with pytest.raises(YamlLoadError, match="期望字符串"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - concurrency-key: 123 -""", - ) - - -# ---------------------------------------------------------------------- # -# 完整流水线集成测试 -# ---------------------------------------------------------------------- # -class TestIntegrationPipeline: - """完整流水线集成测试.""" - - def test_full_ci_pipeline(self) -> None: - """完整 CI 流水线.""" - graph = parse_yaml_string( - """ -strategy: thread -defaults: - verbose: true - env: - CI: "true" - -jobs: - setup: - cmd: ["echo", "setup"] - - build: - needs: [setup] - cmd: ["python", "-m", "build"] - timeout: 300 - retry: {max_attempts: 2} - - test: - needs: [build] - cmd: ["pytest", "-x"] - strategy: - matrix: - python: ["3.8", "3.9", "3.10"] - - lint: - needs: [build] - cmd: ["ruff", "check"] - if: "env.CI" - - deploy: - needs: [test, lint] - cmd: ["twine", "upload"] - if: "env.DEPLOY_TOKEN != ''" - allow-upstream-skip: true -""", - ) - # 验证任务数 - assert len(graph.names) == 1 + 1 + 3 + 1 + 1 # setup/build/test*3/lint/deploy = 7 - # 验证分层 - layers = graph.layers() - assert layers[0] == ["setup"] - assert layers[1] == ["build"] - # 第 3 层含 4 个并行任务 (test*3 + lint) - assert len(layers[2]) == 4 - # deploy 在最后 - assert layers[-1] == ["deploy"] - - def test_dry_run_layers(self) -> None: - """dry-run 验证分层.""" - graph = parse_yaml_string( - """ -jobs: - a: - cmd: ["echo"] - b: - cmd: ["echo"] - c: - needs: [a, b] - cmd: ["echo"] -""", - ) - assert graph.layers() == [["a", "b"], ["c"]] - - def test_execute_simple_pipeline(self) -> None: - """执行简单流水线验证集成正确性.""" - graph = parse_yaml_string( - """ -jobs: - hello: - cmd: ["python", "-c", "print('hello world')"] -""", - ) - report = px.run(graph, strategy="sequential") - assert report.success - assert report.result_of("hello").status == TaskStatus.SUCCESS - - def test_execute_aggregate_job(self) -> None: - """方向 B: 聚合 job (有 needs 无 cmd/fn) 在 run() 中应正常执行.""" - graph = parse_yaml_string( - """ -jobs: - a: - cmd: ["python", "-c", "print('a')"] - b: - cmd: ["python", "-c", "print('b')"] - all: - needs: [a, b] -""", - ) - report = px.run(graph, strategy="sequential") - assert report.success - assert report.result_of("a").status == TaskStatus.SUCCESS - assert report.result_of("b").status == TaskStatus.SUCCESS - assert report.result_of("all").status == TaskStatus.SUCCESS - assert report["all"] is None - - def test_execute_with_failure(self) -> None: - """执行含失败任务的流水线.""" - graph = parse_yaml_string( - """ -jobs: - ok: - cmd: ["python", "-c", "exit(0)"] - fail: - cmd: ["python", "-c", "exit(1)"] - continue-on-error: true -""", - ) - report = px.run(graph, strategy="sequential") - # fail 任务 continue-on-error, 不影响整图 - assert report.success - - def test_execute_skipped_due_to_missing_dep(self) -> None: - """依赖跳过时下游也跳过.""" - graph = parse_yaml_string( - """ -jobs: - skip_me: - cmd: ["nonexistent_cmd_xyz"] - skip-if-missing: true - downstream: - needs: [skip_me] - cmd: ["python", "-c", "print()"] - allow-upstream-skip: true -""", - ) - report = px.run(graph, strategy="sequential") - # skip_me 被跳过, downstream 因 allow-upstream-skip 仍执行 - assert report.result_of("skip_me").status == TaskStatus.SKIPPED - - -# ---------------------------------------------------------------------- # -# fn 字段 + 变量占位符 -# ---------------------------------------------------------------------- # -class TestFnField: - """测试 fn 字段引用注册函数.""" - - def setup_method(self) -> None: - """每个测试前清空 registry.""" - px.FnRegistry.clear() - - def teardown_method(self) -> None: - """每个测试后清空 registry.""" - px.FnRegistry.clear() - - def test_fn_basic(self) -> None: - """fn 引用注册函数.""" - - @px.register_fn("greet") - def _greet(name: str) -> str: - return f"hello {name}" - - graph = parse_yaml_string( - """ -jobs: - greet: - fn: greet - args: ["world"] -""", - ) - report = px.run(graph, strategy="sequential") - assert report.success - assert report["greet"] == "hello world" - - def test_fn_with_kwargs(self) -> None: - """fn 使用 kwargs 传参.""" - - @px.register_fn("add") - def _add(a: int, b: int) -> int: - return a + b - - graph = parse_yaml_string( - """ -jobs: - add: - fn: add - args: [1, 2] -""", - ) - report = px.run(graph, strategy="sequential") - assert report["add"] == 3 - - def test_fn_not_registered(self) -> None: - """fn 引用未注册的函数应报错.""" - with pytest.raises(YamlLoadError, match="未注册"): - parse_yaml_string( - """ -jobs: - t: - fn: nonexistent_fn -""", - ) - - def test_fn_not_string(self) -> None: - """fn 不是字符串应报错.""" - with pytest.raises(YamlLoadError, match="fn 必须是字符串"): - parse_yaml_string( - """ -jobs: - t: - fn: 123 -""", - ) - - def test_fn_args_not_list(self) -> None: - """args 不是列表应报错.""" - with pytest.raises(YamlLoadError, match="args 必须是列表"): - parse_yaml_string( - """ -jobs: - t: - fn: some_fn - args: "not a list" -""", - ) - - def test_fn_kwargs_not_mapping(self) -> None: - """kwargs 不是 mapping 应报错.""" - with pytest.raises(YamlLoadError, match="kwargs 必须是 mapping"): - parse_yaml_string( - """ -jobs: - t: - fn: some_fn - kwargs: [1, 2] -""", - ) - - def test_fn_with_cmd_conflict(self) -> None: - """fn 与 cmd 同时提供应报错.""" - with pytest.raises(YamlLoadError, match="不能同时提供"): - parse_yaml_string( - """ -jobs: - t: - fn: some_fn - cmd: ["echo"] -""", - ) - - def test_fn_with_needs(self) -> None: - """fn 任务可以有 needs 依赖.""" - - @px.register_fn("upper") - def _upper(s: str) -> str: - return s.upper() - - graph = parse_yaml_string( - """ -jobs: - setup: - cmd: ["python", "-c", "print('hello')"] - process: - fn: upper - args: ["data"] - needs: [setup] -""", - ) - assert graph.spec("process").depends_on == ("setup",) - report = px.run(graph, strategy="sequential") - assert report["process"] == "DATA" - - def test_fn_matrix_placeholder_in_args(self) -> None: - """fn 的 args 支持矩阵占位符.""" - - @px.register_fn("echo_val") - def _echo_val(val: str) -> str: - return val - - graph = parse_yaml_string( - """ -jobs: - test: - fn: echo_val - args: ["${{ matrix.mode }}"] - strategy: - matrix: - mode: [release, debug] -""", - ) - assert sorted(graph.names) == ["test_mode-debug", "test_mode-release"] - report = px.run(graph, strategy="sequential") - assert report["test_mode-release"] == "release" - assert report["test_mode-debug"] == "debug" - - -# ---------------------------------------------------------------------- # -# 变量占位符 ${VAR} -# ---------------------------------------------------------------------- # -class TestVariablePlaceholder: - """测试 ${VAR} 变量占位符.""" - - def test_variable_in_cmd(self) -> None: - """变量在 cmd 中替换.""" - graph = parse_yaml_string( - """ -jobs: - echo: - cmd: ["echo", "${MSG}"] -""", - variables={"MSG": "hello"}, - ) - assert graph.spec("echo").cmd == ["echo", "hello"] - - def test_variable_in_run(self) -> None: - """变量在 run 中替换.""" - graph = parse_yaml_string( - """ -jobs: - t: - run: "echo ${MSG} && exit 0" -""", - variables={"MSG": "hi"}, - ) - assert graph.spec("t").cmd == "echo hi && exit 0" - - def test_variable_in_cwd(self) -> None: - """变量在 cwd 中替换.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["pwd"] - cwd: "${BASE_DIR}/sub" -""", - variables={"BASE_DIR": "/tmp"}, - ) - spec = graph.spec("t") - assert spec.cwd is not None - # 跨平台: Windows 上 Path 会把 / 转为 \, 统一用正斜杠比较 - assert "tmp" in str(spec.cwd).replace("\\", "/") - assert "sub" in str(spec.cwd).replace("\\", "/") - - def test_variable_in_env(self) -> None: - """变量在 env 中替换.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo"] - env: - PATH: "${BIN_PATH}" -""", - variables={"BIN_PATH": "/usr/local/bin"}, - ) - assert graph.spec("t").env == {"PATH": "/usr/local/bin"} - - def test_variable_in_fn_args(self) -> None: - """变量在 fn 的 args 中替换.""" - - @px.register_fn("concat") - def _concat(a: str, b: str) -> str: - return a + b - - graph = parse_yaml_string( - """ -jobs: - t: - fn: concat - args: ["${A}", "${B}"] -""", - variables={"A": "foo", "B": "bar"}, - ) - report = px.run(graph, strategy="sequential") - assert report["t"] == "foobar" - - def test_variable_undefined(self) -> None: - """引用未定义变量应报错.""" - with pytest.raises(YamlLoadError, match="变量 'UNDEFINED' 未定义"): - parse_yaml_string( - """ -jobs: - t: - cmd: ["echo", "${UNDEFINED}"] -""", - variables={"OTHER": "x"}, - ) - - def test_multiple_variables(self) -> None: - """多个变量同时替换.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo", "${A}", "${B}", "${A}"] -""", - variables={"A": "x", "B": "y"}, - ) - assert graph.spec("t").cmd == ["echo", "x", "y", "x"] - - def test_variable_no_variables_provided(self) -> None: - """未提供 variables 时, ${VAR} 原样保留.""" - graph = parse_yaml_string( - """ -jobs: - t: - cmd: ["echo", "${VAR}"] -""", - ) - # 没有 variables, 占位符保留 - assert graph.spec("t").cmd == ["echo", "${VAR}"] - - def test_variable_and_matrix_combined(self) -> None: - """变量和矩阵占位符同时替换.""" - graph = parse_yaml_string( - """ -jobs: - test: - cmd: ["echo", "${PREFIX}", "${{ matrix.mode }}"] - strategy: - matrix: - mode: [release, debug] -""", - variables={"PREFIX": "build"}, - ) - for name in graph.names: - spec = graph.spec(name) - cmd = spec.cmd - assert isinstance(cmd, list) - assert cmd[1] == "build" - assert cmd[2] in ("release", "debug") - - def test_load_yaml_with_variables(self, tmp_path: Path) -> None: - """load_yaml 支持 variables 参数.""" - yaml_file = tmp_path / "pipeline.yaml" - yaml_file.write_text( - 'jobs:\n t:\n cmd: ["echo", "${MSG}"]\n', - encoding="utf-8", - ) - graph = load_yaml(yaml_file, variables={"MSG": "loaded"}) - assert graph.spec("t").cmd == ["echo", "loaded"] - - def test_graph_from_yaml_with_variables(self, tmp_path: Path) -> None: - """Graph.from_yaml 支持 variables 参数.""" - yaml_file = tmp_path / "pipeline.yaml" - yaml_file.write_text( - 'jobs:\n t:\n cmd: ["echo", "${MSG}"]\n', - encoding="utf-8", - ) - graph = px.Graph.from_yaml(yaml_file, variables={"MSG": "from_class"}) - assert graph.spec("t").cmd == ["echo", "from_class"] - - -# ---------------------------------------------------------------------- # -# YamlCliRunner -# ---------------------------------------------------------------------- # -class TestYamlCliRunner: - """``YamlCliRunner`` 类测试.""" - - def _write_yaml(self, tmp_path: Path, content: str) -> Path: - """写入临时 YAML 文件.""" - yaml_file = tmp_path / "tool.yaml" - yaml_file.write_text(content, encoding="utf-8") - return yaml_file - - def test_list_with_subcommands(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - """``--list`` 在 subcommands 模式下应打印任务列表而非 help.""" - from pyflowx.yaml_loader import YamlCliRunner - - yaml_file = self._write_yaml( - tmp_path, - """ -cli: - description: "test" - subcommands: - a: {help: "task a"} - all: {help: "aggregate"} -jobs: - a: - cmd: ["echo", "a"] - all: - needs: [a] -""", - ) - rc = YamlCliRunner(yaml_file, ["--list"]).run() - assert rc == 0 - captured = capsys.readouterr().out - assert "任务列表" in captured - assert "a" in captured - assert "all" in captured - - def test_no_args_with_subcommands_prints_help(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - """subcommands 模式下未指定 command 应打印 help.""" - from pyflowx.yaml_loader import YamlCliRunner - - yaml_file = self._write_yaml( - tmp_path, - """ -cli: - description: "test" - subcommands: - a: {help: "task a"} -jobs: - a: - cmd: ["echo", "a"] -""", - ) - rc = YamlCliRunner(yaml_file, []).run() - assert rc == 0 - captured = capsys.readouterr().out - assert "可用命令" in captured - - def test_dry_run_executes(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - """``--dry-run`` 应打印执行计划不实际执行.""" - from pyflowx.yaml_loader import YamlCliRunner - - yaml_file = self._write_yaml( - tmp_path, - """ -cli: - description: "test" - subcommands: - a: {help: "task a"} -jobs: - a: - cmd: ["echo", "a"] -""", - ) - rc = YamlCliRunner(yaml_file, ["a", "--dry-run"]).run() - assert rc == 0 - captured = capsys.readouterr().out - assert "Dry run" in captured - assert "Layer" in captured - - def test_aggregate_job_dry_run(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - """方向 B: 聚合 job 在 YamlCliRunner 中应正常 dry-run.""" - from pyflowx.yaml_loader import YamlCliRunner - - yaml_file = self._write_yaml( - tmp_path, - """ -cli: - description: "test" - subcommands: - all: {help: "aggregate"} -jobs: - a: - cmd: ["echo", "a"] - b: - cmd: ["echo", "b"] - all: - needs: [a, b] -""", - ) - rc = YamlCliRunner(yaml_file, ["all", "--dry-run"]).run() - assert rc == 0 - captured = capsys.readouterr().out - assert "all" in captured - - def test_file_not_exists(self, capsys: pytest.CaptureFixture[str]) -> None: - """文件不存在应返回 1 并打印错误.""" - from pyflowx.yaml_loader import YamlCliRunner - - rc = YamlCliRunner("/nonexistent/path.yaml", []).run() - assert rc == 1 - captured = capsys.readouterr().err - assert "不存在" in captured - - def test_extract_variables_from_positional(self, tmp_path: Path) -> None: - """位置参数应转为变量并替换 ``${VAR}``.""" - from pyflowx.yaml_loader import YamlCliRunner - - yaml_file = self._write_yaml( - tmp_path, - """ -cli: - description: "test" - positional: - - name: MSG - help: "message" -jobs: - echo: - cmd: ["echo", "${MSG}"] -""", - ) - rc = YamlCliRunner(yaml_file, ["hello", "--dry-run"]).run() - assert rc == 0 - - def test_invalid_yaml_returns_1(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - """YAML 格式错误应返回 1.""" - from pyflowx.yaml_loader import YamlCliRunner - - yaml_file = tmp_path / "bad.yaml" - yaml_file.write_text(":\n - invalid: [", encoding="utf-8") - rc = YamlCliRunner(yaml_file, []).run() - assert rc == 1 - - def test_non_mapping_root_returns_1(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - """YAML 根节点非 mapping 应返回 1.""" - from pyflowx.yaml_loader import YamlCliRunner - - yaml_file = self._write_yaml(tmp_path, "- item1\n- item2\n") - rc = YamlCliRunner(yaml_file, []).run() - assert rc == 1 - assert "mapping" in capsys.readouterr().err - - def test_cli_section_invalid_returns_1(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - """cli 段格式错误应返回 1.""" - from pyflowx.yaml_loader import YamlCliRunner - - yaml_file = self._write_yaml(tmp_path, "cli: [1, 2]\njobs:\n a:\n cmd: [echo]\n") - rc = YamlCliRunner(yaml_file, []).run() - assert rc == 1 - assert "cli" in capsys.readouterr().err - - def test_task_failed_returns_1(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - """任务失败应返回 1.""" - from pyflowx.yaml_loader import YamlCliRunner - - yaml_file = self._write_yaml( - tmp_path, - """ -cli: - description: "test" - subcommands: - a: {help: "task a"} -jobs: - a: - cmd: ["nonexistent_cmd_xyz_123"] - skip-if-missing: false -""", - ) - rc = YamlCliRunner(yaml_file, ["a"]).run() - assert rc == 1 - assert "错误" in capsys.readouterr().err - - def test_load_yaml_os_error(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """load_yaml 读取文件失败应抛出 YamlLoadError. - - Windows 上 chmod(0o000) 不可靠, 改用 monkeypatch 模拟 OSError 覆盖该分支. - """ - from pyflowx.yaml_loader import YamlLoadError, load_yaml - - yaml_file = tmp_path / "unreadable.yaml" - yaml_file.write_text("jobs:\n a:\n cmd: [echo]\n", encoding="utf-8") - - def _raise_oserror(self: Path, *args: object, **kwargs: object) -> str: - raise OSError("permission denied") - - monkeypatch.setattr(Path, "read_text", _raise_oserror) - - with pytest.raises(YamlLoadError, match="读取文件失败"): - load_yaml(yaml_file) diff --git a/uv.lock b/uv.lock index 623f3e7..147cb6a 100644 --- a/uv.lock +++ b/uv.lock @@ -3240,7 +3240,6 @@ version = "0.4.7" source = { editable = "." } dependencies = [ { name = "graphlib-backport", marker = "python_full_version < '3.9'" }, - { name = "pyyaml" }, { name = "typing-extensions", version = "4.13.2", source = { registry = "https://mirrors.aliyun.com/pypi/simple/" }, marker = "python_full_version < '3.9'" }, { name = "typing-extensions", version = "4.15.0", source = { registry = "https://mirrors.aliyun.com/pypi/simple/" }, marker = "python_full_version >= '3.9' and python_full_version < '3.13'" }, ] @@ -3274,9 +3273,6 @@ dev = [ { name = "tox-uv", version = "1.13.1", source = { registry = "https://mirrors.aliyun.com/pypi/simple/" }, marker = "python_full_version < '3.9'" }, { name = "tox-uv", version = "1.28.1", source = { registry = "https://mirrors.aliyun.com/pypi/simple/" }, marker = "python_full_version == '3.9.*'" }, { name = "tox-uv", version = "1.35.2", source = { registry = "https://mirrors.aliyun.com/pypi/simple/" }, marker = "python_full_version >= '3.10'" }, - { name = "types-pyyaml", version = "6.0.12.20241230", source = { registry = "https://mirrors.aliyun.com/pypi/simple/" }, marker = "python_full_version < '3.9'" }, - { name = "types-pyyaml", version = "6.0.12.20250915", source = { registry = "https://mirrors.aliyun.com/pypi/simple/" }, marker = "python_full_version == '3.9.*'" }, - { name = "types-pyyaml", version = "6.0.12.20260518", source = { registry = "https://mirrors.aliyun.com/pypi/simple/" }, marker = "python_full_version >= '3.10'" }, ] docs = [ { name = "myst-parser", version = "3.0.1", source = { registry = "https://mirrors.aliyun.com/pypi/simple/" }, marker = "python_full_version < '3.10'" }, @@ -3323,13 +3319,11 @@ requires-dist = [ { name = "pytest-html", marker = "extra == 'dev'", specifier = ">=4.1.1" }, { name = "pytest-mock", marker = "extra == 'dev'", specifier = ">=3.14.0" }, { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.6.1" }, - { name = "pyyaml", specifier = ">=6.0.1" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.8.0" }, { name = "sphinx", marker = "extra == 'docs'", specifier = ">=7.0" }, { name = "sphinx-rtd-theme", marker = "extra == 'docs'", specifier = ">=2.0" }, { name = "tox", marker = "extra == 'dev'", specifier = ">=4.25.0" }, { name = "tox-uv", marker = "extra == 'dev'", specifier = ">=1.13.1" }, - { name = "types-pyyaml", marker = "extra == 'dev'", specifier = ">=6.0.12" }, { name = "typing-extensions", marker = "python_full_version < '3.13'", specifier = ">=4.13.2" }, ] provides-extras = ["dev", "docs", "office"] @@ -5407,74 +5401,6 @@ wheels = [ { url = "https://mirrors.aliyun.com/pypi/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981" }, ] -[[package]] -name = "types-pyyaml" -version = "6.0.12.20241230" -source = { registry = "https://mirrors.aliyun.com/pypi/simple/" } -resolution-markers = [ - "python_full_version >= '3.8.10' and python_full_version < '3.9' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.8.10' and python_full_version < '3.9' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version >= '3.8.10' and python_full_version < '3.9' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.8.10' and python_full_version < '3.9' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.8.10' and python_full_version < '3.9' and sys_platform != 'darwin' and sys_platform != 'linux')", - "python_full_version >= '3.8.1' and python_full_version < '3.8.10'", - "python_full_version < '3.8.1'", -] -sdist = { url = "https://mirrors.aliyun.com/pypi/packages/9a/f9/4d566925bcf9396136c0a2e5dc7e230ff08d86fa011a69888dd184469d80/types_pyyaml-6.0.12.20241230.tar.gz", hash = "sha256:7f07622dbd34bb9c8b264fe860a17e0efcad00d50b5f27e93984909d9363498c" } -wheels = [ - { url = "https://mirrors.aliyun.com/pypi/packages/e8/c1/48474fbead512b70ccdb4f81ba5eb4a58f69d100ba19f17c92c0c4f50ae6/types_PyYAML-6.0.12.20241230-py3-none-any.whl", hash = "sha256:fa4d32565219b68e6dee5f67534c722e53c00d1cfc09c435ef04d7353e1e96e6" }, -] - -[[package]] -name = "types-pyyaml" -version = "6.0.12.20250915" -source = { registry = "https://mirrors.aliyun.com/pypi/simple/" } -resolution-markers = [ - "python_full_version > '3.9' and python_full_version < '3.10'", - "python_full_version == '3.9'", -] -sdist = { url = "https://mirrors.aliyun.com/pypi/packages/7e/69/3c51b36d04da19b92f9e815be12753125bd8bc247ba0470a982e6979e71c/types_pyyaml-6.0.12.20250915.tar.gz", hash = "sha256:0f8b54a528c303f0e6f7165687dd33fafa81c807fcac23f632b63aa624ced1d3" } -wheels = [ - { url = "https://mirrors.aliyun.com/pypi/packages/bd/e0/1eed384f02555dde685fff1a1ac805c1c7dcb6dd019c916fe659b1c1f9ec/types_pyyaml-6.0.12.20250915-py3-none-any.whl", hash = "sha256:e7d4d9e064e89a3b3cae120b4990cd370874d2bf12fa5f46c97018dd5d3c9ab6" }, -] - -[[package]] -name = "types-pyyaml" -version = "6.0.12.20260518" -source = { registry = "https://mirrors.aliyun.com/pypi/simple/" } -resolution-markers = [ - "python_full_version >= '3.15' and sys_platform == 'darwin'", - "python_full_version >= '3.15' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "python_full_version >= '3.15' and sys_platform == 'win32'", - "python_full_version >= '3.15' and sys_platform == 'emscripten'", - "(python_full_version >= '3.15' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.15' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", - "python_full_version == '3.14.*' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform == 'darwin'", - "python_full_version == '3.14.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "python_full_version == '3.14.*' and sys_platform == 'win32'", - "python_full_version == '3.14.*' and sys_platform == 'emscripten'", - "(python_full_version == '3.14.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.14.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", - "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "(python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", - "python_full_version == '3.12.*' and sys_platform == 'win32'", - "python_full_version == '3.12.*' and sys_platform == 'emscripten'", - "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", - "python_full_version == '3.11.*' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and sys_platform == 'emscripten'", - "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", - "python_full_version == '3.10.*' and sys_platform == 'darwin'", - "python_full_version == '3.10.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version == '3.10.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.10.*' and sys_platform != 'darwin' and sys_platform != 'linux')", -] -sdist = { url = "https://mirrors.aliyun.com/pypi/packages/b8/83/4a1afc3fbfcf5b8d46fc390cd95ed6b0dc9010a265f4e9f46314efffa37a/types_pyyaml-6.0.12.20260518.tar.gz", hash = "sha256:d917f83fb38462550338c1297faedd860b3ec83912b96b1e3d73255f7473e466" } -wheels = [ - { url = "https://mirrors.aliyun.com/pypi/packages/06/a2/c01db32be2ae7d6a1689972f3c492b149ee4e164b12fdfd9f64b50888215/types_pyyaml-6.0.12.20260518-py3-none-any.whl", hash = "sha256:d2150f75a231c9fe9c7463bd29487d93e60bac90400287351384bc2284eba7cd" }, -] - [[package]] name = "typing-extensions" version = "4.13.2"