zhou
8d626f606e
CI / Lint, Typecheck & Test (push) Has been cancelled
pf graph 支持 ASCII 分层树/Mermaid/list/deps 四种输出格式； yaml_loader._safe_load 包装 YAMLError 为 ValueError 统一错误处理。
396 lines
13 KiB
Python
396 lines
13 KiB
Python
"""YAML 任务编排（GitHub Actions 风格）。
|
|
|
|
支持 ``jobs``/``needs``/``strategy.matrix``/``if``/``continue-on-error``/
|
|
``env``/``defaults`` 等 CI/CD 概念，从 YAML 文件直接加载为 :class:`Graph`。
|
|
|
|
Schema
|
|
------
|
|
.. code-block:: yaml
|
|
|
|
strategy: thread # 图级默认策略
|
|
defaults: # 图级默认值
|
|
retry: {max_attempts: 3}
|
|
verbose: true
|
|
env: {CI: "true"}
|
|
|
|
jobs:
|
|
setup:
|
|
cmd: ["git", "clone", "..."]
|
|
runs-on: linux
|
|
|
|
build:
|
|
needs: [setup]
|
|
cmd: ["python", "-m", "build"]
|
|
timeout: 300
|
|
|
|
test:
|
|
needs: [build]
|
|
cmd: ["python${{ matrix.version }}", "-m", "pytest"]
|
|
strategy:
|
|
matrix:
|
|
version: ["3.10", "3.11", "3.12"]
|
|
if: "env.CI"
|
|
|
|
字段映射
|
|
--------
|
|
| YAML 字段 | TaskSpec 字段 | 说明 |
|
|
|-----------|---------------|------|
|
|
| cmd / run | cmd | cmd 为列表，run 为 shell 字符串 |
|
|
| needs | depends_on | 矩阵依赖自动展开 |
|
|
| if | conditions | success/always/env.VAR/env.VAR==/env.VAR!= |
|
|
| strategy.matrix | 矩阵扇出 | 笛卡尔积展开 |
|
|
| ${{ matrix.key }} | 占位符 | cmd/run/cwd/env 中替换 |
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from collections.abc import Mapping
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml # type: ignore[import-not-found]
|
|
|
|
from .conditions import BuiltinConditions, Condition
|
|
from .graph import Graph, GraphDefaults
|
|
from .task import RetryPolicy, TaskSpec
|
|
|
|
__all__ = ["load_yaml", "parse_yaml_string"]
|
|
|
|
_MATRIX_PATTERN = re.compile(r"\$\{\{\s*matrix\.(\w+)\s*\}\}")
|
|
|
|
|
|
def _safe_load(text: str) -> Any:
|
|
"""调用 ``yaml.safe_load`` 并将 YAMLError 包装为 ValueError。
|
|
|
|
便于调用方统一捕获 ``ValueError`` 处理所有 YAML 解析失败场景。
|
|
"""
|
|
try:
|
|
return yaml.safe_load(text)
|
|
except yaml.YAMLError as exc:
|
|
raise ValueError(f"YAML 解析失败: {exc}") from exc
|
|
|
|
|
|
def parse_yaml_string(text: str) -> Graph:
|
|
"""从 YAML 字符串解析任务图。
|
|
|
|
Parameters
|
|
----------
|
|
text:
|
|
YAML 格式的字符串。
|
|
|
|
Returns
|
|
-------
|
|
Graph
|
|
解析后的任务图。
|
|
|
|
Raises
|
|
------
|
|
ValueError
|
|
YAML 结构不符合 schema 时。
|
|
"""
|
|
data = _safe_load(text)
|
|
return _build_graph(data)
|
|
|
|
|
|
def load_yaml(path: str | Path) -> Graph:
|
|
"""从 YAML 文件加载任务图。
|
|
|
|
Parameters
|
|
----------
|
|
path:
|
|
YAML 文件路径。
|
|
|
|
Returns
|
|
-------
|
|
Graph
|
|
解析后的任务图。
|
|
"""
|
|
data = _safe_load(Path(path).read_text(encoding="utf-8"))
|
|
return _build_graph(data)
|
|
|
|
|
|
def _build_graph(data: Any) -> Graph:
|
|
"""从解析后的 YAML 字典构建 Graph。"""
|
|
if not isinstance(data, Mapping):
|
|
raise ValueError(f"YAML 根节点必须是映射，收到: {type(data).__name__}")
|
|
|
|
jobs = data.get("jobs")
|
|
if not jobs:
|
|
raise ValueError("YAML 缺少 'jobs' 或 jobs 为空")
|
|
if not isinstance(jobs, Mapping):
|
|
raise ValueError(f"jobs 必须是映射，收到: {type(jobs).__name__}")
|
|
|
|
defaults = _parse_defaults(data.get("defaults"))
|
|
strategy = data.get("strategy")
|
|
if strategy:
|
|
defaults = replace(defaults, strategy=str(strategy))
|
|
|
|
job_expansions = _precompute_expansions(jobs)
|
|
specs = _build_specs(jobs, job_expansions)
|
|
return Graph.from_specs(specs, defaults=defaults)
|
|
|
|
|
|
def _parse_defaults(data: Any) -> GraphDefaults:
|
|
"""解析 defaults 字段为 GraphDefaults。"""
|
|
if not data:
|
|
return GraphDefaults()
|
|
if not isinstance(data, Mapping):
|
|
raise ValueError(f"defaults 必须是映射，收到: {type(data).__name__}")
|
|
|
|
kwargs: dict[str, Any] = {}
|
|
if "retry" in data:
|
|
kwargs["retry"] = _parse_retry(data["retry"])
|
|
if "timeout" in data:
|
|
kwargs["timeout"] = float(data["timeout"])
|
|
if "verbose" in data:
|
|
kwargs["verbose"] = bool(data["verbose"])
|
|
if "env" in data:
|
|
kwargs["env"] = dict(data["env"])
|
|
if "cwd" in data:
|
|
kwargs["cwd"] = Path(data["cwd"])
|
|
if "tags" in data:
|
|
kwargs["tags"] = tuple(data["tags"])
|
|
if "priority" in data:
|
|
kwargs["priority"] = int(data["priority"])
|
|
if "continue_on_error" in data or "continue-on-error" in data:
|
|
kwargs["continue_on_error"] = bool(_get_field(data, "continue_on_error"))
|
|
if "concurrency_key" in data or "concurrency-key" in data:
|
|
kwargs["concurrency_key"] = _get_field(data, "concurrency_key")
|
|
return GraphDefaults(**kwargs)
|
|
|
|
|
|
def _parse_retry(data: Any) -> RetryPolicy:
|
|
"""解析 retry 字段为 RetryPolicy。"""
|
|
if not isinstance(data, Mapping):
|
|
raise ValueError(f"retry 必须是映射，收到: {type(data).__name__}")
|
|
kwargs: dict[str, Any] = {}
|
|
if "max_attempts" in data:
|
|
kwargs["max_attempts"] = int(data["max_attempts"])
|
|
if "delay" in data:
|
|
kwargs["delay"] = float(data["delay"])
|
|
if "backoff" in data:
|
|
kwargs["backoff"] = float(data["backoff"])
|
|
if "jitter" in data:
|
|
kwargs["jitter"] = float(data["jitter"])
|
|
return RetryPolicy(**kwargs)
|
|
|
|
|
|
def _get_field(data: Mapping[str, Any], name: str) -> Any:
|
|
"""获取字段值，支持 hyphen/underscore 兼容。"""
|
|
if name in data:
|
|
return data[name]
|
|
hyphen = name.replace("_", "-")
|
|
if hyphen in data:
|
|
return data[hyphen]
|
|
return None
|
|
|
|
|
|
def _precompute_expansions(jobs: Mapping[str, Any]) -> dict[str, list[tuple[str, dict[str, str]]]]:
|
|
"""预计算每个 job 的矩阵展开。
|
|
|
|
Returns
|
|
-------
|
|
dict[str, list[tuple[str, dict[str, str]]]]
|
|
job_id → [(expanded_name, {key: value}), ...]；非矩阵 job 返回 [(job_id, {})]。
|
|
"""
|
|
expansions: dict[str, list[tuple[str, dict[str, str]]]] = {}
|
|
for job_id, job_data in jobs.items():
|
|
if not isinstance(job_data, Mapping):
|
|
raise ValueError(f"job {job_id!r} 必须是映射，收到: {type(job_data).__name__}")
|
|
matrix_data = _extract_matrix(job_data)
|
|
if not matrix_data:
|
|
expansions[job_id] = [(job_id, {})]
|
|
continue
|
|
combos = _cartesian_product(matrix_data)
|
|
expanded: list[tuple[str, dict[str, str]]] = []
|
|
for combo in combos:
|
|
parts = [f"{k}-{v}" for k, v in combo.items()]
|
|
expanded_name = f"{job_id}_{'_'.join(parts)}"
|
|
expanded.append((expanded_name, combo))
|
|
expansions[job_id] = expanded
|
|
return expansions
|
|
|
|
|
|
def _extract_matrix(job_data: Mapping[str, Any]) -> Mapping[str, list[Any]] | None:
|
|
"""从 job 数据中提取 strategy.matrix。"""
|
|
strategy = _get_field(job_data, "strategy")
|
|
if strategy and isinstance(strategy, Mapping):
|
|
matrix = strategy.get("matrix")
|
|
if isinstance(matrix, Mapping):
|
|
return matrix
|
|
return None
|
|
|
|
|
|
def _cartesian_product(matrix_data: Mapping[str, list[Any]]) -> list[dict[str, str]]:
|
|
"""计算矩阵的笛卡尔积。"""
|
|
keys = list(matrix_data.keys())
|
|
result: list[dict[str, str]] = [{}]
|
|
for key in keys:
|
|
values = matrix_data[key]
|
|
if not isinstance(values, list):
|
|
raise ValueError(f"matrix.{key} 必须是列表，收到: {type(values).__name__}")
|
|
new_result: list[dict[str, str]] = []
|
|
for existing in result:
|
|
for value in values:
|
|
combo = dict(existing)
|
|
combo[key] = str(value)
|
|
new_result.append(combo)
|
|
result = new_result
|
|
return result
|
|
|
|
|
|
def _build_specs(
|
|
jobs: Mapping[str, Any],
|
|
expansions: dict[str, list[tuple[str, dict[str, str]]]],
|
|
) -> list[TaskSpec[Any]]:
|
|
"""从 jobs 构建展开后的 TaskSpec 列表。"""
|
|
specs: list[TaskSpec[Any]] = []
|
|
for job_id, job_data in jobs.items():
|
|
for expanded_name, matrix_values in expansions[job_id]:
|
|
specs.append(_build_spec(job_id, expanded_name, job_data, matrix_values, expansions))
|
|
return specs
|
|
|
|
|
|
def _build_spec(
|
|
job_id: str,
|
|
expanded_name: str,
|
|
job_data: Mapping[str, Any],
|
|
matrix_values: dict[str, str],
|
|
expansions: dict[str, list[tuple[str, dict[str, str]]]],
|
|
) -> TaskSpec[Any]:
|
|
"""构建单个 TaskSpec。"""
|
|
task_cmd = _parse_cmd(job_id, job_data, matrix_values)
|
|
depends_on = _expand_needs(_get_field(job_data, "needs"), expansions)
|
|
if_expr = _get_field(job_data, "if")
|
|
conditions = _parse_condition(if_expr) if if_expr else ()
|
|
|
|
kwargs: dict[str, Any] = {"cmd": task_cmd, "depends_on": tuple(depends_on)}
|
|
if conditions:
|
|
kwargs["conditions"] = conditions
|
|
kwargs.update(_parse_optional_fields(job_data, matrix_values))
|
|
return TaskSpec(name=expanded_name, **kwargs)
|
|
|
|
|
|
def _parse_cmd(job_id: str, job_data: Mapping[str, Any], matrix_values: dict[str, str]) -> list[str] | str:
|
|
"""解析 cmd / run 字段，应用矩阵占位符替换。"""
|
|
cmd_val = _get_field(job_data, "cmd")
|
|
run_val = _get_field(job_data, "run")
|
|
if cmd_val is not None:
|
|
if isinstance(cmd_val, list):
|
|
return [_substitute_matrix_str(str(a), matrix_values) for a in cmd_val]
|
|
return _substitute_matrix_str(str(cmd_val), matrix_values)
|
|
if run_val is not None:
|
|
return _substitute_matrix_str(str(run_val), matrix_values)
|
|
raise ValueError(f"job {job_id!r} 必须提供 cmd 或 run")
|
|
|
|
|
|
def _parse_optional_fields(job_data: Mapping[str, Any], matrix_values: dict[str, str]) -> dict[str, Any]:
|
|
"""解析所有可选字段，返回 kwargs 字典。"""
|
|
kwargs: dict[str, Any] = {}
|
|
|
|
timeout = _get_field(job_data, "timeout")
|
|
if timeout is not None:
|
|
kwargs["timeout"] = float(timeout)
|
|
|
|
retry = _get_field(job_data, "retry")
|
|
if retry is not None:
|
|
kwargs["retry"] = _parse_retry(retry)
|
|
|
|
cwd = _get_field(job_data, "cwd")
|
|
if cwd is not None:
|
|
kwargs["cwd"] = Path(_substitute_matrix_str(str(cwd), matrix_values))
|
|
|
|
env = _get_field(job_data, "env")
|
|
if env is not None:
|
|
kwargs["env"] = _substitute_matrix_env(env, matrix_values)
|
|
|
|
for bool_field in ("verbose", "skip_if_missing", "allow_upstream_skip", "continue_on_error"):
|
|
val = _get_field(job_data, bool_field)
|
|
if val is not None:
|
|
kwargs[bool_field] = bool(val)
|
|
|
|
priority = _get_field(job_data, "priority")
|
|
if priority is not None:
|
|
kwargs["priority"] = int(priority)
|
|
|
|
concurrency_key = _get_field(job_data, "concurrency_key")
|
|
if concurrency_key is not None:
|
|
kwargs["concurrency_key"] = _substitute_matrix_str(str(concurrency_key), matrix_values)
|
|
|
|
tags = _get_field(job_data, "tags")
|
|
tag_list = list(tags) if tags else []
|
|
runs_on = _get_field(job_data, "runs_on")
|
|
if runs_on:
|
|
tag_list.append(str(runs_on))
|
|
if tag_list:
|
|
kwargs["tags"] = tuple(tag_list)
|
|
|
|
return kwargs
|
|
|
|
|
|
def _expand_needs(
|
|
needs: Any,
|
|
expansions: dict[str, list[tuple[str, dict[str, str]]]],
|
|
) -> list[str]:
|
|
"""展开 needs 列表：矩阵 job 依赖展开为所有变体。"""
|
|
if not needs:
|
|
return []
|
|
if isinstance(needs, str):
|
|
needs = [needs]
|
|
result: list[str] = []
|
|
for dep in needs:
|
|
if dep in expansions:
|
|
result.extend(name for name, _ in expansions[dep])
|
|
else:
|
|
result.append(dep)
|
|
return result
|
|
|
|
|
|
def _parse_condition(expr: Any) -> tuple[Condition, ...]:
|
|
"""解析 if 表达式为 Condition 元组。
|
|
|
|
支持：success() / always() / env.VAR / env.VAR == 'x' / env.VAR != 'x'。
|
|
failure() 不支持（PyFlowX 无层屏障概念）。
|
|
"""
|
|
expr = str(expr).strip()
|
|
|
|
if expr in ("success()", "always()"):
|
|
return ()
|
|
if expr == "failure()":
|
|
raise ValueError("failure() 条件不支持；PyFlowX 无层屏障概念")
|
|
|
|
m = re.fullmatch(r"env\.(\w+)", expr)
|
|
if m:
|
|
return (BuiltinConditions.ENV_VAR_EXISTS(m.group(1)),)
|
|
|
|
m = re.fullmatch(r"env\.(\w+)\s*==\s*['\"]([^'\"]*)['\"]", expr)
|
|
if m:
|
|
return (BuiltinConditions.ENV_VAR_EQUALS(m.group(1), m.group(2)),)
|
|
|
|
m = re.fullmatch(r"env\.(\w+)\s*!=\s*['\"]([^'\"]*)['\"]", expr)
|
|
if m:
|
|
return (BuiltinConditions.NOT(BuiltinConditions.ENV_VAR_EQUALS(m.group(1), m.group(2))),)
|
|
|
|
raise ValueError(f"无法解析的条件表达式: {expr!r}")
|
|
|
|
|
|
def _substitute_matrix_str(text: str, matrix_values: dict[str, str]) -> str:
|
|
"""替换字符串中的 ${{ matrix.key }} 占位符。"""
|
|
|
|
def _replacer(m: re.Match[str]) -> str:
|
|
key = m.group(1)
|
|
if key not in matrix_values:
|
|
raise ValueError(f"矩阵变量 matrix.{key} 未定义")
|
|
return matrix_values[key]
|
|
|
|
return _MATRIX_PATTERN.sub(_replacer, text)
|
|
|
|
|
|
def _substitute_matrix_env(env: Any, matrix_values: dict[str, str]) -> dict[str, str]:
|
|
"""替换环境变量中的矩阵占位符。"""
|
|
if not isinstance(env, Mapping):
|
|
raise ValueError(f"env 必须是映射，收到: {type(env).__name__}")
|
|
return {k: _substitute_matrix_str(str(v), matrix_values) for k, v in env.items()}
|