227 lines
6.4 KiB
Python
227 lines
6.4 KiB
Python
"""PyFlowX —— 轻量、类型安全的 DAG 任务调度器。
|
|
|
|
公共 API
|
|
--------
|
|
* :func:`task` / :func:`cmd` —— 装饰器/工厂,从函数或命令快速创建 TaskSpec。
|
|
* :func:`graph` —— 快捷构造图(等价于 ``Graph.from_specs``,接受可变参数)。
|
|
* :class:`TaskSpec` —— 不可变任务描述符(唯一需要配置的东西)。
|
|
* :class:`Graph` —— 由一组 spec 构建的 DAG;负责校验、分层、可视化。
|
|
* :func:`run` ——以 ``sequential`` / ``thread`` / ``async`` / ``dependency``
|
|
策略执行图(默认 ``dependency``)。
|
|
* :class:`RunReport` —— 类型化、可查询的运行结果。
|
|
* :class:`Context` —— 整体上下文注入的标注标记。
|
|
* :class:`RetryPolicy` —— 重试策略(max_attempts/delay/backoff/jitter/retry_on)。
|
|
* :class:`TaskHooks` —— 任务生命周期钩子(pre_run/post_run/on_failure)。
|
|
* :class:`GraphDefaults` —— 图级默认值。
|
|
* :func:`compose` —— 编程式组合多图。
|
|
* :func:`switch` / :func:`branch` —— 条件分支 DAG 构造器。
|
|
* :func:`task_template` —— 批量生成相似 TaskSpec 的工厂。
|
|
* :func:`sh` —— Shell 命令执行辅助(支持 ``list[str]`` / ``str``,统一中文错误处理)。
|
|
* 状态后端::class:`StateBackend`、:class:`MemoryBackend`、:class:`JSONBackend`、:class:`SQLiteBackend`。
|
|
|
|
快速上手
|
|
--------
|
|
import pyflowx as px
|
|
|
|
@px.task
|
|
def extract() -> list[int]: return [1, 2, 3]
|
|
|
|
@px.task
|
|
def double(extract: list[int]) -> list[int]: return [x * 2 for x in extract]
|
|
|
|
graph = px.graph(extract, double) # double 自动依赖 extract
|
|
report = px.run(graph)
|
|
print(report["double"]) # [2, 4, 6]
|
|
|
|
命令行任务示例
|
|
--------------
|
|
import pyflowx as px
|
|
from pyflowx.conditions import IS_WINDOWS, BuiltinConditions
|
|
|
|
graph = px.graph(
|
|
px.cmd(["ls", "-la"]),
|
|
px.cmd("git status", name="check_git"),
|
|
px.cmd(["dir"], name="win_only", conditions=(IS_WINDOWS,)),
|
|
px.cmd(["git", "--version"], name="git_check",
|
|
conditions=(BuiltinConditions.HAS_INSTALLED("git"),)),
|
|
px.cmd(["maturin", "build"], name="optional_build", skip_if_missing=True),
|
|
)
|
|
report = px.run(graph)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from . import fileops
|
|
from .cancellation import CancelToken
|
|
from .command import run_command
|
|
from .compose import GraphComposer, compose
|
|
from .conditions import IS_LINUX, IS_MACOS, IS_POSIX, IS_WINDOWS, BuiltinConditions, Condition, Constants
|
|
from .context import Context, build_call_args, describe_injection
|
|
from .diagnostics import DependencyChain, DiagnosticReport, FailureCluster, diagnose
|
|
from .errors import (
|
|
CycleError,
|
|
DuplicateTaskError,
|
|
InjectionError,
|
|
MissingDependencyError,
|
|
PyFlowXError,
|
|
StorageError,
|
|
TaskFailedError,
|
|
TaskTimeoutError,
|
|
)
|
|
from .executors import Strategy, run, run_iter
|
|
from .graph import Graph, GraphDefaults
|
|
from .history import RunHistory
|
|
from .imaging import image_pipeline
|
|
from .monitoring import MetricsCollector, health_check, start_metrics_server
|
|
from .notification import (
|
|
ALL_LEVELS,
|
|
CallbackNotifier,
|
|
DingTalkNotifier,
|
|
DiscordNotifier,
|
|
FeishuNotifier,
|
|
NotificationLevel,
|
|
Notifier,
|
|
SlackNotifier,
|
|
TelegramNotifier,
|
|
WebhookNotifier,
|
|
WeChatNotifier,
|
|
)
|
|
from .pipelines import branch, command_chain, data_pipeline, fan_out_fan_in, switch
|
|
from .profiling import ProfileReport, TaskProfile
|
|
from .progress import ProgressCallback, RichProgressMonitor
|
|
from .report import RunReport
|
|
from .runner import CliExitCode, CliRunner
|
|
from .shell import sh
|
|
from .storage import JSONBackend, MemoryBackend, SQLiteBackend, StateBackend
|
|
from .task import (
|
|
CacheKeyFn,
|
|
LoopSpec,
|
|
RetryPolicy,
|
|
TaskCmd,
|
|
TaskEvent,
|
|
TaskHooks,
|
|
TaskResult,
|
|
TaskSpec,
|
|
TaskStatus,
|
|
cmd,
|
|
task,
|
|
task_template,
|
|
)
|
|
from .tools import ToolSpec, list_subcommands, list_tools, run_tool, tool
|
|
from .yaml_loader import load_yaml, parse_yaml_string
|
|
|
|
__version__ = "0.4.11"
|
|
|
|
|
|
def graph(
|
|
*specs: TaskSpec[Any] | str,
|
|
defaults: GraphDefaults | None = None,
|
|
namespace: str | None = None,
|
|
) -> Graph:
|
|
"""快捷构造图:等价于 :meth:`Graph.from_specs`,接受可变参数而非列表。
|
|
|
|
对 ``depends_on`` 为空的纯 fn 任务,自动从必需参数名推断依赖
|
|
(匹配图中任务名的参数被加入 ``depends_on``)。
|
|
|
|
Examples
|
|
--------
|
|
>>> import pyflowx as px
|
|
>>> @px.task
|
|
... def extract() -> list[int]: return [1, 2, 3]
|
|
>>> @px.task
|
|
... def double(extract: list[int]) -> list[int]: return [x * 2 for x in extract]
|
|
>>> g = px.graph(extract, double) # double 自动依赖 extract
|
|
"""
|
|
return Graph.from_specs(specs, defaults=defaults, namespace=namespace)
|
|
|
|
|
|
__all__ = [
|
|
"ALL_LEVELS",
|
|
"IS_LINUX",
|
|
"IS_MACOS",
|
|
"IS_POSIX",
|
|
"IS_WINDOWS",
|
|
"BuiltinConditions",
|
|
"CacheKeyFn",
|
|
"CallbackNotifier",
|
|
"CancelToken",
|
|
"CliExitCode",
|
|
"CliRunner",
|
|
"Condition",
|
|
"Constants",
|
|
"Context",
|
|
"CycleError",
|
|
"DependencyChain",
|
|
"DiagnosticReport",
|
|
"DingTalkNotifier",
|
|
"DiscordNotifier",
|
|
"DuplicateTaskError",
|
|
"FailureCluster",
|
|
"FeishuNotifier",
|
|
"Graph",
|
|
"GraphComposer",
|
|
"GraphDefaults",
|
|
"InjectionError",
|
|
"JSONBackend",
|
|
"LoopSpec",
|
|
"MemoryBackend",
|
|
"MetricsCollector",
|
|
"MissingDependencyError",
|
|
"NotificationLevel",
|
|
"Notifier",
|
|
"ProfileReport",
|
|
"ProgressCallback",
|
|
"PyFlowXError",
|
|
"RetryPolicy",
|
|
"RichProgressMonitor",
|
|
"RunHistory",
|
|
"RunReport",
|
|
"SQLiteBackend",
|
|
"SlackNotifier",
|
|
"StateBackend",
|
|
"StorageError",
|
|
"Strategy",
|
|
"TaskCmd",
|
|
"TaskEvent",
|
|
"TaskFailedError",
|
|
"TaskHooks",
|
|
"TaskProfile",
|
|
"TaskResult",
|
|
"TaskSpec",
|
|
"TaskStatus",
|
|
"TaskTimeoutError",
|
|
"TelegramNotifier",
|
|
"ToolSpec",
|
|
"WeChatNotifier",
|
|
"WebhookNotifier",
|
|
"branch",
|
|
"build_call_args",
|
|
"cmd",
|
|
"command_chain",
|
|
"compose",
|
|
"data_pipeline",
|
|
"describe_injection",
|
|
"diagnose",
|
|
"fan_out_fan_in",
|
|
"fileops",
|
|
"graph",
|
|
"health_check",
|
|
"image_pipeline",
|
|
"list_subcommands",
|
|
"list_tools",
|
|
"load_yaml",
|
|
"parse_yaml_string",
|
|
"run",
|
|
"run_command",
|
|
"run_iter",
|
|
"run_tool",
|
|
"sh",
|
|
"start_metrics_server",
|
|
"switch",
|
|
"task",
|
|
"task_template",
|
|
"tool",
|
|
]
|