Files
pyflowx/tests/test_pipelines.py
T

781 lines
26 KiB
Python

"""通用 DAG 构造器测试.
验证 :func:`data_pipeline` / :func:`command_chain` / :func:`fan_out_fan_in`
/ :func:`switch` / :func:`branch` 五个构造器的拓扑生成、执行结果、错误处理与 DAG 组合能力.
测试组织:
* ``TestDataPipeline`` —— 数据流水线拓扑/执行/命名/错误
* ``TestCommandChain`` —— 命令链拓扑/执行/透传/错误
* ``TestFanOutFanIn`` —— map-reduce 拓扑/执行/顺序/错误
* ``TestPipelinesWithFileops`` —— 与 fileops 模块联用组合示例
* ``TestSwitch`` —— 条件分支 (多路分发) 拓扑/执行/默认/条件共存
* ``TestBranch`` —— 二路分支 (if/else) 拓扑/执行/predicate
"""
from __future__ import annotations
import sys
from pathlib import Path
import pytest
import pyflowx as px
from pyflowx.fileops import append_text, find, read_text, write_text
from pyflowx.pipelines import command_chain, data_pipeline, fan_out_fan_in
# ======================================================================
# data_pipeline
# ======================================================================
class TestDataPipeline:
"""data_pipeline 构造器测试."""
def test_basic_three_step_chain(self) -> None:
"""三步链式数据流: extract → transform → load."""
def extract() -> list[int]:
return [1, 2, 3]
def transform(extract: list[int]) -> list[int]:
return [x * 2 for x in extract]
def load(transform: list[int]) -> int:
return sum(transform)
graph = data_pipeline([extract, transform, load])
specs = graph.all_specs()
assert len(specs) == 3
assert specs["extract"].depends_on == ()
assert specs["transform"].depends_on == ("extract",)
assert specs["load"].depends_on == ("transform",)
report = px.run(graph, strategy="sequential")
assert report.success
assert report["extract"] == [1, 2, 3]
assert report["transform"] == [2, 4, 6]
assert report["load"] == 12
def test_single_step(self) -> None:
"""单步骤流水线: 1 个任务无依赖."""
def only() -> int:
return 42
graph = data_pipeline([only])
specs = graph.all_specs()
assert len(specs) == 1
assert specs["only"].depends_on == ()
report = px.run(graph, strategy="sequential")
assert report.success
assert report["only"] == 42
def test_custom_names(self) -> None:
"""自定义命名覆盖函数 __name__."""
def first() -> int:
return 1
def second(first: int) -> int:
return first + 1
graph = data_pipeline([first, second], names=["first", "second"])
specs = graph.all_specs()
assert "first" in specs
assert "second" in specs
assert specs["second"].depends_on == ("first",)
report = px.run(graph, strategy="sequential")
assert report.success
assert report["first"] == 1
assert report["second"] == 2
def test_lambda_fallback_naming(self) -> None:
"""lambda 函数回退为 step 命名."""
graph = data_pipeline([
lambda: 1,
lambda step: step + 10,
])
specs = graph.all_specs()
assert "step" in specs
assert "step_1" in specs
assert specs["step_1"].depends_on == ("step",)
report = px.run(graph, strategy="sequential")
assert report.success
assert report["step"] == 1
assert report["step_1"] == 11
def test_duplicate_function_names_auto_indexed(self) -> None:
"""同名函数自动追加索引避免冲突."""
def fn() -> int:
return 0
# 两个同名函数 (不同对象, 相同 __name__)
def fn2() -> int:
return 1
fn2.__name__ = "fn"
graph = data_pipeline([fn, fn2])
specs = graph.all_specs()
assert "fn" in specs
assert "fn_1" in specs
def test_empty_steps_raises(self) -> None:
"""空 steps 抛 ValueError."""
try:
data_pipeline([])
except ValueError as exc:
assert "steps" in str(exc)
else:
raise AssertionError("应抛 ValueError")
def test_names_length_mismatch_raises(self) -> None:
"""names 长度不匹配抛 ValueError."""
def a() -> int:
return 1
try:
data_pipeline([a], names=["x", "y"])
except ValueError as exc:
assert "names" in str(exc)
else:
raise AssertionError("应抛 ValueError")
def test_explicit_duplicate_name_raises(self) -> None:
"""显式 names 重复抛 ValueError."""
def a() -> int:
return 1
def b() -> int:
return 2
try:
data_pipeline([a, b], names=["dup", "dup"])
except ValueError as exc:
assert "重复" in str(exc)
else:
raise AssertionError("应抛 ValueError")
def test_data_injection_via_param_name(self) -> None:
"""参数名注入: 下游参数名匹配上游任务名."""
def source() -> str:
return "hello"
def upper(source: str) -> str:
return source.upper()
def exclaim(upper: str) -> str:
return f"{upper}!"
graph = data_pipeline([source, upper, exclaim])
report = px.run(graph, strategy="sequential")
assert report.success
assert report["source"] == "hello"
assert report["upper"] == "HELLO"
assert report["exclaim"] == "HELLO!"
# ======================================================================
# command_chain
# ======================================================================
class TestCommandChain:
"""command_chain 构造器测试."""
def test_basic_chain_topology(self) -> None:
"""三命令链式依赖."""
graph = command_chain([
["echo", "first"],
["echo", "second"],
["echo", "third"],
])
specs = graph.all_specs()
assert len(specs) == 3
names = list(specs.keys())
assert specs[names[0]].depends_on == ()
assert specs[names[1]].depends_on == (names[0],)
assert specs[names[2]].depends_on == (names[1],)
def test_auto_naming_with_first_arg(self) -> None:
"""自动命名用 cmd_{i:02d}_{first_arg}."""
graph = command_chain([
["echo", "hello"],
["echo", "world"],
])
specs = graph.all_specs()
assert "cmd_00_echo" in specs
assert "cmd_01_echo" in specs
def test_string_command_auto_naming(self) -> None:
"""shell 字符串命令自动命名取首个 token."""
graph = command_chain([
"echo hello",
"ls -la",
])
specs = graph.all_specs()
assert "cmd_00_echo" in specs
assert "cmd_01_ls" in specs
def test_custom_names(self) -> None:
"""自定义命名覆盖自动命名."""
graph = command_chain(
[["echo", "a"], ["echo", "b"]],
names=["step_a", "step_b"],
)
specs = graph.all_specs()
assert "step_a" in specs
assert "step_b" in specs
assert specs["step_b"].depends_on == ("step_a",)
assert "cmd_00_echo" not in specs
def test_execution_succeeds(self) -> None:
"""命令链实际执行成功."""
graph = command_chain([
[sys.executable, "-c", "print('first')"],
[sys.executable, "-c", "print('second')"],
])
report = px.run(graph, strategy="sequential")
assert report.success
def test_cwd_passthrough(self, tmp_path: Path) -> None:
"""cwd 透传给所有 TaskSpec."""
graph = command_chain(
[["echo", "hello"]],
cwd=tmp_path,
)
spec = next(iter(graph.all_specs().values()))
assert spec.cwd == tmp_path
def test_env_passthrough(self) -> None:
"""env 透传给所有 TaskSpec."""
env = {"MY_VAR": "value"}
graph = command_chain(
[["echo", "hello"]],
env=env,
)
spec = next(iter(graph.all_specs().values()))
assert spec.env == env
def test_continue_on_error_flag(self) -> None:
"""continue_on_error 透传给所有 TaskSpec."""
graph = command_chain(
[["echo", "hello"]],
continue_on_error=True,
)
spec = next(iter(graph.all_specs().values()))
assert spec.continue_on_error is True
def test_empty_commands_raises(self) -> None:
"""空 commands 抛 ValueError."""
try:
command_chain([])
except ValueError as exc:
assert "commands" in str(exc)
else:
raise AssertionError("应抛 ValueError")
def test_names_length_mismatch_raises(self) -> None:
"""names 长度不匹配抛 ValueError."""
try:
command_chain([["echo", "a"]], names=["x", "y"])
except ValueError as exc:
assert "names" in str(exc)
else:
raise AssertionError("应抛 ValueError")
def test_duplicate_name_raises(self) -> None:
"""显式 names 重复抛 ValueError."""
try:
command_chain(
[["echo", "a"], ["echo", "b"]],
names=["dup", "dup"],
)
except ValueError as exc:
assert "重复" in str(exc)
else:
raise AssertionError("应抛 ValueError")
def test_command_failure_aborts_chain(self) -> None:
"""命令失败时抛 TaskFailedError (continue_on_error=False)."""
graph = command_chain([
[sys.executable, "-c", "import sys; sys.exit(1)"],
[sys.executable, "-c", "print('should not run')"],
])
with pytest.raises(px.TaskFailedError):
px.run(graph, strategy="sequential")
def test_command_failure_continues_with_flag(self) -> None:
"""continue_on_error=True 时失败任务不抛异常, 标记为 FAILED."""
graph = command_chain(
[
[sys.executable, "-c", "import sys; sys.exit(1)"],
[sys.executable, "-c", "print('runs anyway')"],
],
continue_on_error=True,
)
report = px.run(graph, strategy="sequential")
specs = graph.all_specs()
names = list(specs.keys())
# 第一个任务失败但不抛异常 (continue_on_error 生效)
result = report.result_of(names[0])
assert result is not None
assert result.status == px.TaskStatus.FAILED
# 第二个任务因硬依赖失败而被 SKIPPED
result2 = report.result_of(names[1])
assert result2 is not None
assert result2.status == px.TaskStatus.SKIPPED
# ======================================================================
# fan_out_fan_in
# ======================================================================
class TestFanOutFanIn:
"""fan_out_fan_in 构造器测试."""
def test_basic_map_reduce(self) -> None:
"""基本 map-reduce: 4 个 worker + 1 个 reduce."""
graph = fan_out_fan_in(
items=[1, 2, 3, 4],
worker=lambda x: x**2,
reduce=sum,
)
specs = graph.all_specs()
assert len(specs) == 5 # 4 worker + 1 reduce
assert "worker_00" in specs
assert "worker_01" in specs
assert "worker_02" in specs
assert "worker_03" in specs
assert "reduce" in specs
assert specs["reduce"].depends_on == ("worker_00", "worker_01", "worker_02", "worker_03")
report = px.run(graph, strategy="thread")
assert report.success
assert report["worker_00"] == 1
assert report["worker_01"] == 4
assert report["worker_02"] == 9
assert report["worker_03"] == 16
assert report["reduce"] == 30
def test_fan_out_only_without_reduce(self) -> None:
"""无 reduce 时仅生成 fan-out."""
graph = fan_out_fan_in(
items=[1, 2, 3],
worker=lambda x: x * 10,
)
specs = graph.all_specs()
assert len(specs) == 3
assert "reduce" not in specs
assert all(spec.depends_on == () for spec in specs.values())
report = px.run(graph, strategy="sequential")
assert report.success
assert report["worker_00"] == 10
assert report["worker_01"] == 20
assert report["worker_02"] == 30
def test_custom_worker_name_prefix(self) -> None:
"""自定义 worker 名称前缀."""
graph = fan_out_fan_in(
items=[1, 2],
worker=lambda x: x,
reduce=sum,
worker_name="fetch",
)
specs = graph.all_specs()
assert "fetch_00" in specs
assert "fetch_01" in specs
assert "reduce" in specs
assert specs["reduce"].depends_on == ("fetch_00", "fetch_01")
def test_custom_reduce_name(self) -> None:
"""自定义 reduce 名称."""
graph = fan_out_fan_in(
items=[1, 2],
worker=lambda x: x,
reduce=sum,
reduce_name="aggregate",
)
specs = graph.all_specs()
assert "aggregate" in specs
assert specs["aggregate"].depends_on == ("worker_00", "worker_01")
def test_results_preserve_item_order(self) -> None:
"""reduce 接收的结果保持 items 顺序."""
items = [10, 20, 30, 40, 50]
graph = fan_out_fan_in(
items=items,
worker=lambda x: x,
reduce=lambda r: r, # 直接返回列表
)
report = px.run(graph, strategy="thread")
assert report.success
assert report["reduce"] == [10, 20, 30, 40, 50]
def test_single_item(self) -> None:
"""单个 item 也能正常工作."""
graph = fan_out_fan_in(
items=[42],
worker=lambda x: x + 1,
reduce=lambda r: r[0],
)
report = px.run(graph, strategy="sequential")
assert report.success
assert report["worker_00"] == 43
assert report["reduce"] == 43
def test_empty_items_raises(self) -> None:
"""空 items 抛 ValueError."""
try:
fan_out_fan_in([], worker=lambda x: x)
except ValueError as exc:
assert "items" in str(exc)
else:
raise AssertionError("应抛 ValueError")
def test_duplicate_reduce_name_raises(self) -> None:
"""reduce_name 与 worker 名冲突抛 ValueError."""
try:
fan_out_fan_in(
items=[1, 2],
worker=lambda x: x,
reduce=sum,
worker_name="task",
reduce_name="task_00", # 与第一个 worker 名冲突
)
except ValueError as exc:
assert "重复" in str(exc)
else:
raise AssertionError("应抛 ValueError")
def test_worker_with_complex_objects(self) -> None:
"""worker 处理复杂对象 (字典)."""
items = [{"name": "a", "value": 1}, {"name": "b", "value": 2}]
graph = fan_out_fan_in(
items=items,
worker=lambda item: {item["name"]: item["value"]},
reduce=lambda results: {k: v for r in results for k, v in r.items()},
)
report = px.run(graph, strategy="thread")
assert report.success
assert report["reduce"] == {"a": 1, "b": 2}
def test_reduce_receives_all_worker_results(self) -> None:
"""reduce 通过 Context 接收所有 worker 结果."""
graph = fan_out_fan_in(
items=[1, 2, 3],
worker=lambda x: x * 100,
reduce=len, # 返回接收到的结果数
)
report = px.run(graph, strategy="sequential")
assert report.success
assert report["reduce"] == 3
def test_dependency_driven_strategy(self) -> None:
"""dependency 策略下 map-reduce 正常工作."""
graph = fan_out_fan_in(
items=[1, 2, 3, 4],
worker=lambda x: x + 1,
reduce=sum,
)
report = px.run(graph, strategy="dependency")
assert report.success
assert report["reduce"] == 2 + 3 + 4 + 5
# ======================================================================
# 与 fileops 联用组合示例
# ======================================================================
class TestPipelinesWithFileops:
"""构造器与 fileops 模块联用组合示例."""
def test_data_pipeline_with_fileops(self, tmp_path: Path) -> None:
"""data_pipeline + fileops: 查找 → 读取 → 汇总."""
(tmp_path / "a.txt").write_text("hello")
(tmp_path / "b.txt").write_text("world")
def find_files() -> list[Path]:
return find(tmp_path, "*.txt")
def read_all(find_files: list[Path]) -> list[str]:
return [read_text(p) for p in find_files]
def total_length(read_all: list[str]) -> int:
return sum(len(s) for s in read_all)
graph = data_pipeline([find_files, read_all, total_length])
report = px.run(graph, strategy="sequential")
assert report.success
assert report["total_length"] == 10 # "hello" + "world"
def test_fan_out_fan_in_batch_copy(self, tmp_path: Path) -> None:
"""fan_out_fan_in + fileops: 批量读取文件内容并聚合."""
files = []
for i, content in enumerate(["aaa", "bb", "c"]):
p = tmp_path / f"f{i}.txt"
write_text(p, content)
files.append(p)
def read_file(path: Path) -> str:
return read_text(path)
graph = fan_out_fan_in(
items=files,
worker=read_file,
reduce="".join,
)
report = px.run(graph, strategy="thread")
assert report.success
assert report["reduce"] == "aaabbc"
def test_command_chain_with_file_writes(self, tmp_path: Path) -> None:
"""command_chain 串联文件写入命令."""
out1 = tmp_path / "out1.txt"
out2 = tmp_path / "out2.txt"
graph = command_chain(
[
[sys.executable, "-c", f"open({str(out1)!r}, 'w').write('step1')"],
[sys.executable, "-c", f"open({str(out2)!r}, 'w').write('step2')"],
],
names=["write1", "write2"],
)
report = px.run(graph, strategy="sequential")
assert report.success
assert read_text(out1) == "step1"
assert read_text(out2) == "step2"
def test_data_pipeline_append_log(self, tmp_path: Path) -> None:
"""data_pipeline + fileops.append_text: 日志追加流水线."""
log = tmp_path / "log.txt"
def first() -> int:
return append_text(log, "first\n")
def second(first: int) -> int:
return append_text(log, "second\n")
def third(second: int) -> str:
return read_text(log)
graph = data_pipeline([first, second, third])
report = px.run(graph, strategy="sequential")
assert report.success
assert report["third"] == "first\nsecond\n"
# ---------------------------------------------------------------------- #
# 条件分支: switch / branch
# ---------------------------------------------------------------------- #
class TestSwitch:
"""测试 px.switch() 条件分支构造器."""
def test_switch_executes_matching_case(self) -> None:
"""selector 结果匹配的 case 执行,其余跳过。"""
def classify() -> str:
return "positive"
def handle_positive(classify: str) -> str:
return f"pos:{classify}"
def handle_negative(classify: str) -> str:
return f"neg:{classify}"
graph = px.switch(
px.TaskSpec("classify", classify),
{
"positive": px.TaskSpec("handle_positive", handle_positive),
"negative": px.TaskSpec("handle_negative", handle_negative),
},
)
report = px.run(graph)
assert report.success
assert report["classify"] == "positive"
assert report["handle_positive"] == "pos:positive"
assert report.results["handle_negative"].status == px.TaskStatus.SKIPPED
def test_switch_negative_case(self) -> None:
"""selector 返回 negative 时执行 negative case。"""
def classify() -> str:
return "negative"
def handle_positive(classify: str) -> str:
return "pos"
def handle_negative(classify: str) -> str:
return "neg"
graph = px.switch(
px.TaskSpec("classify", classify),
{
"positive": px.TaskSpec("handle_positive", handle_positive),
"negative": px.TaskSpec("handle_negative", handle_negative),
},
)
report = px.run(graph)
assert report.success
assert report["handle_negative"] == "neg"
assert report.results["handle_positive"].status == px.TaskStatus.SKIPPED
def test_switch_default_when_no_match(self) -> None:
"""无 case 匹配时执行 default 任务。"""
def classify() -> str:
return "unknown"
def handle_a(classify: str) -> str:
return "a"
def handle_default(classify: str) -> str:
return "default"
graph = px.switch(
px.TaskSpec("classify", classify),
{"a": px.TaskSpec("handle_a", handle_a)},
default=px.TaskSpec("handle_default", handle_default),
)
report = px.run(graph)
assert report.success
assert report.results["handle_a"].status == px.TaskStatus.SKIPPED
assert report["handle_default"] == "default"
def test_switch_all_skipped_without_default(self) -> None:
"""无 case 匹配且无 default 时所有 case 跳过。"""
def classify() -> str:
return "unknown"
def handle_a(classify: str) -> str:
return "a"
graph = px.switch(
px.TaskSpec("classify", classify),
{"a": px.TaskSpec("handle_a", handle_a)},
)
report = px.run(graph)
assert report.success
assert report.results["handle_a"].status == px.TaskStatus.SKIPPED
def test_switch_preserves_existing_conditions(self) -> None:
"""switch 附加的条件与已有条件共存(AND 语义由执行器保证)。"""
def classify() -> str:
return "go"
def handle_go(classify: str) -> str:
return "went"
spec = px.TaskSpec("handle_go", handle_go, conditions=(px.IS_LINUX,))
graph = px.switch(
px.TaskSpec("classify", classify),
{"go": spec},
)
report = px.run(graph)
# IS_LINUX + DEP_EQUALS("classify","go") 均满足时才执行
if px.IS_LINUX({}):
assert report["handle_go"] == "went"
else:
assert report.results["handle_go"].status == px.TaskStatus.SKIPPED
class TestBranch:
"""测试 px.branch() 二路分支构造器."""
def test_branch_true_path_executes(self) -> None:
"""predicate 返回 True 时执行 if_true。"""
def compute() -> int:
return 42
def handle_positive(compute: int) -> str:
return f"positive:{compute}"
def handle_non_positive(compute: int) -> str:
return "non-positive"
graph = px.branch(
px.TaskSpec("compute", compute),
lambda x: x > 0,
px.TaskSpec("handle_positive", handle_positive),
if_false=px.TaskSpec("handle_non_positive", handle_non_positive),
)
report = px.run(graph)
assert report.success
assert report["handle_positive"] == "positive:42"
assert report.results["handle_non_positive"].status == px.TaskStatus.SKIPPED
def test_branch_false_path_executes(self) -> None:
"""predicate 返回 False 时执行 if_false。"""
def compute() -> int:
return -5
def handle_positive(compute: int) -> str:
return "positive"
def handle_non_positive(compute: int) -> str:
return f"non-positive:{compute}"
graph = px.branch(
px.TaskSpec("compute", compute),
lambda x: x > 0,
px.TaskSpec("handle_positive", handle_positive),
if_false=px.TaskSpec("handle_non_positive", handle_non_positive),
)
report = px.run(graph)
assert report.success
assert report["handle_non_positive"] == "non-positive:-5"
assert report.results["handle_positive"].status == px.TaskStatus.SKIPPED
def test_branch_without_if_false(self) -> None:
"""无 if_false 时,predicate 返回 False 则 if_true 被跳过。"""
def compute() -> int:
return -1
def handle_positive(compute: int) -> str:
return "positive"
graph = px.branch(
px.TaskSpec("compute", compute),
lambda x: x > 0,
px.TaskSpec("handle_positive", handle_positive),
)
report = px.run(graph)
assert report.success
assert report.results["handle_positive"].status == px.TaskStatus.SKIPPED
def test_branch_with_predicate_function(self) -> None:
"""使用命名函数作为 predicate。"""
def compute() -> list[int]:
return [1, 2, 3]
def is_long(items: list[int]) -> bool:
return len(items) > 2
def handle_long(compute: list[int]) -> str:
return "long"
def handle_short(compute: list[int]) -> str:
return "short"
graph = px.branch(
px.TaskSpec("compute", compute),
is_long,
px.TaskSpec("handle_long", handle_long),
if_false=px.TaskSpec("handle_short", handle_short),
)
report = px.run(graph)
assert report.success
assert report["handle_long"] == "long"
assert report.results["handle_short"].status == px.TaskStatus.SKIPPED