feat: 新增任务跳过原因记录，完善上游任务跳过传播逻辑
1. 为TaskResult和TaskEvent新增reason字段记录跳过原因 2. 为同步/异步任务执行器添加上游任务跳过检测，自动跳过下游任务 3. 完善任务跳过的原因判断，支持条件不满足、缓存命中、上游跳过场景 4. 优化gittool工具，新增排除目录配置和更灵活的git操作流程 5. 重构测试用例格式，新增上游任务跳过的测试覆盖 6. 默认启用verbose输出，优化跳过任务的日志提示
This commit is contained in:
+128
-71
@@ -26,12 +26,10 @@ def test_sequential_basic() -> None:
|
||||
def double(extract: list[int]) -> list[int]:
|
||||
return [x * 2 for x in extract]
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("extract", extract),
|
||||
px.TaskSpec("double", double, depends_on=("extract",)),
|
||||
]
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("extract", extract),
|
||||
px.TaskSpec("double", double, depends_on=("extract",)),
|
||||
])
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert report["extract"] == [1, 2, 3]
|
||||
@@ -48,14 +46,12 @@ def test_sequential_diamond() -> None:
|
||||
|
||||
return fn
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", make("a")),
|
||||
px.TaskSpec("b", make("b"), depends_on=("a",)),
|
||||
px.TaskSpec("c", make("c"), depends_on=("a",)),
|
||||
px.TaskSpec("d", make("d"), depends_on=("b", "c")),
|
||||
]
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("a", make("a")),
|
||||
px.TaskSpec("b", make("b"), depends_on=("a",)),
|
||||
px.TaskSpec("c", make("c"), depends_on=("a",)),
|
||||
px.TaskSpec("d", make("d"), depends_on=("b", "c")),
|
||||
])
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert report["d"] == "d"
|
||||
@@ -69,12 +65,10 @@ def test_failure_propagates() -> None:
|
||||
def downstream(_boom: None) -> int:
|
||||
return 1
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("boom", boom),
|
||||
px.TaskSpec("downstream", downstream, depends_on=("boom",)),
|
||||
]
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("boom", boom),
|
||||
px.TaskSpec("downstream", downstream, depends_on=("boom",)),
|
||||
])
|
||||
with pytest.raises(TaskFailedError) as exc_info:
|
||||
_ = px.run(graph, strategy="sequential")
|
||||
assert exc_info.value.task == "boom"
|
||||
@@ -116,13 +110,11 @@ def test_threaded_parallelism() -> None:
|
||||
time.sleep(0.3)
|
||||
return "done"
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", slow),
|
||||
px.TaskSpec("b", slow),
|
||||
px.TaskSpec("c", slow),
|
||||
]
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("a", slow),
|
||||
px.TaskSpec("b", slow),
|
||||
px.TaskSpec("c", slow),
|
||||
])
|
||||
start = time.time()
|
||||
report = px.run(graph, strategy="thread", max_workers=3)
|
||||
elapsed = time.time() - start
|
||||
@@ -145,13 +137,11 @@ def test_threaded_layer_barrier() -> None:
|
||||
|
||||
return fn
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", make("a")),
|
||||
px.TaskSpec("b", make("b")),
|
||||
px.TaskSpec("c", make("c"), depends_on=("a", "b")),
|
||||
]
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("a", make("a")),
|
||||
px.TaskSpec("b", make("b")),
|
||||
px.TaskSpec("c", make("c"), depends_on=("a", "b")),
|
||||
])
|
||||
report = px.run(graph, strategy="thread", max_workers=2)
|
||||
assert report.success
|
||||
# c must finish after both a and b.
|
||||
@@ -170,12 +160,10 @@ def test_async_basic() -> None:
|
||||
async def transform(fetch: int) -> int:
|
||||
return fetch * 2
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("fetch", fetch),
|
||||
px.TaskSpec("transform", transform, depends_on=("fetch",)),
|
||||
]
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("fetch", fetch),
|
||||
px.TaskSpec("transform", transform, depends_on=("fetch",)),
|
||||
])
|
||||
report = px.run(graph, strategy="async")
|
||||
assert report.success
|
||||
assert report["transform"] == 84
|
||||
@@ -187,13 +175,11 @@ def test_async_parallelism() -> None:
|
||||
await asyncio.sleep(0.3)
|
||||
return "done"
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", slow),
|
||||
px.TaskSpec("b", slow),
|
||||
px.TaskSpec("c", slow),
|
||||
]
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("a", slow),
|
||||
px.TaskSpec("b", slow),
|
||||
px.TaskSpec("c", slow),
|
||||
])
|
||||
start = time.time()
|
||||
report = px.run(graph, strategy="async")
|
||||
elapsed = time.time() - start
|
||||
@@ -209,12 +195,10 @@ def test_async_mixed_sync_and_async() -> None:
|
||||
await asyncio.sleep(0.01)
|
||||
return sync_task + 5
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("sync_task", sync_task),
|
||||
px.TaskSpec("async_task", async_task, depends_on=("sync_task",)),
|
||||
]
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("sync_task", sync_task),
|
||||
px.TaskSpec("async_task", async_task, depends_on=("sync_task",)),
|
||||
])
|
||||
report = px.run(graph, strategy="async")
|
||||
assert report.success
|
||||
assert report["async_task"] == 15
|
||||
@@ -262,12 +246,10 @@ def test_memory_backend_resume() -> None:
|
||||
|
||||
return fn
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", make("a")),
|
||||
px.TaskSpec("b", make("b"), depends_on=("a",)),
|
||||
]
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("a", make("a")),
|
||||
px.TaskSpec("b", make("b"), depends_on=("a",)),
|
||||
])
|
||||
backend = MemoryBackend()
|
||||
_ = px.run(graph, strategy="sequential", state=backend)
|
||||
assert runs == ["a", "b"]
|
||||
@@ -393,12 +375,10 @@ def test_threaded_skips_cached_tasks() -> None:
|
||||
|
||||
return fn
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", make("a")),
|
||||
px.TaskSpec("b", make("b"), depends_on=("a",)),
|
||||
]
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("a", make("a")),
|
||||
px.TaskSpec("b", make("b"), depends_on=("a",)),
|
||||
])
|
||||
backend = px.MemoryBackend()
|
||||
# 第一次运行填充缓存
|
||||
_ = px.run(graph, strategy="thread", max_workers=2, state=backend)
|
||||
@@ -438,12 +418,10 @@ def test_async_skips_cached_tasks() -> None:
|
||||
runs.append("b")
|
||||
return a + "b"
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("a", a),
|
||||
px.TaskSpec("b", b, depends_on=("a",)),
|
||||
]
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("a", a),
|
||||
px.TaskSpec("b", b, depends_on=("a",)),
|
||||
])
|
||||
backend = px.MemoryBackend()
|
||||
_ = px.run(graph, strategy="async", state=backend)
|
||||
assert runs == ["a", "b"]
|
||||
@@ -507,3 +485,82 @@ def test_run_empty_graph() -> None:
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert len(report) == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 上游任务被 SKIPPED 后，下游任务也应被 SKIPPED
|
||||
# ---------------------------------------------------------------------- #
|
||||
def test_downstream_skipped_when_upstream_skipped_sequential() -> None:
|
||||
"""上游任务被 SKIPPED 后，下游任务也应被 SKIPPED（sequential 策略）."""
|
||||
never_true = lambda: False # noqa: E731
|
||||
|
||||
def downstream(upstream: str) -> str:
|
||||
return upstream + "_processed"
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("upstream", cmd=["echo", "hello"], conditions=(never_true,)),
|
||||
px.TaskSpec("downstream", downstream, depends_on=("upstream",)),
|
||||
])
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert report.result_of("upstream").status == px.TaskStatus.SKIPPED
|
||||
assert report.result_of("downstream").status == px.TaskStatus.SKIPPED
|
||||
|
||||
|
||||
def test_downstream_skipped_when_upstream_skipped_thread() -> None:
|
||||
"""上游任务被 SKIPPED 后，下游任务也应被 SKIPPED（thread 策略）."""
|
||||
never_true = lambda: False # noqa: E731
|
||||
|
||||
def downstream(upstream: str) -> str:
|
||||
return upstream + "_processed"
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("upstream", cmd=["echo", "hello"], conditions=(never_true,)),
|
||||
px.TaskSpec("downstream", downstream, depends_on=("upstream",)),
|
||||
])
|
||||
report = px.run(graph, strategy="thread", max_workers=2)
|
||||
assert report.success
|
||||
assert report.result_of("upstream").status == px.TaskStatus.SKIPPED
|
||||
assert report.result_of("downstream").status == px.TaskStatus.SKIPPED
|
||||
|
||||
|
||||
def test_downstream_skipped_when_upstream_skipped_async() -> None:
|
||||
"""上游任务被 SKIPPED 后，下游任务也应被 SKIPPED（async 策略）."""
|
||||
|
||||
async def upstream() -> str:
|
||||
return "hello"
|
||||
|
||||
async def downstream(upstream: str) -> str:
|
||||
return upstream + "_processed"
|
||||
|
||||
never_true = lambda: False # noqa: E731
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("upstream", upstream, conditions=(never_true,)),
|
||||
px.TaskSpec("downstream", downstream, depends_on=("upstream",)),
|
||||
])
|
||||
report = px.run(graph, strategy="async")
|
||||
assert report.success
|
||||
assert report.result_of("upstream").status == px.TaskStatus.SKIPPED
|
||||
assert report.result_of("downstream").status == px.TaskStatus.SKIPPED
|
||||
|
||||
|
||||
def test_downstream_executes_when_upstream_succeeds() -> None:
|
||||
"""上游任务成功时，下游任务应正常执行."""
|
||||
always_true = lambda: True # noqa: E731
|
||||
|
||||
def upstream() -> str:
|
||||
return "hello"
|
||||
|
||||
def downstream(upstream: str) -> str:
|
||||
return upstream + "_processed"
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("upstream", upstream, conditions=(always_true,)),
|
||||
px.TaskSpec("downstream", downstream, depends_on=("upstream",)),
|
||||
])
|
||||
report = px.run(graph, strategy="sequential")
|
||||
assert report.success
|
||||
assert report.result_of("upstream").status == px.TaskStatus.SUCCESS
|
||||
assert report.result_of("downstream").status == px.TaskStatus.SUCCESS
|
||||
assert report["downstream"] == "hello_processed"
|
||||
|
||||
Reference in New Issue
Block a user