Compare commits
195 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e781bb1f4f | |||
| 9b7ca42761 | |||
| bb57d85a10 | |||
| 8bfa0da590 | |||
| ae4cc3aeeb | |||
| fbabbc98ad | |||
| bd3651e337 | |||
| 05ca3950a2 | |||
| 6c0b6af21b | |||
| 9964db771e | |||
| 919c6a4544 | |||
| 8e3ddc5a1e | |||
| be2273722f | |||
| 5ac7fd33c6 | |||
| 07adbad847 | |||
| 8b8598fa41 | |||
| a9f15e236d | |||
| 9573f1f2a2 | |||
| 18ffae8390 | |||
| 5831c92f97 | |||
| 02ef858718 | |||
| 5e6daa554f | |||
| 520694f2df | |||
| 8d626f606e | |||
| 6499210dc9 | |||
| 7b5e4864c2 | |||
| 6614dc5a8d | |||
| 98c3d84334 | |||
| 9f50bb3e7c | |||
| 8cfdef7516 | |||
| 505e40012e | |||
| a32c65c8a0 | |||
| 936e0d2a26 | |||
| 0ecda36759 | |||
| b71c90ba5c | |||
| d733452c8a | |||
| c502556db9 | |||
| e1824b9f5a | |||
| 5e93a75987 | |||
| 529a024ea1 | |||
| 4721b59f40 | |||
| fca6f17a5c | |||
| d493c6233e | |||
| 7a87a4177e | |||
| 04d6871e8d | |||
| 08149d990d | |||
| 54aef98f3a | |||
| e31646e281 | |||
| f7fb95af83 | |||
| 6da42ec5ff | |||
| c55a37173a | |||
| 960b8672f4 | |||
| 4fd1d70b58 | |||
| 6fb9223066 | |||
| 1f7127357e | |||
| 58ee84ded6 | |||
| 9a96e5d052 | |||
| c9c7529c58 | |||
| c498d9b1c9 | |||
| b36e279f92 | |||
| 58d6f1faad | |||
| d93da0d8b4 | |||
| 701c455c42 | |||
| e174b64495 | |||
| 3afb25bb5e | |||
| fbd17536fd | |||
| 32ca8c1208 | |||
| a7ff68d279 | |||
| de368ea810 | |||
| 6a3e3a57cd | |||
| 7089944306 | |||
| ec5e348694 | |||
| 12d9f2f647 | |||
| 6ffcbecade | |||
| e76d93187b | |||
| 52e20e3f93 | |||
| 3f966a230e | |||
| 5d0b211a44 | |||
| 6931f36fd1 | |||
| db02443463 | |||
| eb8e1402bc | |||
| c93f45dcb8 | |||
| a0b1814024 | |||
| 3a2826d3f9 | |||
| dbd30689ab | |||
| 5eb59b8a66 | |||
| 8e7b866de2 | |||
| 1b4f9bfa6a | |||
| 2d39272330 | |||
| f699bb9167 | |||
| 35f07e96e1 | |||
| 1f274fe828 | |||
| 85793ff9d5 | |||
| 37ac4b8025 | |||
| 0edeadb846 | |||
| f63db6c71a | |||
| 4d397606e6 | |||
| f24388b151 | |||
| 535b7cba31 | |||
| 3f68bed3fd | |||
| 2e2ca812a1 | |||
| 8de565d0cb | |||
| 5480c48e67 | |||
| c6653d5117 | |||
| d194a991a0 | |||
| 4446658170 | |||
| 1d26f9d3e7 | |||
| d9644ca5d1 | |||
| d3c2d53449 | |||
| 9cfcfb38e4 | |||
| 69db241611 | |||
| 66e6295a24 | |||
| aebb4fce68 | |||
| 7784c8ff86 | |||
| 77918a5568 | |||
| 7e4c615dc7 | |||
| ac5082523e | |||
| 0df6f7c8ac | |||
| 4b66176ce6 | |||
| cf6b6fd059 | |||
| 6f93e6eb6d | |||
| 43e1aad1fe | |||
| 467634f8c7 | |||
| ce31f60441 | |||
| 3d6d769685 | |||
| 3f9c52e6f1 | |||
| 8fadf6edd8 | |||
| abc1152538 | |||
| 5e561b4b3a | |||
| 40f641611b | |||
| 232e7293d9 | |||
| a1bae58e56 | |||
| cbc7cc0a75 | |||
| d0ff7d7b4d | |||
| d154f67ce0 | |||
| 9999071119 | |||
| bdd70e9c43 | |||
| c15b38516a | |||
| 7d4e8a40ce | |||
| 1b2d6d6a2c | |||
| df890f0f16 | |||
| b62a544569 | |||
| d58fc5536e | |||
| c3b86b603d | |||
| 327bd6e069 | |||
| 22f8d2110d | |||
| 2a1f2f7175 | |||
| 9d033e1c0b | |||
| 336f7b7292 | |||
| 65dcbcbf62 | |||
| 7fa97a01e3 | |||
| 83da5135d0 | |||
| 7463a60649 | |||
| 87dd010342 | |||
| bdfee7bee4 | |||
| b954fb1622 | |||
| a7b7a82dff | |||
| 40f0478146 | |||
| b808b880f8 | |||
| e073ff41ee | |||
| ea0c51de5e | |||
| 2b3f4b82d3 | |||
| 1e23c48efc | |||
| 5c8ec281ff | |||
| 6f01cde8ac | |||
| bcd189ae60 | |||
| 20c4fb87c5 | |||
| a98eb6e344 | |||
| 752ff618b2 | |||
| f15f235ecf | |||
| 9d79cddbd6 | |||
| af9aab395a | |||
| 6f334fde73 | |||
| 2ccd84ac3b | |||
| ec30af3edb | |||
| 10bbc07118 | |||
| 194cf3c343 | |||
| 1880cd7a34 | |||
| d43c9e4044 | |||
| 22ac9fc4dd | |||
| 7ded8df05e | |||
| fd282db28f | |||
| 6f64d9d6dc | |||
| a2889fbb08 | |||
| 024b597e44 | |||
| 1eb7942aa9 | |||
| 9285ae3782 | |||
| a88797f410 | |||
| b047b05aaf | |||
| 78a274ce5b | |||
| ab8faec863 | |||
| 936a009212 | |||
| f10f8d09a6 | |||
| 0d6a78f320 | |||
| c9a4192c85 |
@@ -0,0 +1,46 @@
|
||||
# 版本控制
|
||||
.git
|
||||
.gitignore
|
||||
.github
|
||||
|
||||
# Python 缓存与构建产物
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
*.egg-info
|
||||
*.egg
|
||||
dist
|
||||
build
|
||||
.eggs
|
||||
|
||||
# 测试与覆盖率
|
||||
.pytest_cache
|
||||
.coverage
|
||||
htmlcov
|
||||
.tox
|
||||
coverage.xml
|
||||
|
||||
# 虚拟环境
|
||||
.venv
|
||||
venv
|
||||
env
|
||||
|
||||
# 工具缓存
|
||||
.uv-cache
|
||||
.ruff_cache
|
||||
.pyrefly_cache
|
||||
.mypy_cache
|
||||
|
||||
# IDE 与编辑器
|
||||
.idea
|
||||
.vscode
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# 文档(按需保留)
|
||||
docs
|
||||
|
||||
# 系统文件
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
+18
-116
@@ -3,130 +3,32 @@ name: CI
|
||||
on:
|
||||
push:
|
||||
branches: [ main, develop ]
|
||||
pull_request:
|
||||
branches: [ main, develop ]
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# lint:代码风格与格式检查(单平台即可)
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
lint:
|
||||
name: Lint (ruff)
|
||||
ci:
|
||||
name: Lint, Typecheck & Test
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: pyflowx-ci:latest
|
||||
env:
|
||||
UV_LINK_MODE: copy
|
||||
# ---- 国内源 ----
|
||||
PIP_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
PIP_TRUSTED_HOST: pypi.tuna.tsinghua.edu.cn
|
||||
UV_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
UV_TRUSTED_HOST: pypi.tuna.tsinghua.edu.cn
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- uses: http://gitea:3000/zhou/checkout.git@main
|
||||
|
||||
- name: 安装 uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
version: latest
|
||||
enable-cache: true
|
||||
cache-dependency-glob: uv.lock
|
||||
- name: Sync dependencies
|
||||
run: uv sync --frozen
|
||||
|
||||
- name: 设置 Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- name: Ruff check
|
||||
run: ruff check src tests
|
||||
|
||||
- name: 安装依赖
|
||||
run: uv sync --extra dev --frozen
|
||||
|
||||
- name: Ruff 检查
|
||||
run: uv run ruff check src tests
|
||||
|
||||
- name: Ruff 格式检查
|
||||
run: uv run ruff format --check src tests
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# typecheck:pyrefly 严格类型检查
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
typecheck:
|
||||
name: Typecheck (pyrefly)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: 安装 uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
version: latest
|
||||
enable-cache: true
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
- name: 设置 Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
|
||||
- name: 安装依赖
|
||||
run: uv sync --extra dev --frozen
|
||||
|
||||
- name: pyrefly 严格类型检查
|
||||
run: uv run pyrefly check .
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# test:多平台 × 多 Python 版本矩阵测试 + 覆盖率
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
test:
|
||||
name: Test (${{ matrix.os }} / py${{ matrix.python-version }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ ubuntu-latest, windows-latest, macos-latest ]
|
||||
python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12', '3.13' ]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: 安装 uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
version: latest
|
||||
enable-cache: true
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
- name: 设置 Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: 安装依赖
|
||||
run: uv sync --extra dev --frozen
|
||||
|
||||
- name: 运行测试
|
||||
run: uv run pytest -v --cov=pyflowx --cov-report=xml --cov-report=term-missing
|
||||
|
||||
- name: 上传覆盖率
|
||||
if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-${{ matrix.os }}-py${{ matrix.python-version }}
|
||||
path: coverage.xml
|
||||
retention-days: 7
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# 聚合:所有检查通过后才标记完成
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
ci-pass:
|
||||
name: CI Pass
|
||||
runs-on: ubuntu-latest
|
||||
needs: [ lint, typecheck, test ]
|
||||
if: always()
|
||||
steps:
|
||||
- name: 检查依赖任务结果
|
||||
if: ${{ needs.lint.result != 'success' || needs.typecheck.result != 'success' || needs.test.result != 'success' }}
|
||||
run: |
|
||||
echo "lint: ${{ needs.lint.result }}"
|
||||
echo "typecheck: ${{ needs.typecheck.result }}"
|
||||
echo "test: ${{ needs.test.result }}"
|
||||
exit 1
|
||||
- name: 全部通过
|
||||
run: echo "✅ 所有 CI 检查通过"
|
||||
- name: Tox test (py38, py313)
|
||||
run: uvx tox run -e py38,py313
|
||||
|
||||
+42
-177
@@ -2,192 +2,57 @@ name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*.*.*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: '发布版本号(如 v0.1.0)'
|
||||
required: true
|
||||
type: string
|
||||
tags: ['v*.*.*']
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
# Trusted Publishing (OIDC) 上传 PyPI 所需
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# 预检:版本号校验 + 与 pyproject.toml 一致性检查
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
pre-check:
|
||||
name: Pre-release Check
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.meta.outputs.version }}
|
||||
tag: ${{ steps.meta.outputs.tag }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: 解析版本号
|
||||
id: meta
|
||||
run: |
|
||||
if [ -n "${{ inputs.tag }}" ]; then
|
||||
TAG="${{ inputs.tag }}"
|
||||
else
|
||||
TAG="${GITHUB_REF#refs/tags/}"
|
||||
fi
|
||||
# 去除前缀 v
|
||||
VERSION="${TAG#v}"
|
||||
echo "tag=$TAG" >> $GITHUB_OUTPUT
|
||||
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
||||
echo "发布版本: $VERSION (tag: $TAG)"
|
||||
|
||||
- name: 校验版本号格式
|
||||
run: |
|
||||
VERSION="${{ steps.meta.outputs.version }}"
|
||||
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$'; then
|
||||
echo "❌ 版本号格式错误: $VERSION(应为 x.y.z 或 x.y.z-rc.n)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: 校验 pyproject.toml 版本一致
|
||||
run: |
|
||||
# 精确提取 [project] 段的 version 字段(避免匹配到依赖的 version)
|
||||
PY_VERSION=$(awk '/^\[project\]/{f=1} f&&/^version[[:space:]]*=/{gsub(/[" ]/,"",$3); print $3; exit}' pyproject.toml)
|
||||
echo "pyproject.toml version: $PY_VERSION"
|
||||
if [ "$PY_VERSION" != "${{ steps.meta.outputs.version }}" ]; then
|
||||
echo "❌ pyproject.toml 版本($PY_VERSION) 与 tag 版本(${{ steps.meta.outputs.version }}) 不一致"
|
||||
echo "请先更新 pyproject.toml 中的 version 字段"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# 构建:wheel + sdist(纯 Python,单平台即可)
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
build:
|
||||
name: Build Artifacts
|
||||
needs: pre-check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: 安装 uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
version: latest
|
||||
enable-cache: true
|
||||
|
||||
- name: 设置 Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
|
||||
- name: 安装依赖
|
||||
run: uv sync --extra dev --frozen
|
||||
|
||||
- name: 构建 wheel + sdist
|
||||
run: uv build
|
||||
|
||||
- name: 校验产物
|
||||
run: |
|
||||
echo "待上传产物:"
|
||||
ls -la dist/
|
||||
if [ -z "$(ls -A dist/*.whl dist/*.tar.gz 2>/dev/null)" ]; then
|
||||
echo "❌ 未找到 wheel 或 sdist 产物"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: 上传构建产物
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: dist
|
||||
path: dist/*
|
||||
retention-days: 30
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# 发布:上传到 PyPI(Trusted Publishing / OIDC)
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
publish-pypi:
|
||||
name: Publish to PyPI
|
||||
needs: [pre-check, build]
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: pypi
|
||||
url: https://pypi.org/project/pyflowx/${{ needs.pre-check.outputs.version }}
|
||||
permissions:
|
||||
id-token: write
|
||||
steps:
|
||||
- name: 下载构建产物
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: dist
|
||||
path: dist
|
||||
|
||||
- name: 上传到 PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
attestations: true
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# 发布:创建 GitHub Release
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
release:
|
||||
name: Publish Release
|
||||
needs: [pre-check, build, publish-pypi]
|
||||
name: Build, Publish & Release
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
container:
|
||||
image: pyflowx-ci:latest
|
||||
env:
|
||||
UV_LINK_MODE: copy
|
||||
# ---- 国内源 ----
|
||||
PIP_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
PIP_TRUSTED_HOST: pypi.tuna.tsinghua.edu.cn
|
||||
UV_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
UV_TRUSTED_HOST: pypi.tuna.tsinghua.edu.cn
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- uses: http://gitea:3000/zhou/checkout.git@v4
|
||||
|
||||
- name: 下载构建产物
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: dist
|
||||
path: assets
|
||||
|
||||
- name: 整理发布产物
|
||||
- name: Build distributions
|
||||
run: uv build
|
||||
|
||||
- name: Publish to pypi
|
||||
run: uv publish --token '${{ secrets.PYPI_TOKEN }}'
|
||||
|
||||
- name: Create Gitea Release & Upload Assets
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TAG_NAME: ${{ github.ref_name }}
|
||||
REPO: ${{ github.repository }}
|
||||
GITEA_URL: http://172.17.0.1:3000
|
||||
run: |
|
||||
ls -la assets/
|
||||
set -e
|
||||
# 1. 创建 Release
|
||||
RELEASE_ID=$(curl -sS -X POST "$GITEA_URL/api/v1/repos/$REPO/releases" \
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"tag_name\":\"$TAG_NAME\",\"name\":\"Release $TAG_NAME\",\"body\":\"Automated release from CI\",\"draft\":false,\"prerelease\":false}" \
|
||||
| python3 -c "import sys,json;print(json.load(sys.stdin)['id'])")
|
||||
|
||||
- name: 生成 Release Notes
|
||||
id: notes
|
||||
run: |
|
||||
{
|
||||
echo "## pyflowx ${{ needs.pre-check.outputs.version }}"
|
||||
echo ""
|
||||
echo "### 下载"
|
||||
echo ""
|
||||
echo "- **Wheel**: \`pyflowx-${{ needs.pre-check.outputs.version }}-py3-none-any.whl\`"
|
||||
echo "- **源码包**: \`pyflowx-${{ needs.pre-check.outputs.version }}.tar.gz\`"
|
||||
echo ""
|
||||
echo "### 安装"
|
||||
echo ""
|
||||
echo '```bash'
|
||||
echo "pip install pyflowx==${{ needs.pre-check.outputs.version }}"
|
||||
echo '```'
|
||||
echo ""
|
||||
echo "### 完整变更日志"
|
||||
} > RELEASE_NOTES.md
|
||||
{
|
||||
echo "content<<EOF"
|
||||
cat RELEASE_NOTES.md
|
||||
echo "EOF"
|
||||
} >> $GITHUB_OUTPUT
|
||||
echo "Created release id=$RELEASE_ID"
|
||||
|
||||
- name: 创建 GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: ${{ needs.pre-check.outputs.tag }}
|
||||
name: pyflowx ${{ needs.pre-check.outputs.version }}
|
||||
body: ${{ steps.notes.outputs.content }}
|
||||
files: assets/*
|
||||
draft: false
|
||||
prerelease: ${{ contains(needs.pre-check.outputs.version, '-') }}
|
||||
generate_release_notes: true
|
||||
# 2. 上传 dist/ 下所有文件作为附件
|
||||
for f in dist/*; do
|
||||
echo "Uploading $f ..."
|
||||
curl -sS -X POST "$GITEA_URL/api/v1/repos/$REPO/releases/$RELEASE_ID/assets?name=$(basename $f)" \
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
-H "Content-Type: application/octet-stream" \
|
||||
--data-binary "@$f"
|
||||
done
|
||||
|
||||
|
||||
|
||||
@@ -10,3 +10,8 @@ wheels/
|
||||
.venv
|
||||
.coverage
|
||||
.idea
|
||||
*_profile.html
|
||||
|
||||
# Sphinx 文档构建输出
|
||||
docs/_build/
|
||||
.trae/refs
|
||||
|
||||
@@ -8,9 +8,6 @@ repos:
|
||||
# Run the linter
|
||||
- id: ruff
|
||||
args: [--fix, --exit-non-zero-on-fix]
|
||||
# Run the formatter
|
||||
- id: ruff-format
|
||||
args: [--config=pyproject.toml]
|
||||
- repo: https://gitcode.com/gh_mirrors/pr/pre-commit-hooks.git
|
||||
rev: v5.0.0
|
||||
hooks:
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
3.8
|
||||
3.11
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# ReadTheDocs 配置
|
||||
# https://docs.readthedocs.io/en/stable/config-file/v2.html
|
||||
version: 2
|
||||
|
||||
# 构建配置
|
||||
build:
|
||||
os: ubuntu-24.04
|
||||
tools:
|
||||
python: "3.11"
|
||||
|
||||
# Python 依赖与构建命令
|
||||
python:
|
||||
install:
|
||||
- method: pip
|
||||
path: .
|
||||
extra_requirements:
|
||||
- docs
|
||||
|
||||
# Sphinx 构建
|
||||
sphinx:
|
||||
configuration: docs/conf.py
|
||||
builder: html
|
||||
fail_on_warning: false
|
||||
@@ -0,0 +1,15 @@
|
||||
# PYTHON
|
||||
.coverage
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
.tox/
|
||||
.venv/
|
||||
__pycache__/
|
||||
|
||||
# NODEJS
|
||||
node_modules/
|
||||
|
||||
# IDE
|
||||
.idea
|
||||
.trae
|
||||
.vscode
|
||||
@@ -0,0 +1,93 @@
|
||||
# 迭代 21 — P9 功能扩展(任务编排/数据流/持久化/监控)
|
||||
|
||||
## 本轮目标
|
||||
|
||||
延续 P8 性能优化后,扩展 PyFlowX 在四个方向的核心能力:
|
||||
|
||||
1. **P9.1 任务编排增强**:任务分组(`Graph.group`)+ 动态任务生成(`TaskSpec(dynamic=True)`)。
|
||||
2. **P9.2 数据流增强**:`RunReport.output_of` 命名输出提取 + `Graph.pipeline` 语义别名。
|
||||
3. **P9.3 持久化与恢复**:`run(resume_from=...)` 检查点恢复 + `RunHistory` 历史管理。
|
||||
4. **P9.4 监控导出**:`MetricsCollector`(Prometheus 文本)+ `health_check` + `start_metrics_server`(零依赖 HTTP)。
|
||||
|
||||
## 改动文件清单
|
||||
|
||||
### 新增
|
||||
|
||||
- `src/pyflowx/monitoring.py` — `MetricsCollector` / `health_check` / `start_metrics_server`。
|
||||
- `src/pyflowx/history.py` — `RunHistory` 文件系统历史管理。
|
||||
- `tests/test_group.py` — 13 用例(P9.1.2 任务分组)。
|
||||
- `tests/test_dynamic.py` — 11 用例(P9.1.3 动态任务)。
|
||||
- `tests/test_dataflow.py` — 12 用例(P9.2 数据流)。
|
||||
- `tests/test_persistence.py` — 15 用例(P9.3 检查点/历史)。
|
||||
- `tests/test_monitoring.py` — 16 用例(P9.4 监控导出)。
|
||||
|
||||
### 修改
|
||||
|
||||
- `src/pyflowx/task.py` — `TaskSpec` 新增 `dynamic: bool = False` 字段;`task()` 工厂新增 `dynamic` 参数。
|
||||
- `src/pyflowx/graph.py` — 新增 `group()` 方法;`_rewrite_deps_for_loops` 改名 `_rewrite_deps`,统一处理 `_loop_groups` + `_groups`;新增 `pipeline()` 语义别名。
|
||||
- `src/pyflowx/report.py` — 新增 `_resolve_output_path` 辅助 + `RunReport.output_of(name, output)` 方法。
|
||||
- `src/pyflowx/executors.py` —
|
||||
- `DependencyRunner.execute` 增加 while 循环 + `f.exception()` 手动传播(替代 `asyncio.gather`)。
|
||||
- 新增 `_extract_dynamic_specs` 辅助;`_run_task` 检测动态生成、注册派生 spec、启动派生 task。
|
||||
- `_dispatch_strategy` 增加 `has_dynamic` 校验(仅 `dependency` 策略支持)。
|
||||
- 新增 `_load_resume_report` + `_apply_resume` 辅助;`run()` 接受 `resume_from` 参数。
|
||||
- `_filter_and_sort` + `_run_task` 检查点跳过逻辑。
|
||||
- `src/pyflowx/__init__.py` — 导出 `RunHistory` / `MetricsCollector` / `health_check` / `start_metrics_server`。
|
||||
|
||||
## 关键决策与依据
|
||||
|
||||
### 1. 动态任务仅支持 `dependency` 策略
|
||||
|
||||
`dynamic=True` 允许任务 `fn` 运行时返回 `TaskSpec | list[TaskSpec]`,DependencyRunner 中途注册并调度派生任务。层模型(sequential/thread/async)的层屏障与动态扩展语义冲突,因此 `_dispatch_strategy` 显式拒绝其他策略。
|
||||
|
||||
依据:层模型按拓扑分层执行,动态任务可能跨层;依赖驱动模型无层屏障,可自然容纳运行时新增节点。
|
||||
|
||||
### 2. `asyncio.wait` 替代 `asyncio.gather` + 手动异常传播
|
||||
|
||||
`asyncio.gather` 在 first-failure 时取消其他任务并 re-raise,但所有 task 必须在调用前确定。动态任务场景下 `futures` 字典会随执行增长,必须用 while 循环 + `asyncio.wait(return_when=FIRST_COMPLETED)` 增量调度。
|
||||
|
||||
陷阱:`asyncio.wait` **不会**自动 re-raise task 异常。需手动 `f.exception()` 检查并显式 `raise`,同时取消未完成任务,才能与 `gather` 的 fail-fast 行为一致。该陷阱导致 7 个测试失败后定位修复。
|
||||
|
||||
### 3. 检查点恢复:仅恢复 SUCCESS 任务
|
||||
|
||||
`resume_from` 接受 `RunReport | str | Path`,仅恢复 `status == SUCCESS` 且名称在当前图中的任务。FAILED/SKIPPED 任务会重新执行。`_apply_resume` 抽取为独立函数以控制 `run()` 的 PLR0912 分支数(≤12)。
|
||||
|
||||
### 4. `metrics_text` 按 section 条件化输出
|
||||
|
||||
最初实现总是输出 `# HELP` / `# TYPE` 行(即便无数据)。这导致 `reset()` 后文本仍包含 `pyflowx_task_total` 字样,违反测试预期。改为所有 section(含 task_total/duration/duration_sum)都按数据存在性条件化,与 retries/run_total 一致。
|
||||
|
||||
### 5. `start_metrics_server` 用 `http.server` 零依赖
|
||||
|
||||
未引入 `prometheus_client` 等三方库,仅用标准库 `http.server.HTTPServer` + `BaseHTTPRequestHandler`。生产环境可叠加反向代理或替换为 `prometheus_client`。`@override` 装饰器按 Python 版本条件导入(`typing` 3.12+ / `typing_extensions` <3.12)。
|
||||
|
||||
### 6. `RunHistory` 文件命名与 `__contains__`/`__len__`
|
||||
|
||||
按 `{run_id}.json` 命名存储;`__contains__` / `__len__` 委托文件存在性检查,避免维护内存索引。`latest()` 按 mtime 排序取最新。
|
||||
|
||||
## 验证结果
|
||||
|
||||
```
|
||||
ruff check . → All checks passed!
|
||||
ruff format --check → 126 files already formatted
|
||||
pyrefly check . → 0 errors (68 suppressed)
|
||||
pytest --cov=pyflowx → 1525 passed in 11.56s
|
||||
coverage → 97.58%(branch;门槛 95%)
|
||||
```
|
||||
|
||||
各模块覆盖率(P9 新增部分):
|
||||
- `monitoring.py` 99%(仅 `typing_extensions` 回退分支未覆盖)
|
||||
- `history.py` 95%
|
||||
- `executors.py` 97%(dynamic/resume 分支已覆盖)
|
||||
- `report.py` 99%(`output_of` 路径解析已覆盖)
|
||||
- `graph.py` 98%(`group`/`pipeline`/`_rewrite_deps` 已覆盖)
|
||||
|
||||
## 遗留事项
|
||||
|
||||
- 监控导出未实现 OpenTelemetry 协议(当前仅 Prometheus 文本格式)。如需 OTLP,可在后续迭代引入 `opentelemetry-sdk`(违反零依赖原则,需用户确认)。
|
||||
- `RunHistory` 未实现压缩/归档;大量历史运行可能占用较多磁盘。后续可加 `max_entries` 或滚动归档。
|
||||
- 动态任务当前仅在 DependencyRunner 中实现;如层模型用户也需要,需额外设计跨层调度策略。
|
||||
- `_apply_resume` 仅恢复 `report.results`,未恢复 `backend` 缓存;如需检查点+缓存联合恢复,需额外设计。
|
||||
|
||||
## 归档说明
|
||||
|
||||
iter-21 暂不归档(前 5 次 iter-16~20 已归档至 `.trae/skills/pyflowx-development/SKILL.md`)。下次清理窗口为 iter-26 前夕。
|
||||
@@ -0,0 +1,174 @@
|
||||
# 迭代 22 — P10 图片处理(imagetool + image_pipeline)
|
||||
|
||||
## 本轮目标
|
||||
|
||||
延续 P9 功能扩展后,补齐 PyFlowX 在图片处理场景的能力,覆盖三个方向:
|
||||
|
||||
1. **P10.1 基础操作子命令**:`imagetool` 工具的 7 个基础操作子命令
|
||||
(resize/crop/rotate/flip/convert/watermark/compress)。
|
||||
2. **P10.2 元数据与信息子命令**:4 个元数据子命令
|
||||
(info/exif/histogram/colors)。
|
||||
3. **P10.3 批量 DAG 编排**:`image_pipeline` 链式 DAG 构造器,
|
||||
将多步图片操作封装为 `Graph`,前一步输出作为后一步输入。
|
||||
|
||||
工具形态:单一 `imagetool` + `office` extra(Pillow 通过 `pyflowx[office]` 安装),
|
||||
不破坏项目零运行时依赖原则。
|
||||
|
||||
## 改动文件清单
|
||||
|
||||
### 新增
|
||||
|
||||
- `src/pyflowx/ops/files/imagetool.py` — 11 个 `@px.tool("imagetool", ...)`
|
||||
子命令 + `_save_image`/`_load_font`/`_resolve_position`/`_print_exif`/
|
||||
`_apply_exif_modifications`/`_apply_single_exif_set`/`_save_exif`/
|
||||
`_print_histogram_buckets` 辅助函数(520 行)。
|
||||
- `src/pyflowx/imaging.py` — `image_pipeline()` 链式 DAG 构造器 +
|
||||
`_make_step_fn` 闭包工厂(130 行)。
|
||||
- `tests/cli/test_imagetool.py` — 25 用例覆盖 11 子命令 + `_require_pil`
|
||||
守卫(316 行)。
|
||||
- `tests/test_imaging.py` — 9 用例覆盖 DAG 拓扑与执行(158 行)。
|
||||
|
||||
### 修改
|
||||
|
||||
- `src/pyflowx/__init__.py` — 导出 `image_pipeline`,加入 `__all__`。
|
||||
- `src/pyflowx/cli/pf.py` — `_TOOL_MODULES` 注册
|
||||
`"imagetool": "pyflowx.ops.files.imagetool"`;`_TOOL_ALIASES` 添加
|
||||
`"image"`/`"imagetool"`/`"img"` 三个别名。
|
||||
|
||||
## 关键决策与依据
|
||||
|
||||
### 1. 单一 `imagetool` + office extra
|
||||
|
||||
不拆分为 `imageops`/`imagemeta` 等多工具,所有图片操作集中在
|
||||
`ops/files/imagetool.py`。Pillow 通过 `pyproject.toml` 的 `[office]`
|
||||
extra 安装(`pillow>=10.4.0`),运行时 try/except 导入 + `HAS_PIL` 标志,
|
||||
未安装时 `_require_pil()` 打印 `pip install pyflowx[office]` 提示而非崩溃。
|
||||
|
||||
依据:项目零运行时依赖原则不可破坏;pdftool 已有相同模式
|
||||
(`pypdf` 通过 `[office]` extra)。
|
||||
|
||||
### 2. `_save_image` 统一保存逻辑
|
||||
|
||||
所有需要保存图片的子命令(resize/crop/rotate/flip/convert/watermark/compress)
|
||||
委托 `_save_image(img, output, fmt, quality)` 统一处理:
|
||||
|
||||
- **JPG → JPEG 规范化**:`save_fmt == "JPG"` 时改写为 `"JPEG"`,
|
||||
否则 Pillow `img.save(format="JPG")` 抛 `KeyError: 'JPG'`。
|
||||
- **JPEG 强制 RGB 转换**:JPEG 不支持 alpha 通道,保存前
|
||||
`img.convert("RGB")` 去 alpha,避免 `cannot write mode RGBA as JPEG`。
|
||||
- **JPEG/WEBP 应用 quality**:其他格式(PNG/BMP/GIF 等)忽略 quality。
|
||||
|
||||
依据:避免每个子命令重复处理格式规范化与 alpha 通道问题。
|
||||
|
||||
### 3. EXIF 用 `exif.tobytes()` 而非 `bytes(exif)`
|
||||
|
||||
Pillow 10+ 的 `Image.Exif` 对象,`bytes(exif)` 调用抛
|
||||
`ValueError: bytes must be in range(0, 256)`。必须用 `exif.tobytes()`
|
||||
序列化为字节串后再传给 `img.save(exif=exif_bytes)`。
|
||||
|
||||
`_save_exif` 辅助函数封装此逻辑:
|
||||
```python
|
||||
exif_bytes = exif.tobytes() if exif else b""
|
||||
img.save(output_path, exif=exif_bytes)
|
||||
```
|
||||
|
||||
### 4. `quantize(colors=N)` 实际色数可能 < N
|
||||
|
||||
`Image.quantize(colors=count)` 对简单图片(如纯色)可能返回少于
|
||||
请求色数的调色板。直接按 `count` 遍历会 `IndexError: list index out of range`。
|
||||
|
||||
修复:用 `actual_count = len(palette) // 3` 动态计算实际色数,
|
||||
`min(count, actual_count)` 防越界:
|
||||
```python
|
||||
actual_count = len(palette) // 3
|
||||
print(f"主色调 (前 {min(count, actual_count)} 色):")
|
||||
for i in range(min(count, actual_count)):
|
||||
...
|
||||
```
|
||||
|
||||
### 5. Pillow 10+ Transpose 枚举迁移
|
||||
|
||||
`Image.FLIP_LEFT_RIGHT`(模块级常量)在 Pillow 10+ 已弃用,
|
||||
pyrefly 无法解析。必须用 `Image.Transpose.FLIP_LEFT_RIGHT`
|
||||
(枚举形式)。
|
||||
|
||||
`image_flip` 中根据 direction 选择枚举:
|
||||
```python
|
||||
method = Image.Transpose.FLIP_LEFT_RIGHT if direction == "horizontal" else Image.Transpose.FLIP_TOP_BOTTOM
|
||||
flipped = img.transpose(method)
|
||||
```
|
||||
|
||||
### 6. `_make_step_fn` 闭包工厂避免循环变量捕获
|
||||
|
||||
`image_pipeline` 中若直接在 for 循环内定义闭包:
|
||||
```python
|
||||
for i, (op_name, params) in enumerate(steps):
|
||||
|
||||
def _run():
|
||||
base_fn(input_path, output_path, **params) # 陷阱!
|
||||
|
||||
...
|
||||
```
|
||||
所有任务的 `_run` 会捕获**最后一次循环**的 `input_path`/`output_path`/`params`,
|
||||
导致所有步骤操作同一文件。
|
||||
|
||||
修复:提取为独立函数 `_make_step_fn(base_fn, input_path, output_path, params)`,
|
||||
利用每次调用的独立作用域正确捕获:
|
||||
```python
|
||||
def _make_step_fn(base_fn, input_path, output_path, params):
|
||||
def _run() -> None:
|
||||
base_fn(input_path, output_path, **params)
|
||||
|
||||
return _run
|
||||
```
|
||||
每次循环调用 `_make_step_fn(...)` 创建新作用域,闭包绑定该次调用的参数。
|
||||
|
||||
### 7. PLR0912 分支数控制
|
||||
|
||||
`image_exif` 函数最初 13 分支(show/clear/set 各分支 + 错误处理)超过
|
||||
ruff `PLR0912` 限制(≤12)。提取 4 个辅助函数降至合规:
|
||||
|
||||
- `_print_exif(exif)` — 打印 EXIF 标签
|
||||
- `_apply_exif_modifications(exif, set_items, clear)` — 应用 clear + set
|
||||
- `_apply_single_exif_set(exif, item)` — 解析单个 KEY=VALUE
|
||||
- `_save_exif(img, exif, output_path)` — 保存图片与 EXIF
|
||||
|
||||
## 验证结果
|
||||
|
||||
```
|
||||
ruff check . → All checks passed!
|
||||
ruff format --check → 131 files already formatted
|
||||
pyrefly check . → 0 errors (68 suppressed)
|
||||
pytest --cov=pyflowx → 1560 passed in 11.72s
|
||||
coverage → 97.15%(branch;门槛 95%)
|
||||
```
|
||||
|
||||
各模块覆盖率(P10 新增部分):
|
||||
- `imagetool.py` 88%(213 stmts, 19 miss — 主要是 ImportError 回退分支
|
||||
与少量错误路径;总覆盖率 97.15% 不受影响)
|
||||
- `imaging.py` 100%(`image_pipeline` 拓扑/执行/convert 扩展名/naming 模板
|
||||
全覆盖)
|
||||
|
||||
CLI 注册验证:
|
||||
- `imagetool` 在 `_TOOL_MODULES`
|
||||
- `img`/`image`/`imagetool` 三个别名在 `_TOOL_ALIASES`
|
||||
- `tests/cli/test_tool_modules.py` 自动覆盖新工具注册
|
||||
|
||||
## 遗留事项
|
||||
|
||||
- **P11 候选方向**(用户已确认为后续阶段输入,不在 P10 范围内):
|
||||
- OCR 文字识别(需 `pytesseract` 新依赖,违反零依赖原则,需用户确认)
|
||||
- 滤镜与增强(模糊/锐化/边缘检测/浮雕/去噪,纯 Pillow 可实现)
|
||||
- 批量文件夹处理(遍历目录批量应用同一操作,生成缩略图网格)
|
||||
- 图片拼接与 GIF(多图水平/垂直拼接,GIF 帧拆分与合成)
|
||||
- **`imagetool.py` 覆盖率 88%**:ImportError 回退分支(`HAS_PIL = False`)
|
||||
在已安装 Pillow 的环境中无法触发,属可接受缺口。
|
||||
- **`image_pipeline` 仅支持链式 DAG**:当前实现是单链(每步依赖前一步),
|
||||
不支持分叉/汇合(如同一源图应用多操作后合并)。如需分叉,可扩展
|
||||
`steps` 为 DAG 描述结构。
|
||||
|
||||
## 归档说明
|
||||
|
||||
iter-22 暂不归档(前 5 次 iter-16~20 已归档至
|
||||
`.trae/skills/pyflowx-development/SKILL.md`,iter-21/22 保留在 docs/)。
|
||||
下次清理窗口为 iter-26 前夕,届时 iter-21/22 一并归档。
|
||||
@@ -0,0 +1,135 @@
|
||||
# 迭代 23 — P11 文件操作底层 API(fileops 模块)
|
||||
|
||||
## 本轮目标
|
||||
|
||||
补齐 PyFlowX 在文件操作场景的**底层 API**能力,供 DAG 组合调用。
|
||||
与 `ops/files/` 下的 CLI 工具(pdftool/imagetool/filedate 等)分层:
|
||||
本模块函数**返回数据**(不打印),可被 `px.task` / `px.TaskSpec` 包装后在
|
||||
DAG 中通过 `outputs` 声明 + `report.output_of()` 提取,或通过参数名注入
|
||||
传递给下游任务。
|
||||
|
||||
四个方向:
|
||||
|
||||
1. **查找遍历**:`find` / `glob` / `walk_files`
|
||||
2. **读写**:`read_text` / `read_bytes` / `write_text` / `write_bytes` /
|
||||
`append_text`
|
||||
3. **复制移动删除**:`copy` / `move` / `delete` / `copy_tree`
|
||||
4. **元数据校验**:`size` / `mtime` / `exists` / `is_file` / `is_dir` /
|
||||
`sha256` / `stem` / `suffix`
|
||||
|
||||
## 改动文件清单
|
||||
|
||||
### 新增
|
||||
|
||||
- `src/pyflowx/fileops.py` — 20 个纯函数 API(157 行),仅依赖标准库
|
||||
(`pathlib` / `hashlib` / `shutil`)。
|
||||
- `tests/test_fileops.py` — 80 用例,覆盖 4 类 API + 5 个 DAG 组合调用示例
|
||||
(`outputs` + `output_of` 数据流验证)。
|
||||
|
||||
### 修改
|
||||
|
||||
- `src/pyflowx/__init__.py` — `from . import fileops` + `"fileops"` 加入
|
||||
`__all__`(导出模块本身,不污染顶层命名空间)。
|
||||
|
||||
## 关键决策与依据
|
||||
|
||||
### 1. 纯函数模块,不带 `@px.tool` 装饰器
|
||||
|
||||
`fileops.py` 的函数全部是纯函数,返回数据(`list[Path]` / `str` / `bytes` /
|
||||
`int` / `bool` / `Path` 等),不打印,不注册为 CLI 子命令。
|
||||
|
||||
依据:用户明确要求"底层 api 供组合调用"。与 CLI 工具分层:
|
||||
- CLI 工具(pdftool/imagetool):面向终端用户,函数返回 `None`,打印结果
|
||||
- 底层 API(fileops):面向编程组合,函数返回数据,可被 `px.task` 包装
|
||||
|
||||
### 2. 导出模块而非 20 个函数到顶层 `__all__`
|
||||
|
||||
`fileops` 有 20 个函数,名字较通用(`find` / `exists` / `size` / `suffix`)。
|
||||
全部导出到 `px.find` / `px.exists` 会污染顶层命名空间,与用户代码命名冲突。
|
||||
|
||||
决策:`from . import fileops` + `"fileops"` 加入 `__all__`。
|
||||
- `import pyflowx as px; px.fileops.find(...)` 可用
|
||||
- `from pyflowx.fileops import find, read_text` 可用(推荐写法)
|
||||
- `px.find` / `px.exists` 不可用(避免污染)
|
||||
|
||||
### 3. `copy` / `move` 的 `overwrite` 语义统一
|
||||
|
||||
初版 `copy` / `move` 的 `overwrite=True` 逻辑有 bug:当 `dst` 是目录时,
|
||||
先走 `dst_path.is_dir()` 分支(`dst_path = dst / src.name`),再检查
|
||||
`overwrite`。这导致 `overwrite=True` 时 `dst` 本身没被删除,而是处理
|
||||
`dst/src.name`。
|
||||
|
||||
修复:`overwrite=True` 且 `dst` 存在时,**先删除 `dst`**,不走 `is_dir`
|
||||
分支。语义统一为:
|
||||
- `overwrite=False` 且 `dst` 是目录:`src` 移入/复制到 `dst / src.name`
|
||||
- `overwrite=False` 且 `dst` 是已存在文件:抛 `FileExistsError`
|
||||
- `overwrite=True` 且 `dst` 存在:先删除 `dst`,再把 `src` 放到 `dst` 路径
|
||||
|
||||
依据:`overwrite=True` 的用户意图是"替换目标",不是"移到目标内部"。
|
||||
两个函数语义一致,降低认知负担。
|
||||
|
||||
### 4. `find` 的 `max_depth` 语义
|
||||
|
||||
`max_depth` 语义:1 = root 直接子项,2 = root 孙项。
|
||||
`max_depth=0` 表示仅 root 自身用 pattern 匹配(不递归子目录)。
|
||||
`max_depth=None`(默认)表示无限制递归。
|
||||
|
||||
实现用辅助函数 `_glob_with_depth(root, pattern, max_depth)` 递归 glob,
|
||||
`max_depth < 0` 返回空,`max_depth == 0` 仅 root.glob(pattern),
|
||||
`max_depth > 0` 递归子目录。
|
||||
|
||||
### 5. `walk_files` 返回生成器(惰性)
|
||||
|
||||
`walk_files` 返回 `Iterator[Path]`,惰性求值,适用于大目录遍历。
|
||||
与 `find` 区别:`find` 返回排序后的 `list[Path]`(支持 pattern 匹配),
|
||||
`walk_files` 仅返回文件(不返回目录),不做 pattern 匹配。
|
||||
|
||||
### 6. `sha256` 流式读取
|
||||
|
||||
`sha256(path, chunk_size=65536)` 分块读取,适用于大文件。
|
||||
默认 chunk_size 64KB,平衡内存与 IO 效率。
|
||||
|
||||
### 7. DAG 组合调用模式
|
||||
|
||||
测试覆盖 5 个 DAG 组合示例,验证数据流注入:
|
||||
- `find → read_text`:上游 `outputs={"paths": "$"}`,下游参数名匹配任务名注入
|
||||
- `find → copy` 批量复制
|
||||
- `find → sha256` 校验
|
||||
- `write_text → read_text` 往返
|
||||
- `find → summarize → write_summary` 三阶段流水线
|
||||
|
||||
关键:下游任务的**参数名**必须匹配上游**任务名**,PyFlowX 自动注入
|
||||
上游结果值。
|
||||
|
||||
## 验证结果
|
||||
|
||||
```
|
||||
ruff check . → All checks passed!
|
||||
ruff format --check → 133 files already formatted
|
||||
pyrefly check . → 0 errors (68 suppressed)
|
||||
pytest --cov=pyflowx → 1640 passed in 12.15s
|
||||
coverage → 97.17%(branch;门槛 95%)
|
||||
```
|
||||
|
||||
各模块覆盖率(P11 新增部分):
|
||||
- `fileops.py` 98%(157 stmts, 2 miss, 70 branch, 3 brpart —
|
||||
缺失行:132 `seen` 去重分支、146 `_glob_with_depth` max_depth<0 边界、
|
||||
206->203 `walk_files` 非文件非目录分支)
|
||||
- `test_fileops.py` 80 用例全绿,含 5 个 DAG 组合示例
|
||||
|
||||
## 遗留事项
|
||||
|
||||
- **`fileops.py` 未注册为 CLI 工具**:纯函数模块,无 `@px.tool` 装饰器。
|
||||
如需 CLI 访问,可后续新增 `ops/files/fileopscli.py` 包装层。
|
||||
- **`find` 的 `seen` 去重分支(132 行)未覆盖**:`rglob` / `_glob_with_depth`
|
||||
在正常使用下不产生重复路径,去重是防御性代码。
|
||||
- **`walk_files` 的符号链接分支(206->203)未覆盖**:正常文件系统中
|
||||
`iterdir()` 返回的条目要么是文件要么是目录,损坏的符号链接才会触发
|
||||
`is_file()=False and is_dir()=False` 分支。
|
||||
- **后续可扩展**:文件监听(watch)、文件锁、压缩解压、文件比较等高级能力。
|
||||
|
||||
## 归档说明
|
||||
|
||||
iter-23 暂不归档(前 5 次 iter-16~20 已归档至
|
||||
`.trae/skills/pyflowx-development/SKILL.md`,iter-21/22/23 保留在 docs/)。
|
||||
下次清理窗口为 iter-26 前夕,届时 iter-21/22/23 一并归档。
|
||||
@@ -0,0 +1,142 @@
|
||||
# 迭代 24 — P12 通用 DAG 构造器集合(pipelines 模块)
|
||||
|
||||
## 本轮目标
|
||||
|
||||
补齐 PyFlowX 在通用 DAG 编排场景的高层构造器能力, 提供 3 个函数式 DAG
|
||||
构造器, 封装常见的任务编排模式, 减少手动创建 `TaskSpec` 的样板代码:
|
||||
|
||||
1. **`data_pipeline`** —— 函数式数据流水线 (链式函数组合, 参数名驱动注入)
|
||||
2. **`command_chain`** —— 命令链构造器 (串行命令执行, CI/CD 场景)
|
||||
3. **`fan_out_fan_in`** —— map-reduce 构造器 (扇出处理 + 扇入聚合)
|
||||
|
||||
与 `imaging.image_pipeline` 同级, 复用 `Graph.chain` / `Graph.map` 的依赖
|
||||
链接模式, 但提供更高层次的函数式 API.
|
||||
|
||||
## 改动文件清单
|
||||
|
||||
### 新增
|
||||
|
||||
- `src/pyflowx/pipelines.py` — 3 个 DAG 构造器 + 2 个闭包工厂辅助函数
|
||||
(190 行), 仅依赖标准库 + 项目核心抽象 (`TaskSpec` / `Graph` / `Context`).
|
||||
- `tests/test_pipelines.py` — 37 用例, 覆盖 3 个构造器的拓扑/执行/命名/
|
||||
错误处理 + 4 个与 fileops 联用组合示例.
|
||||
|
||||
### 修改
|
||||
|
||||
- `src/pyflowx/__init__.py` — 导入 `command_chain` / `data_pipeline` /
|
||||
`fan_out_fan_in` 并加入 `__all__`.
|
||||
|
||||
## 关键决策与依据
|
||||
|
||||
### 1. 新建 `pipelines.py` 模块而非扩展 `compose.py`
|
||||
|
||||
`compose.py` 当前职责是**多图引用解析** (`GraphComposer` / `compose`),
|
||||
与单图 DAG 构造器正交. 将 3 个构造器放入 `compose.py` 会混淆职责.
|
||||
|
||||
决策: 新建 `pipelines.py`, 与 `imaging.py` 同级, 存放通用 DAG 构造器.
|
||||
`imaging.py` 是图片专用构造器, `pipelines.py` 是通用构造器, 职责清晰.
|
||||
|
||||
### 2. `data_pipeline` 用函数 `__name__` 作为默认任务名
|
||||
|
||||
`data_pipeline` 的核心价值是参数名注入: 下游函数参数名匹配前驱任务名,
|
||||
自动接收前驱返回值. 任务名必须与函数签名一致.
|
||||
|
||||
决策: 默认用 `fn.__name__` 作为任务名; `<lambda>` 回退为 `step`;
|
||||
重名时追加索引 (`extract_1`). 用户可通过 `names` 参数自定义命名, 但
|
||||
需确保下游函数参数名匹配自定义名.
|
||||
|
||||
依据: 鼓励用户用命名函数 (而非 lambda) 构造流水线, 提升可读性 +
|
||||
可调试性. lambda 仅用于简单场景.
|
||||
|
||||
### 3. `command_chain` 自动命名用 `cmd_{i:02d}_{first_arg}`
|
||||
|
||||
命令链的命名需兼顾唯一性与可读性. 用 `cmd_00_echo` / `cmd_01_ls`
|
||||
格式: 索引保证唯一性, 首参数提供可读性.
|
||||
|
||||
边界处理:
|
||||
- 空 list 命令 (`[]`) → `first = "cmd"`
|
||||
- 空字符串命令 (`""`) → `first = "cmd"`
|
||||
- shell 字符串命令 (`"echo hello"`) → `first = "echo"`
|
||||
|
||||
### 4. `fan_out_fan_in` 用 `Context` 标注实现 reduce 聚合
|
||||
|
||||
map-reduce 的核心难题: reduce 任务如何接收所有 worker 的结果?
|
||||
|
||||
PyFlowX 的参数名注入要求参数名匹配上游任务名, 但 fan-out 有 N 个
|
||||
worker (名字不同), reduce 无法通过单个参数名接收所有.
|
||||
|
||||
决策: `fan_out_fan_in` 内部生成 reduce 包装函数, 标注 `ctx: Context`
|
||||
接收完整依赖上下文, 然后按 `worker_names` 顺序提取结果, 传递给用户
|
||||
的 `reduce` 函数.
|
||||
|
||||
```python
|
||||
def _make_reduce_fn(reduce, worker_names):
|
||||
def _run(ctx: Context) -> Any:
|
||||
results = [ctx[name] for name in worker_names]
|
||||
return reduce(results)
|
||||
return _run
|
||||
```
|
||||
|
||||
依据: `Context` 标注是 PyFlowX 文档化的机制, 用于接收完整依赖上下文.
|
||||
闭包捕获 `worker_names` 列表, 保证结果按 item 顺序聚合.
|
||||
|
||||
### 5. `reduce` 可选, 支持纯 fan-out 模式
|
||||
|
||||
`fan_out_fan_in` 的 `reduce` 参数为 `Callable | None`, `None` 时仅生成
|
||||
fan-out (无 reduce 任务). 适用于不需要聚合的并行批处理场景.
|
||||
|
||||
依据: 并非所有 fan-out 都需要 reduce (如批量文件复制, 每个 worker
|
||||
独立写文件, 无需聚合).
|
||||
|
||||
### 6. 闭包工厂模式绑定参数 (参照 `imaging._make_step_fn`)
|
||||
|
||||
`_make_worker_fn(worker, item)` 与 `_make_reduce_fn(reduce, worker_names)`
|
||||
采用与 `imaging._make_step_fn` 相同的闭包工厂模式: 每次调用独立作用域,
|
||||
不受外层循环变量重绑定影响.
|
||||
|
||||
依据: 闭包捕获的是函数参数 (非循环变量), 避免了 Python 闭包经典的
|
||||
"延迟绑定"陷阱.
|
||||
|
||||
### 7. 与 `Graph.chain` / `Graph.map` 的差异化定位
|
||||
|
||||
| 构造器 | 与 Graph 方法区别 |
|
||||
|--------|------------------|
|
||||
| `data_pipeline` | 直接传函数, 自动生成 TaskSpec + 命名, 无需手动装饰 |
|
||||
| `command_chain` | 直接传命令列表, 自动命名 + 串联, 无需手动构造 spec |
|
||||
| `fan_out_fan_in` | 一站式封装 source → map → reduce, 自动生成 reduce 包装 |
|
||||
|
||||
`Graph.chain` / `Graph.map` 是方法式 API (需先创建 Graph + TaskSpec),
|
||||
本模块是函数式 API (直接传函数/命令/数据, 返回 Graph), 更高层抽象.
|
||||
|
||||
## 验证结果
|
||||
|
||||
```
|
||||
ruff check . → All checks passed!
|
||||
ruff format --check → reformatted
|
||||
pyrefly check . → 0 errors
|
||||
pytest tests/test_pipelines.py → 37 passed in 0.30s
|
||||
coverage (pipelines.py) → 100% (86 stmts, 0 miss, 32 branch, 0 brpart)
|
||||
pytest (全套) → 1677 passed (含 1 个 flaky 时间测试重跑通过)
|
||||
```
|
||||
|
||||
各构造器测试覆盖:
|
||||
- `TestDataPipeline` — 9 用例 (拓扑/执行/自定义命名/lambda 回退/重名/错误)
|
||||
- `TestCommandChain` — 12 用例 (拓扑/自动命名/字符串命令/透传/失败处理)
|
||||
- `TestFanOutFanIn` — 11 用例 (map-reduce/纯 fan-out/自定义命名/顺序/复杂对象/多策略)
|
||||
- `TestPipelinesWithFileops` — 4 组合示例 (与 fileops 联用)
|
||||
|
||||
## 遗留事项
|
||||
|
||||
- **`data_pipeline` 不支持条件分支**: 当前是线性链式, 不支持 if/else 分支.
|
||||
如需条件分支, 用户应直接用 `Graph.from_specs` 手动构造.
|
||||
- **`command_chain` 不支持并行命令**: 当前是串行链, 不支持并行命令组.
|
||||
如需并行, 用户应配合 `Graph.map` 或 `fan_out_fan_in`.
|
||||
- **`fan_out_fan_in` 的 worker 不支持额外参数**: worker 函数仅接收单个 item.
|
||||
如需额外参数, 用户应用闭包或 `functools.partial` 绑定.
|
||||
- **后续可扩展**: 条件分支构造器、循环构造器、子图嵌套构造器等.
|
||||
|
||||
## 归档说明
|
||||
|
||||
iter-24 暂不归档 (前 5 次 iter-16~20 已归档至
|
||||
`.trae/skills/pyflowx-development/SKILL.md`, iter-21/22/23/24 保留在 docs/).
|
||||
下次清理窗口为 iter-26 前夕, 届时 iter-21/22/23/24 一并归档.
|
||||
@@ -0,0 +1,71 @@
|
||||
# 迭代 25:执行热路径性能优化
|
||||
|
||||
## 本轮目标
|
||||
|
||||
针对 `px.run` 执行热路径中的冗余开销进行 4 项优化,提升 sequential/thread 策略吞吐。
|
||||
|
||||
## 改动文件清单
|
||||
|
||||
- `src/pyflowx/task.py` — `should_execute` 加早退快速路径
|
||||
- `src/pyflowx/executors.py` — `env_context` 快速路径、`storage_key` 内联、`resolved_spec` 去重
|
||||
|
||||
## 关键决策与依据
|
||||
|
||||
### 1. env_context 快速路径(最大开销项)
|
||||
|
||||
**问题**:每个 fn 任务执行时 `with spec.env_context():` 都会创建 `_GeneratorContextManager` 对象 + `__enter__`/`__exit__`,即使任务无 env/cwd。500 noop 任务 = 500 次无谓对象创建。
|
||||
|
||||
**方案**:在 `SyncTaskRunner.run` 和 `_submit_sync_task.fn_call` 内联判断 `spec.env is None and spec.cwd is None`,直接调用 `effective_fn`,跳过上下文管理器。
|
||||
|
||||
**依据**:`_env_and_cwd` 本身已有 `if not env and cwd is None: yield; return` 快速路径,但 contextlib.contextmanager 包装器的对象创建 + 协议开销仍存在。内联判断完全消除这层开销。
|
||||
|
||||
### 2. should_execute 早退快速路径
|
||||
|
||||
**问题**:无 conditions 且无 skip_if_missing 的任务(最常见场景)仍分配空 list `failed_conditions` + 迭代空序列。
|
||||
|
||||
**方案**:在 `should_execute` 开头加 `if not self.conditions and not self.skip_if_missing: return True, None`。
|
||||
|
||||
### 3. resolved_spec 去重
|
||||
|
||||
**问题**:`_filter_and_sort` 已为每个任务计算 `graph.resolved_spec(name)` 并存入 `specs` dict,但三个 Layer runner(Sequential/Threaded/Async)各自又调用一次。500 任务 = 500 次冗余缓存查询。
|
||||
|
||||
**方案**:
|
||||
- `_filter_and_sort` 返回 `(to_run, specs)` 元组,将 specs dict 透传给 runner
|
||||
- `_build_semaphores` 签名从 `graph: Graph` 改为 `specs: Mapping[str, TaskSpec]`
|
||||
- 三个 Layer runner 用 `specs[name]` 替代 `graph.resolved_spec(name)`
|
||||
- `DependencyRunner` 启动时构建 `all_specs` dict 传入 `_build_semaphores`;`_run_task` 内保持 `graph.resolved_spec(name)`(动态任务不在 all_specs 中)
|
||||
|
||||
**依据**:`resolved_spec` 虽有 `cached_property`/`lru_cache`,但每次仍有字典查找 + 函数调用开销。消除冗余调用后每任务省 1 次查找。
|
||||
|
||||
### 4. storage_key 内联快速路径
|
||||
|
||||
**问题**:`_apply_cached` 和 `_store_result` 各调用 `spec.storage_key(ctx)`。无 cache_key 时(最常见场景)该函数仅返回 `self.name`,但有函数调用开销。
|
||||
|
||||
**方案**:内联 `spec.name if spec.cache_key is None else spec.storage_key(ctx)`,避免函数调用。
|
||||
|
||||
**依据**:`storage_key` 内部首行已是 `if self.cache_key is None: return self.name`,内联仅省函数调用开销,但对 500 任务累积可见。
|
||||
|
||||
## 验证结果
|
||||
|
||||
### 门禁
|
||||
|
||||
- ruff: All checks passed
|
||||
- pyrefly: 0 errors
|
||||
- pytest: 1677 passed
|
||||
- coverage: 97.23%(≥95% 门槛)
|
||||
|
||||
### 基准对比(500 noop 任务)
|
||||
|
||||
| 策略 | 优化前 | 优化后 | 提升 |
|
||||
|------|--------|--------|------|
|
||||
| sequential | 764 ops/s | 930 ops/s | +22% |
|
||||
| thread | 108 ops/s | 119 ops/s | +10% |
|
||||
| async | 49 ops/s | 50 ops/s | ~持平(固有开销主导) |
|
||||
| dependency | 49 ops/s | 44 ops/s | ~持平(固有开销主导,噪声范围) |
|
||||
|
||||
sequential 路径提升最显著(+22%),因每任务节省的开销在总耗时中占比最大。thread 次之(+10%)。async/dependency 受事件循环与线程池提交的固有开销主导,per-task 优化相对影响小。
|
||||
|
||||
## 遗留事项
|
||||
|
||||
- async/dependency 策略对 noop 任务的固有并发开销仍较高,需真实 I/O 任务才能体现并行优势(属设计预期,非缺陷)。
|
||||
- `datetime.now()` 每任务调用 2 次(started_at + finished_at),约占 sequential 预算的可观比例,但涉及 duration 正确性,本轮未优化。
|
||||
@@ -0,0 +1,65 @@
|
||||
# 迭代 26:API 简洁性与易用性
|
||||
|
||||
## 本轮目标
|
||||
|
||||
改进 PyFlowX 的 API 简洁性和易用性,减少样板代码,让用户用最少的代码表达 DAG 任务图。
|
||||
|
||||
## 改动文件清单
|
||||
|
||||
| 文件 | 改动 |
|
||||
|------|------|
|
||||
| `src/pyflowx/graph.py` | 新增 `_auto_infer_deps_single` 方法;`add()` 和 `from_specs()` 调用自动推断 |
|
||||
| `src/pyflowx/context.py` | `_is_context_annotation` 重命名为公开 `is_context_annotation`(解决 pyrefly 私有名解析问题) |
|
||||
| `src/pyflowx/__init__.py` | 新增 `px.graph(*specs)` 快捷构造;重写快速上手 docstring |
|
||||
| `src/pyflowx/fileops.py` | DAG 组合示例现代化(`@px.task` + `px.graph()`) |
|
||||
| `src/pyflowx/compose.py` | `compose()` 示例现代化(`px.graph` + `px.cmd`) |
|
||||
| `src/pyflowx/task.py` | `@px.task` 装饰器示例现代化 |
|
||||
| `tests/test_context.py` | 更新 `is_context_annotation` 引用 |
|
||||
| `tests/test_graph.py` | 新增 16 个测试(自动推断 11 个 + px.graph 5 个) |
|
||||
| `.trae/skills/pyflowx-development/SKILL.md` | 第十章新增自动推断与 px.graph 模式 |
|
||||
|
||||
## 关键决策与依据
|
||||
|
||||
### 1. depends_on 自动推断
|
||||
|
||||
**决策**:`depends_on` 为空的纯 fn 任务,从必需参数名自动推断依赖。
|
||||
|
||||
**依据**:用户最常写的模式是 `def double(extract: list[int]) -> ...`,参数名
|
||||
`extract` 就是依赖名,但还要显式写 `depends_on=("extract",)` 是冗余的。Airflow
|
||||
DAGs 也有类似的 `auto_follow` 机制。
|
||||
|
||||
**安全规则**(避免意外行为):
|
||||
- 仅当 `depends_on` 为空时触发——显式声明优先
|
||||
- 仅匹配必需参数(有默认值的不推断)——避免把可选参数误认为依赖
|
||||
- 仅对纯 fn 任务(cmd 任务不推断)——cmd 任务的参数无签名语义
|
||||
- 排除 Context/Mapping 标注参数——这些参数接收完整上下文映射
|
||||
- 排除 `*args`/`**kwargs`——可变参数不映射具体依赖
|
||||
- 排除已在 `soft_depends_on` 中的参数名——避免硬/软依赖重叠报错
|
||||
- 排除与任务自身同名的参数——避免自依赖
|
||||
|
||||
### 2. _is_context_annotation → is_context_annotation
|
||||
|
||||
**决策**:将私有 `_is_context_annotation` 重命名为公开 `is_context_annotation`。
|
||||
|
||||
**依据**:pyrefly 无法解析跨模块导入的私有名(即使 `__all__` 包含),重命名为
|
||||
公开名是最简解决方案。同时该函数在 graph.py 的自动推断逻辑中也被使用,公开化
|
||||
是合理的。
|
||||
|
||||
### 3. px.graph(*specs) 快捷构造
|
||||
|
||||
**决策**:新增 `px.graph(*specs)` 作为 `Graph.from_specs(list)` 的可变参数版本。
|
||||
|
||||
**依据**:`px.graph(a, b, c)` 比 `px.Graph.from_specs([a, b, c])` 更简洁,
|
||||
特别是结合 `@px.task` 装饰器后,链式写法非常自然。
|
||||
|
||||
## 验证结果
|
||||
|
||||
- ruff check:通过
|
||||
- ruff format:通过(131 files formatted)
|
||||
- pyrefly check:0 errors(15 suppressed)
|
||||
- pytest:1694 passed
|
||||
- coverage:97.25%(≥95% 门槛)
|
||||
|
||||
## 遗留事项
|
||||
|
||||
无。三项改进(自动推断 + px.graph + 文档现代化)均已完成并验证。
|
||||
@@ -0,0 +1,84 @@
|
||||
# iter-27:功能扩展与性能优化
|
||||
|
||||
## 本轮目标
|
||||
|
||||
用户请求"继续扩展功能和优化性能",选定 3 个方向:
|
||||
1. **P1:任务优先级调度** —— DependencyRunner 支持按 `priority` 降序调度就绪任务
|
||||
2. **P2:条件分支/switch 模式** —— 新增 `px.switch()` 和 `px.branch()` DAG 构造器
|
||||
3. **P3:大图内存与并发优化** —— 消除 `_find_ready()` 的 O(N) 每轮扫描
|
||||
|
||||
## 改动文件清单
|
||||
|
||||
| 文件 | 变更 |
|
||||
|------|------|
|
||||
| `src/pyflowx/executors.py` | DependencyRunner 重写为事件驱动优先级调度;`_build_dependency_index` 模块级函数;`_on_complete`/`_register_dynamic` 闭包;模块 docstring 更新 |
|
||||
| `src/pyflowx/pipelines.py` | 新增 `switch()` 和 `branch()` 函数 |
|
||||
| `src/pyflowx/__init__.py` | 导出 `switch`/`branch` |
|
||||
| `tests/test_advanced_features.py` | 新增 2 个优先级调度测试 |
|
||||
| `tests/test_pipelines.py` | 新增 `TestSwitch`(5 测试)和 `TestBranch`(4 测试) |
|
||||
| `benchmarks/__main__.py` | 新增 `bench_large_graph()` 大图基准(1k/5k/10k 任务) |
|
||||
| `.trae/skills/pyflowx-development/SKILL.md` | 第三节新增"事件驱动优先级调度"和"增量就绪集优化";第十七节新增 switch/branch |
|
||||
|
||||
## 关键决策与依据
|
||||
|
||||
### P1:事件驱动优先级调度
|
||||
|
||||
**决策**:重写 DependencyRunner 从"一次性创建所有 asyncio Task"为"事件驱动按需创建"。
|
||||
|
||||
**依据**:
|
||||
- 旧实现 `asyncio.gather(*[create_task(n) for n in all])` 一次性创建所有 Task,
|
||||
无法按优先级排序创建顺序(Task 一旦创建即进入事件循环调度队列)。
|
||||
- 新实现用 `completed`/`in_flight`/`remaining` 三集合跟踪生命周期,
|
||||
仅当任务依赖全部完成时才创建 asyncio Task,按 `priority` 降序创建。
|
||||
- 测试验证:`concurrency_key="g"` + `concurrency_limits={"g": 1}` 串行化无依赖
|
||||
任务,使优先级顺序可观测(`order == ["high", "mid", "low"]`)。
|
||||
|
||||
### P2:条件分支构造器
|
||||
|
||||
**决策**:基于已有 `BuiltinConditions` 原语构建 switch/branch,不引入新条件类型。
|
||||
|
||||
**依据**:
|
||||
- `DEP_EQUALS`/`DEP_MATCHES`/`NOT`/`AND` 已覆盖 switch/branch 所需语义。
|
||||
- `switch(selector, cases, *, default=None)`:每个 case 追加
|
||||
`DEP_EQUALS(selector.name, case_key)`;default 追加
|
||||
`AND(NOT(DEP_EQUALS(..., k)) for k in cases)`。
|
||||
- `branch(selector, predicate, if_true, *, if_false=None)`:if_true 追加
|
||||
`DEP_MATCHES`;if_false 追加 `NOT(DEP_MATCHES)`。
|
||||
- 用 `dataclasses.replace` 追加 `depends_on` 和 `conditions`,不修改原 spec。
|
||||
|
||||
### P3:增量就绪集优化
|
||||
|
||||
**决策**:用 `in_degree` 计数器 + `dependents` 反向邻接表替代每轮 O(N) 扫描。
|
||||
|
||||
**依据**:
|
||||
- 旧 `_find_ready()` 每轮扫描全部 `remaining` 任务检查依赖是否满足,
|
||||
O(N·D) per round、O(N²·D) total。10k 任务链式图(10k 轮)成为瓶颈。
|
||||
- 新方案预计算 `in_degree[name]`(尚未完成的硬+软依赖数)+ `dependents[d]`
|
||||
(反向邻接表)。任务完成时 `_on_complete` 仅遍历 `dependents[name]`(O(D_out)),
|
||||
递减 `in_degree`;降为 0 时加入 `ready` 集合。
|
||||
- 动态任务通过 `_register_dynamic` 接入增量结构:spawner 此时未标记 completed
|
||||
(仍在 in_flight),故计入 unsatisfied;spawner 完成后 `_on_complete` 递减并触发就绪。
|
||||
- 提取 `_build_dependency_index` 为模块级函数,控制 `execute` 的 PLR0912 分支数 ≤12。
|
||||
|
||||
## 验证结果
|
||||
|
||||
### 基准(dependency 策略,noop 任务)
|
||||
|
||||
| 图形状 | 1k | 5k | 10k |
|
||||
|--------|-----|-----|------|
|
||||
| chain | 87ms | 459ms | 948ms |
|
||||
| diamond | 53ms | 320ms | 741ms |
|
||||
| wide | 37ms | 247ms | 789ms |
|
||||
|
||||
chain 线性扩展(1k→5k→10k ≈ 5.3x→10.9x),确认 O(N) 调度复杂度。
|
||||
|
||||
### 门禁
|
||||
|
||||
- pytest:1705 passed
|
||||
- coverage:97.21%(branch,≥95% 门槛)
|
||||
- ruff:All checks passed
|
||||
- pyrefly:0 errors
|
||||
|
||||
## 遗留事项
|
||||
|
||||
- 无。P1/P2/P3 全部交付完毕。
|
||||
@@ -0,0 +1,70 @@
|
||||
# iter-28:px.sh 命令执行封装
|
||||
|
||||
## 本轮目标
|
||||
|
||||
ops/ 目录下 57 处 `subprocess.run` 调用存在重复的 `try/except CalledProcessError +
|
||||
FileNotFoundError + print` 模式。封装一个轻量的 `px.sh` 辅助函数,统一错误处理
|
||||
与中文输出,支持 `list[str]`(直接执行)和 `str`(`shell=True`)两种输入。
|
||||
|
||||
## 改动文件清单
|
||||
|
||||
| 文件 | 变更 |
|
||||
|------|------|
|
||||
| `src/pyflowx/shell.py` | 新建模块,实现 `sh()` 函数 |
|
||||
| `src/pyflowx/__init__.py` | 导出 `sh`,更新 docstring |
|
||||
| `tests/test_shell.py` | 新建,16 个测试(list/str/mocked/exported) |
|
||||
| `src/pyflowx/ops/infra/sglang.py` | 重构为使用 `px.sh`,移除 try/except 样板 |
|
||||
| `tests/cli/test_llm.py` | 适配 sglang.py 重构后的错误信息格式 |
|
||||
|
||||
## 关键决策与依据
|
||||
|
||||
### 命名与定位
|
||||
|
||||
- **`px.sh` 而非 `px.run`**:`px.run` 已被占用(执行 Graph DAG),`sh` 简短易记,
|
||||
语义清晰("执行 shell 命令")。
|
||||
- **轻量封装,不替代所有 `subprocess.run`**:需要 `capture_output` + 手动检查
|
||||
返回码等精细控制的场景(如 `lscalc.py`/`piptool.py`/`which.py`)保留原样。
|
||||
`px.sh` 定位为"简单执行 + 报错"场景的快捷方式。
|
||||
|
||||
### 输入类型
|
||||
|
||||
- **`list[str]` → `shell=False`**:直接执行,安全无注入风险。
|
||||
- **`str` → `shell=True`**:支持管道/重定向等 shell 语法;ops 工具场景下命令可信。
|
||||
- **不支持 `func`**:ops 工具本身就是 Python 函数,直接 `func()` 调用即可,
|
||||
封装反而增加复杂度。
|
||||
|
||||
### 返回值设计
|
||||
|
||||
- **成功返回 `CompletedProcess[str]`**:`text=True` 始终启用,输出为字符串。
|
||||
- **失败返回 `None` + 打印中文错误**:不抛异常,调用方用 `result is None` 判断。
|
||||
统一 `CalledProcessError`(打印 returncode)和 `FileNotFoundError`(打印"命令未找到")。
|
||||
- **`capture` 参数**:`False`(默认)输出到终端;`True` 捕获 stdout/stderr。
|
||||
|
||||
### sglang.py 迁移示例
|
||||
|
||||
重构前(12 行 try/except 样板):
|
||||
```python
|
||||
try:
|
||||
subprocess.run(["uv", "install", "sglang[all]"], check=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"安装失败 (returncode={e.returncode}): {e}")
|
||||
except FileNotFoundError:
|
||||
print("uv 命令未找到,请确认 uv 已安装并在 PATH 中")
|
||||
```
|
||||
|
||||
重构后(1 行):
|
||||
```python
|
||||
px.sh(["uv", "install", "sglang[all]"], label="安装 sglang")
|
||||
```
|
||||
|
||||
## 验证结果
|
||||
|
||||
- pytest:1721 passed(新增 16 个 shell 测试)
|
||||
- coverage:97.20%(shell.py 100%、sglang.py 100%)
|
||||
- ruff:All checks passed
|
||||
- pyrefly:0 errors
|
||||
|
||||
## 遗留事项
|
||||
|
||||
- 其余 ~55 处 `subprocess.run` 可在后续迭代中按需迁移到 `px.sh`。
|
||||
优先迁移"简单执行 + 报错"模式的调用;`capture_output` 等精细控制场景保留原样。
|
||||
@@ -0,0 +1,167 @@
|
||||
# P10 图片处理收尾计划(文档同步 + 提交)
|
||||
|
||||
## Context
|
||||
|
||||
P10 图片处理代码已完整实现并通过全部门禁,但**尚未提交**,且过程文档、SKILL 行为同步、project_memory 约定记录三项收尾工作未完成。本计划聚焦于把 P10 工作闭环:编写 iter-22 迭代记录、同步 SKILL.md 行为变更、追加 project_memory 约定、git commit + push。
|
||||
|
||||
用户已确认本次计划范围 = 仅收尾 P10(不含 P11 扩展)。P11 高级功能(OCR/滤镜/批量文件夹/拼接GIF)作为后续阶段输入参考,不在本计划内。
|
||||
|
||||
## 当前状态分析
|
||||
|
||||
### 已完成(未提交)
|
||||
|
||||
```
|
||||
git status --short
|
||||
M src/pyflowx/__init__.py # 导出 image_pipeline
|
||||
M src/pyflowx/cli/pf.py # 注册 imagetool + 别名
|
||||
?? src/pyflowx/imaging.py # image_pipeline DAG 构造器 (130 行)
|
||||
?? src/pyflowx/ops/files/imagetool.py # 11 子命令 (520 行)
|
||||
?? tests/cli/test_imagetool.py # 25 测试 (316 行)
|
||||
?? tests/test_imaging.py # 9 测试 (158 行)
|
||||
?? .trae/documents/ # 本计划文件所在目录
|
||||
```
|
||||
|
||||
### 门禁状态(已验证通过)
|
||||
|
||||
- ruff check / format → 全绿
|
||||
- pyrefly check → 0 错误(68 suppressed)
|
||||
- pytest → 1560 passed
|
||||
- coverage → 97.15%(branch;门槛 95%)
|
||||
- CLI 注册 → `imagetool` 在 `_TOOL_MODULES`,`img`/`image`/`imagetool` 在 `_TOOL_ALIASES`
|
||||
|
||||
### 分支跟踪
|
||||
|
||||
- 当前分支:`develop`
|
||||
- 跟踪远程:`origin/develop`(push 允许,无需询问)
|
||||
|
||||
## 实现步骤
|
||||
|
||||
### 步骤 1:创建 iter-22 迭代记录
|
||||
|
||||
**新建 `.trae/docs/iter-22-image-processing.md`**,参照 [iter-21-p9-feature-extension.md](file:///home/zhou/pyflowx/.trae/docs/iter-21-p9-feature-extension.md) 的结构:
|
||||
|
||||
1. **本轮目标** — P10 三方向:基础操作 7 子命令 / 元数据 4 子命令 / `image_pipeline` DAG 构造器
|
||||
2. **改动文件清单** — 新增 4 文件 + 修改 2 文件(列出每个文件的职责)
|
||||
3. **关键决策与依据**(6 条,基于实际实现踩坑):
|
||||
- **JPG → JPEG 规范化**:`_save_image` 中 `save_fmt == "JPG"` 时改写为 `"JPEG"`,否则 Pillow `img.save(format="JPG")` 抛 `KeyError`
|
||||
- **JPEG 强制 RGB 转换**:JPEG 不支持 alpha 通道,保存前 `img.convert("RGB")` 去 alpha
|
||||
- **EXIF 用 `exif.tobytes()` 而非 `bytes(exif)`**:Pillow 10+ 的 `Image.Exif` 对象 `bytes()` 调用抛 `ValueError: bytes must be in range(0, 256)`,必须用 `tobytes()`
|
||||
- **`quantize(colors=N)` 实际色数可能 < N**:简单图片量化后调色板可能少于请求色数,用 `actual_count = len(palette) // 3` 动态计算并 `min(count, actual_count)` 防越界
|
||||
- **Pillow 10+ Transpose 枚举**:`Image.FLIP_LEFT_RIGHT`(旧)已弃用,用 `Image.Transpose.FLIP_LEFT_RIGHT`(新)
|
||||
- **`_make_step_fn` 闭包工厂**:`image_pipeline` 中若直接在循环内定义闭包,所有任务会捕获最后一次循环的 `input_path`/`output_path`/`params`;提取为独立函数 `_make_step_fn(base_fn, input_path, output_path, params)` 利用每次调用的独立作用域正确捕获
|
||||
4. **验证结果** — 贴上门禁输出(ruff/pyrefly/pytest 1560 passed/coverage 97.15%)
|
||||
5. **遗留事项** — P11 候选方向(OCR/滤镜/批量文件夹/拼接GIF);`imagetool.py` 覆盖率 88%(ImportError 回退分支未覆盖,可接受)
|
||||
6. **归档说明** — iter-22 暂不归档(下次清理窗口为 iter-26 前夕,与 iter-21 一并归档)
|
||||
|
||||
### 步骤 2:同步 SKILL.md 行为变更
|
||||
|
||||
**修改 [.trae/skills/pyflowx-development/SKILL.md](file:///home/zhou/pyflowx/.trae/skills/pyflowx-development/SKILL.md)**:
|
||||
|
||||
1. **更新 frontmatter description** — 末尾追加 "并同步迭代 22(P10)的行为变更",提及图片处理工具
|
||||
2. **更新开头说明**(第 8 行附近)— "并同步迭代 21(P9 功能扩展)与迭代 22(P10 图片处理)的行为变更"
|
||||
3. **新增第十五章 `## 十五、图片处理(P10,iter-22)`**(追加在第十四 章之后),包含 4 个小节:
|
||||
|
||||
#### 可复用模式
|
||||
- **可选依赖 try/except + `_require_*` 守卫**:顶部 `try: from PIL import Image; HAS_PIL = True; except ImportError: HAS_PIL = False`,每个子命令入口 `if not _require_pil(): return` 打印 `pip install pyflowx[office]` 提示
|
||||
- **`@px.tool` 装饰器注册子命令**:`@px.tool("imagetool", subcommand="r", help="调整尺寸")`,subcommand 短别名(r/c/ro/fl/cv/wm/cp/i/e/hi/co)
|
||||
- **`image_pipeline` DAG 构造器**:参照 `compose()` 函数式构造器,每个 step 生成 `TaskSpec`,`depends_on` 链式依赖,`_make_step_fn` 闭包工厂正确捕获每轮参数
|
||||
- **`naming` 模板占位符**:`{stem}`(累积 stem)/`{step}`(操作名)/`{ext}`(当前扩展名,convert 步骤可能改变)
|
||||
|
||||
#### 设计决策
|
||||
- **单一 `imagetool` + office extra**:不拆分为多个工具,所有图片操作集中在 `ops/files/imagetool.py`;Pillow 通过 `pyflowx[office]` extra 安装,零运行时依赖原则不破坏
|
||||
- **`_save_image` 统一保存逻辑**:JPG→JPEG 规范化 + JPEG 强制 RGB + JPEG/WEBP 应用 quality 参数;避免每个子命令重复处理
|
||||
- **`image_pipeline` 委托 imagetool 函数**:不重新实现图片操作,`operations` 字典映射操作名到 `imagetool.image_*` 函数,保持单一职责
|
||||
|
||||
#### 踩坑总结
|
||||
- **Pillow 10+ API 迁移**:`Image.FLIP_LEFT_RIGHT` 已弃用 → `Image.Transpose.FLIP_LEFT_RIGHT`;`bytes(exif)` 抛错 → `exif.tobytes()`
|
||||
- **`quantize(colors=N)` 实际色数可能 < N**:简单图片量化后调色板少于请求色数,必须用 `actual_count = len(palette) // 3` 动态计算
|
||||
- **闭包捕获循环变量陷阱**:`image_pipeline` 中直接在循环内定义闭包会捕获最后一次循环的变量;提取 `_make_step_fn(base_fn, input_path, output_path, params)` 为独立函数利用每次调用的独立作用域
|
||||
- **PLR0912 分支数限制**:`image_exif` 函数最初 13 分支超过 12 限制,提取 `_print_exif`/`_apply_exif_modifications`/`_apply_single_exif_set`/`_save_exif` 4 个辅助函数降至合规
|
||||
|
||||
### 步骤 3:更新 project_memory.md
|
||||
|
||||
**修改 [project_memory.md](file:///home/zhou/.trae-cn/memory/projects/-home-zhou-pyflowx/project_memory.md)**:
|
||||
|
||||
1. **Engineering Conventions 末尾追加** P10 约定(约 8 条):
|
||||
- `imagetool` 模式:try/except PIL 导入 + `HAS_PIL` 标志 + `_require_pil()` 守卫 + `@px.tool` 装饰器
|
||||
- `_save_image` 统一保存:JPG→JPEG 规范化 + JPEG 强制 RGB + JPEG/WEBP 应用 quality
|
||||
- EXIF 序列化用 `exif.tobytes()`(Pillow 10+),不用 `bytes(exif)`
|
||||
- `quantize(colors=N)` 实际色数可能 < N,用 `actual_count = len(palette) // 3` 动态计算
|
||||
- Pillow 10+ 用 `Image.Transpose.FLIP_LEFT_RIGHT`(非弃用的 `Image.FLIP_LEFT_RIGHT`)
|
||||
- `image_pipeline` 用 `_make_step_fn(base_fn, input_path, output_path, params)` 闭包工厂避免循环变量捕获陷阱
|
||||
- `image_pipeline` 委托 `imagetool.image_*` 函数,不重新实现操作(单一职责)
|
||||
- `pytest.importorskip("PIL")` 跳过未安装 Pillow 的测试模块
|
||||
|
||||
2. **Lessons Learned 末尾追加** iter-22 总结条目:
|
||||
- `iter-22 (P10 image processing) completed: imagetool 11 subcommands (resize/crop/rotate/flip/convert/watermark/compress/info/exif/histogram/colors) + image_pipeline DAG constructor; 1560 tests green, coverage 97.15%; SKILL.md synced (added section 15); next archive window at iter-26`
|
||||
|
||||
### 步骤 4:Git commit + push
|
||||
|
||||
按文件名逐个 `git add`(不用 `git add .`/`-A`),然后 commit + push。
|
||||
|
||||
**暂存文件**(8 个):
|
||||
```
|
||||
git add src/pyflowx/ops/files/imagetool.py
|
||||
git add src/pyflowx/imaging.py
|
||||
git add src/pyflowx/__init__.py
|
||||
git add src/pyflowx/cli/pf.py
|
||||
git add tests/cli/test_imagetool.py
|
||||
git add tests/test_imaging.py
|
||||
git add .trae/docs/iter-22-image-processing.md
|
||||
git add .trae/skills/pyflowx-development/SKILL.md
|
||||
```
|
||||
|
||||
**Commit message**(遵循 [.trae/rules/git-commit-message.md](file:///home/zhou/pyflowx/.trae/rules/git-commit-message.md) 中文风格,参照 `5ac7fd3` P9 提交):
|
||||
|
||||
```
|
||||
feat: P10 图片处理 — imagetool 11 子命令(resize/crop/rotate/flip/convert/watermark/compress/info/exif/histogram/colors) + image_pipeline DAG 构造器;1560 测试全绿,覆盖率 97.15%
|
||||
```
|
||||
|
||||
**Push**:`git push`(develop 跟踪 origin/develop,直接 push)
|
||||
|
||||
## 关键文件
|
||||
|
||||
| 操作 | 文件 |
|
||||
|------|------|
|
||||
| 新建 | `.trae/docs/iter-22-image-processing.md`(iter-22 迭代记录) |
|
||||
| 修改 | `.trae/skills/pyflowx-development/SKILL.md`(新增第十五章 + frontmatter 更新) |
|
||||
| 修改 | `/home/zhou/.trae-cn/memory/projects/-home-zhou-pyflowx/project_memory.md`(追加 P10 约定) |
|
||||
| 提交 | 上述 3 文件 + 已实现的 5 文件(imagetool.py/imaging.py/__init__.py/pf.py/2 测试文件) |
|
||||
|
||||
## 复用的现有模式
|
||||
|
||||
- **iter-21 迭代记录结构**:[iter-21-p9-feature-extension.md](file:///home/zhou/pyflowx/.trae/docs/iter-21-p9-feature-extension.md)(本轮目标 / 改动文件清单 / 关键决策与依据 / 验证结果 / 遗留事项 / 归档说明)
|
||||
- **SKILL.md 章节结构**:参照第十四章 `## 十四、任务编排与持久化(P9,iter-21)` 的 `### 可复用模式` / `### 设计决策` / `### 踩坑总结` 三段式
|
||||
- **commit message 风格**:参照 `5ac7fd3` P9 提交(`feat: P9 功能扩展 — ...;1525 测试全绿,覆盖率 97.58%`)
|
||||
- **project_memory 条目风格**:参照第 125 行 iter-21 总结条目
|
||||
|
||||
## 验证步骤
|
||||
|
||||
1. **文档完整性**:
|
||||
- `iter-22-image-processing.md` 包含全部 6 个小节(目标/文件/决策/验证/遗留/归档)
|
||||
- `SKILL.md` 第十五章包含可复用模式/设计决策/踩坑总结三小节
|
||||
- `project_memory.md` Engineering Conventions 追加 8 条、Lessons Learned 追加 1 条
|
||||
|
||||
2. **Git 状态**:
|
||||
- `git status` 干净(无未提交文件)
|
||||
- `git log --oneline -1` 显示新 commit
|
||||
- `git push` 成功(origin/develop 已更新)
|
||||
|
||||
3. **门禁复跑**(可选,确保文档修改未误改代码):
|
||||
```bash
|
||||
uvx --from pyflowx pymake tc # ruff + pyrefly + pytest
|
||||
```
|
||||
|
||||
## 验收标准
|
||||
|
||||
- `.trae/docs/iter-22-image-processing.md` 创建完成,结构对齐 iter-21
|
||||
- `.trae/skills/pyflowx-development/SKILL.md` 新增第十五章,frontmatter description 更新
|
||||
- `project_memory.md` 追加 P10 约定(Engineering Conventions + Lessons Learned)
|
||||
- Git commit 成功,push 到 origin/develop 成功
|
||||
- `git status` 干净
|
||||
|
||||
## 不在范围内
|
||||
|
||||
- P11 高级功能(OCR/滤镜/批量文件夹/拼接GIF)— 用户已确认为后续阶段输入,本次不实现
|
||||
- 代码修改 — P10 代码已实现完成,本次仅文档同步 + 提交
|
||||
- 归档 — iter-22 暂不归档(下次清理窗口 iter-26 前夕,与 iter-21 一并归档到 SKILL.md)
|
||||
@@ -0,0 +1,174 @@
|
||||
# P10 图片处理开发计划
|
||||
|
||||
## Context
|
||||
|
||||
PyFlowX 现有 `ops/files/` 工具集已覆盖 PDF(pdftool)、截图(screenshot)、文件日期(filedate)等场景,但缺少图片处理能力。`pyproject.toml` 的 `office` extra 已声明 `pillow>=10.4.0`,却无对应工具消费它。
|
||||
|
||||
本计划新增 `imagetool` 工具,提供基础操作(resize/crop/rotate/flip/convert/watermark/compress)、元数据与信息(info/exif/histogram/colors)、批量 DAG 编排(`image_pipeline` 便捷构造器)三类能力。用户已确认范围:基础操作 + 元数据 + 批量 DAG(不含 OCR),工具形态为单一 `imagetool` + office extra。
|
||||
|
||||
## 实现步骤
|
||||
|
||||
### P10.1 基础操作子命令
|
||||
|
||||
**新建 `src/pyflowx/ops/files/imagetool.py`**,参照 [pdftool.py](file:///home/zhou/pyflowx/src/pyflowx/ops/files/pdftool.py) 模式:
|
||||
|
||||
- 顶部 `try: from PIL import Image; HAS_PIL = True; except ImportError: HAS_PIL = False`
|
||||
- `_require_pil() -> bool` 检查并打印 `pip install pyflowx[office]` 提示
|
||||
- 7 个 `@px.tool("imagetool", subcommand=..., help=...)` 子命令:
|
||||
- `resize` (`r`) — `image_resize(input: Path, output: Path, width: int, height: int | None = None, keep_ratio: bool = True)`
|
||||
- `crop` (`c`) — `image_crop(input: Path, output: Path, left: int, top: int, right: int, bottom: int)`
|
||||
- `rotate` (`ro`) — `image_rotate(input: Path, output: Path, degrees: float, expand: bool = False)`
|
||||
- `flip` (`fl`) — `image_flip(input: Path, output: Path, direction: str = "horizontal")`
|
||||
- `convert` (`cv`) — `image_convert(input: Path, output: Path, format: str | None = None, quality: int = 85)`
|
||||
- `watermark` (`wm`) — `image_watermark(input: Path, output: Path, text: str, position: str = "bottom-right", opacity: float = 0.5, font_size: int = 32)`
|
||||
- `compress` (`cp`) — `image_compress(input: Path, output: Path, quality: int = 85)`
|
||||
|
||||
实现要点:
|
||||
- 用 `Image.open(input)` 打开,操作后 `img.save(output, format=..., quality=...)` 保存
|
||||
- `resize` + `keep_ratio=True` 时用 `Image.thumbnail((width, height))` 保比缩放;`keep_ratio=False` 用 `Image.resize((width, height))`
|
||||
- `watermark` 用 `ImageDraw.Draw(img).text(...)` + `ImageFont.truetype()`(字体缺失回退 `load_default()`)
|
||||
- `convert` 根据 `output.suffix` 推断格式;JPEG 系列需先 `img.convert("RGB")` 去 alpha 通道
|
||||
- `__all__` 列出所有 `image_*` 函数名
|
||||
|
||||
### P10.2 元数据与信息子命令
|
||||
|
||||
继续在 `imagetool.py` 追加 4 个子命令:
|
||||
|
||||
- `info` (`i`) — `image_info(input: Path, json: bool = False)`:打印尺寸/格式/模式/色彩空间/EXIF 摘要。`json=True` 输出 JSON 格式
|
||||
- `exif` (`e`) — `image_exif(input: Path, output: Path | None = None, show: bool = True, set: list[str] | None = None, clear: bool = False)`:读写 EXIF。`--set KEY=VAL` 修改,`--clear` 清空,`output` 另存
|
||||
- `histogram` (`hi`) — `image_histogram(input: Path, channel: str = "rgb")`:打印 RGB/Luminance 直方图统计(每通道 8 桶,纯文本表格输出)
|
||||
- `colors` (`co`) — `image_colors(input: Path, count: int = 5)`:主色调提取,用 `Image.quantize(colors=count).getpalette()` 提取并打印十六进制色值
|
||||
|
||||
实现要点:
|
||||
- `info` 用 `img.size` / `img.format` / `img.mode` / `img.info.get("exif")`
|
||||
- `exif` 用 `img.getexif()` / `img.save(exif=bytes(exif))`
|
||||
- `histogram` 用 `img.histogram()` 分桶统计
|
||||
- `colors` 用 `img.quantize(colors=count).getpalette()` 取前 N 色
|
||||
|
||||
### P10.3 批量 DAG 编排
|
||||
|
||||
**新建 `src/pyflowx/imaging.py`**(类似 [compose.py](file:///home/zhou/pyflowx/src/pyflowx/compose.py) 的独立模块),提供链式 DAG 构造器:
|
||||
|
||||
```python
|
||||
def image_pipeline(
|
||||
source: str | Path,
|
||||
steps: list[tuple[str, dict[str, Any]]],
|
||||
*,
|
||||
output_dir: str | Path | None = None,
|
||||
naming: str = "{stem}_{step}{ext}",
|
||||
) -> Graph:
|
||||
"""图片处理流水线 DAG 构造器。
|
||||
|
||||
每个 step 是 (操作名, 参数字典),操作名对应 imagetool 子命令
|
||||
(resize/crop/rotate/flip/convert/watermark/compress)。
|
||||
前一步输出作为后一步输入,形成链式 DAG。
|
||||
"""
|
||||
```
|
||||
|
||||
示例:
|
||||
```python
|
||||
graph = px.image_pipeline(
|
||||
"input.jpg",
|
||||
steps=[
|
||||
("resize", {"width": 800}),
|
||||
("watermark", {"text": "© 2026"}),
|
||||
("convert", {"format": "webp", "quality": 85}),
|
||||
],
|
||||
)
|
||||
report = px.run(graph, strategy="sequential")
|
||||
```
|
||||
|
||||
实现要点:
|
||||
- 每个 step 生成一个 `TaskSpec`,`fn` 调用对应的 `image_*` 函数
|
||||
- 任务名按 `naming` 模板生成(默认 `{stem}_{step}{ext}`,如 `input_resize.jpg`)
|
||||
- 任务间用 `depends_on` 链式依赖,上游输出路径 = 下游输入路径
|
||||
- `output_dir` 默认为源文件所在目录
|
||||
- 在 `__init__.py` 导出 `image_pipeline`
|
||||
|
||||
### P10.4 验证与文档
|
||||
|
||||
**测试 `tests/cli/test_imagetool.py`**:
|
||||
- 用 `pytest.importorskip("PIL")` 跳过未安装 Pillow 的环境
|
||||
- 用 `tmp_path` 创建真实测试图片(`Image.new("RGB", (100, 100), "red").save(tmp_path / "test.png")`)
|
||||
- 每个子命令至少 1 个测试(基础操作验证输出文件存在 + 尺寸/格式正确)
|
||||
- 元数据测试验证输出内容(info 的 JSON 结构、exif 读写往返、histogram 桶数、colors 色值数)
|
||||
- `image_pipeline` 测试验证 DAG 拓扑(3 步链式,每步输出存在且最终输出为 webp)
|
||||
- 测试 mock 优先级遵循 `python-standards.md`:优先用真实 Pillow 操作(非 subprocess),仅字体加载等不可控部分用 `monkeypatch`
|
||||
|
||||
**新增 `tests/imaging/test_image_pipeline.py`**(或在 `tests/test_imaging.py`):
|
||||
- 测试 `image_pipeline` 生成的 Graph 拓扑正确(任务数、依赖链)
|
||||
- 测试实际执行后输出文件链完整
|
||||
|
||||
**注册到 CLI**:
|
||||
- 编辑 [src/pyflowx/cli/pf.py](file:///home/zhou/pyflowx/src/pyflowx/cli/pf.py) 的 `_TOOL_MODULES`:添加 `"imagetool": "pyflowx.ops.files.imagetool"`
|
||||
- 编辑 `_TOOL_ALIASES`:添加 `"imagetool": "imagetool"`、`"image": "imagetool"`、`"img": "imagetool"`
|
||||
- 编辑 [src/pyflowx/__init__.py](file:///home/zhou/pyflowx/src/pyflowx/__init__.py) 导出 `image_pipeline`
|
||||
|
||||
**文档同步**:
|
||||
- `.trae/docs/iter-22-image-processing.md` 迭代记录
|
||||
- `.trae/skills/pyflowx-development/SKILL.md` 同步行为变更(新增"十五、图片处理工具"小节,或在十二章 CLI 工具下追加)
|
||||
- `project_memory.md` 追加 P10 工程约定
|
||||
|
||||
## 关键文件
|
||||
|
||||
| 操作 | 文件 |
|
||||
|------|------|
|
||||
| 新建 | `src/pyflowx/ops/files/imagetool.py`(11 子命令) |
|
||||
| 新建 | `src/pyflowx/imaging.py`(`image_pipeline` 构造器) |
|
||||
| 新建 | `tests/cli/test_imagetool.py`(子命令测试) |
|
||||
| 新建 | `tests/test_imaging.py`(DAG 构造器测试) |
|
||||
| 修改 | `src/pyflowx/cli/pf.py`(注册 imagetool + 别名) |
|
||||
| 修改 | `src/pyflowx/__init__.py`(导出 `image_pipeline`) |
|
||||
| 新建 | `.trae/docs/iter-22-image-processing.md` |
|
||||
| 修改 | `.trae/skills/pyflowx-development/SKILL.md`(行为变更同步) |
|
||||
|
||||
## 复用的现有模式
|
||||
|
||||
- **`@px.tool` 装饰器 + ToolSpec**:[tools.py L116-L180](file:///home/zhou/pyflowx/src/pyflowx/tools.py#L116-L180),每个子命令一个装饰器调用
|
||||
- **可选依赖 try/except + `_require_*`**:[pdftool.py L31-L59](file:///home/zhou/pyflowx/src/pyflowx/ops/files/pdftool.py#L31-L59)
|
||||
- **`_TOOL_MODULES` 注册**:[pf.py L112-L137](file:///home/zhou/pyflowx/src/pyflowx/cli/pf.py#L112-L137)
|
||||
- **`pytest.importorskip` 跳过缺依赖**:[test_pdftool.py L22](file:///home/zhou/pyflowx/tests/cli/test_pdftool.py#L22)
|
||||
- **`image_pipeline` 委托模式**:参照 [compose.py](file:///home/zhou/pyflowx/src/pyflowx/compose.py) 的 `compose()` 函数式构造器
|
||||
- **TaskSpec 链式依赖**:用 `Graph.from_specs` + `depends_on` 构建链式 DAG(P9.2 `pipeline()` 同模式)
|
||||
|
||||
## 验证方法
|
||||
|
||||
```bash
|
||||
# 1. 单元测试
|
||||
uv run pytest tests/cli/test_imagetool.py tests/test_imaging.py -v
|
||||
|
||||
# 2. 全套门禁
|
||||
uv run ruff check .
|
||||
uv run ruff format --check .
|
||||
uv run pyrefly check .
|
||||
uv run pytest --cov=pyflowx --cov-branch
|
||||
|
||||
# 3. CLI 实测(需 pip install pyflowx[office])
|
||||
pf imagetool info test.png
|
||||
pf imagetool resize test.png out.png --width 50
|
||||
pf imagetool watermark test.png wm.png --text "TEST"
|
||||
pf imagetool convert test.png out.webp --quality 80
|
||||
|
||||
# 4. DAG 编排实测
|
||||
python -c "
|
||||
import pyflowx as px
|
||||
g = px.image_pipeline('test.png', steps=[
|
||||
('resize', {'width': 50}),
|
||||
('watermark', {'text': 'X'}),
|
||||
('convert', {'format': 'webp', 'quality': 80}),
|
||||
])
|
||||
r = px.run(g, strategy='sequential')
|
||||
print(r.success)
|
||||
"
|
||||
|
||||
# 5. 覆盖率门槛 ≥ 95%(branch)
|
||||
```
|
||||
|
||||
## 验收标准
|
||||
|
||||
- ruff/pyrefly 0 错误,pytest 全绿,覆盖率 ≥ 95%(branch)
|
||||
- `pf imagetool --help` 列出全部 11 个子命令
|
||||
- `pf imagetool <sub> --help` 显示参数说明
|
||||
- `image_pipeline()` 生成的 DAG 可执行,链式输出完整
|
||||
- 未安装 office extra 时 `pf imagetool <sub>` 打印友好提示而非崩溃
|
||||
- `tests/cli/test_tool_modules.py` 自动覆盖新工具注册(无需额外修改)
|
||||
@@ -0,0 +1,23 @@
|
||||
# 开发流程约束
|
||||
|
||||
本规则补充 `self-driven.md`,定义执行过程记录与规则变更的两条硬约束。
|
||||
|
||||
## 执行过程记录
|
||||
|
||||
- **每次迭代**(一个完整的"计划 → 实现 → 测试 → 文档 → 验证"闭环)必须记录到
|
||||
`.trae/docs/` 目录,文件名形如 `iter-NN-<主题>.md`,`NN` 为零填充序号。
|
||||
- 记录内容至少包括:本轮目标、改动文件清单、关键决策与依据、验证结果、遗留事项。
|
||||
- **每 5 次迭代后**清理 `.trae/docs/`:
|
||||
- 把需要长期保留的整合性内容(可复用模式、踩坑总结、设计决策)
|
||||
归档到 `.trae/skills/` 对应技能文档中;
|
||||
- 其余过程性记录删除;
|
||||
- docs 目录只保留最近未归档的迭代记录(即第 6 次迭代开始前,清理 1–5 号记录)。
|
||||
- 归档动作需在迭代记录中标注"已归档至 skills/<文件>"。
|
||||
|
||||
## 规则变更约束
|
||||
|
||||
- 任何对 `.trae/rules/` 下文件的修改(新增、编辑、删除)**必须先询问用户**,
|
||||
获得明确授权后方可变动;**未授权前不得擅自修改 rules**。
|
||||
- 用户在对话中显式要求写入的规则内容(如"把 X 写入 rules")视为已授权,
|
||||
可直接写入,写入后在回复中说明改了哪个文件、加了什么。
|
||||
- 规则变更后需同步更新 `project_memory.md` 中的"开发约定"小节。
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
alwaysApply: true
|
||||
scene: git_message
|
||||
---
|
||||
|
||||
在此处编写规则,自定义 AI 生成提交信息的风格。
|
||||
|
||||
## 提交信息格式
|
||||
- 提交信息必须使用中文。
|
||||
- 提交信息必须包含变更的类型(例如 "fix"、"feat"、"refactor" 等)。
|
||||
- 提交信息必须尽简洁明了,不要超过一段落。
|
||||
@@ -0,0 +1,157 @@
|
||||
# Python 开发规范
|
||||
|
||||
本规范结合 Python 最佳实践,作为编写与审查 Python 代码的统一标准。
|
||||
详细操作指南见 `.agents/skills/` 下相应技能。
|
||||
|
||||
## 工具链(以 pyproject.toml 为准)
|
||||
|
||||
| 工具 | 用途 | 配置要点 |
|
||||
|------|------|---------|
|
||||
| **ruff** | lint + format | `line-length=120`,`target-version="py38"` |
|
||||
| **pyrefly** | 类型检查 | `preset="strict"`,`python-version="3.8"` |
|
||||
| **pytest** | 测试 | `asyncio_default_fixture_loop_scope="function"`,marker `slow` |
|
||||
| **coverage** | 覆盖率 | `branch=true`,`fail_under=95`,`concurrency=["thread"]` |
|
||||
| **pre-commit** | 提交前检查 | ruff `--fix` + trailing-whitespace + end-of-file-fixer |
|
||||
|
||||
验证(每次修改后必做):
|
||||
|
||||
```bash
|
||||
uvx --from pyflowx pymake tc
|
||||
uvx --from pyflowx pymake cov
|
||||
```
|
||||
|
||||
## 兼容性
|
||||
|
||||
- **最低 Python 3.8**:用 `from __future__ import annotations` 延迟注解求值;
|
||||
按版本用 `typing.List`(3.8) → 内置泛型(3.9) → `X | Y`(3.10) → `typing.override`(3.12)。
|
||||
- **版本守卫**:`if sys.version_info >= (3, X):` 引入高版本 API;低版本回退分支加 `# pragma: no cover`。
|
||||
- **零运行时依赖**:仅依赖标准库(需 `typing-extensions` 用于 `override`/`TypeVar` 前向兼容)。
|
||||
新增依赖须审慎,优先用标准库。
|
||||
|
||||
## 类型注解
|
||||
|
||||
- **公共 API 必须有完整类型注解**,包括返回类型;私有函数也应有注解。
|
||||
- 泛型用 `TypeVar`;PEP 696 `default=` 仅 3.13+ 标准库支持,3.8–3.12 用 `typing_extensions.TypeVar`。
|
||||
- `Mapping`/`Sequence` 用于只读参数,`dict`/`list` 用于可变返回。
|
||||
- `Any` 仅用于真正动态场景(如 `Context` 跨任务异构映射);任务内部类型必须完全静态。
|
||||
- 禁用裸 `# type: ignore`;确需时加具体规则码(如 `# type: ignore[union-attr]`)。
|
||||
- **`TYPE_CHECKING` 守卫**:仅类型检查需要的导入放 `if TYPE_CHECKING:` 块内,避免循环依赖。
|
||||
- **类型收窄**:用 `assert isinstance(x, Y)` 辅助 pyrefly 推断;`cast()` 仅用于类型系统无法表达的场景。
|
||||
|
||||
## 数据结构
|
||||
|
||||
- **不可变优先**:配置/描述类用 `@dataclass(frozen=True)`;可变类属性标注 `RUF012` 豁免。
|
||||
- **缓存**:实例级用 `functools.cached_property`,按参数键控用 `functools.lru_cache`;
|
||||
不可哈希参数需 try/except 回退。修改被缓存数据源后必须手动清空缓存。
|
||||
- **抽象基类**:接口用 `abc.ABC` + `@abstractmethod`(如 `StateBackend`)。
|
||||
- **枚举**:状态/标志值用 `enum.Enum`(如 `TaskStatus`),禁止裸字符串/魔术数字;枚举值用 `UPPER_SNAKE`。
|
||||
- **`__repr__`**:可变类实现 `__repr__`(含关键字段);`frozen=True` dataclass 自动生成。
|
||||
|
||||
## 模块与导入
|
||||
|
||||
- **单一职责**:每模块只做一件事(`task.py` 数据结构、`executors.py` 执行、`command.py` 命令、`compose.py` 组合)。禁止跨职责边界。
|
||||
- **导入顺序**(ruff isort):`__future__` → 标准库 → 第三方 → 本地,各组间空行。
|
||||
- **惰性导入**:仅为打破循环依赖时使用,函数体内导入并注释说明;顶层导入是默认。
|
||||
- **`__all__`**:定义 `__all__` 显式声明导出符号,位置仅次于 `__future__` 之后。
|
||||
- **禁用 star imports**:`from x import *` 污染命名空间、破坏类型检查(`__init__.py` 聚合经 `__all__` 控制为例外)。
|
||||
- **避免 `utils.py`/`helpers.py`**:按职责归入对应模块。
|
||||
|
||||
## 函数设计
|
||||
|
||||
- **模块级函数优于 Mixin**:共享逻辑用模块级函数,类只持有状态与薄方法。
|
||||
- **静态方法慎用**:纯函数直接放模块级。
|
||||
- **参数 ≤ 5 个**为宜;超出用 dataclass 封装参数对象。
|
||||
- **单一职责**:一个函数做一件事;过长函数考虑拆分。
|
||||
- **异常范围要窄**:只捕获预期异常(如 `(TypeError, ValueError, KeyError, AttributeError)`),
|
||||
**禁止** `except Exception` 掩盖 bug;捕获后至少 `logger.warning` 记录。
|
||||
- **可变默认参数**:`def f(x=[])` 是经典坑;用 `None` 哨兵或 `field(default_factory=list)`。
|
||||
|
||||
## 异常处理
|
||||
|
||||
- **自定义异常家族**:继承公共基类(如 `PyFlowXError`),按错误场景分类。
|
||||
- **异常包装**:`raise NewError(...) from exc` 保留因果链。
|
||||
- **不要吞异常**:捕获后必须处理(记录/包装/重抛),禁止空 `except: pass`。
|
||||
- **钩子/回调异常**:第三方回调异常仅记录,不影响主流程。
|
||||
|
||||
## 并发与线程安全
|
||||
|
||||
- **进程全局状态**(`os.environ`/`os.chdir`)在并发场景下必须用全局锁(`threading.RLock`)序列化。
|
||||
- **条件评估不可有可变状态**:组合条件(NOT/AND/OR)不得修改共享 `_reason`,避免竞态。
|
||||
- **批量 I/O**:循环内多次写盘改为批量一次(`contextmanager` 包裹延迟落盘)。
|
||||
- **信号量限流**:`concurrency_key` + `Semaphore` 按组限流。
|
||||
|
||||
## 测试
|
||||
|
||||
详细操作指南见 `.agents/skills/pyflowx-testing` 技能。硬约束:
|
||||
|
||||
- **覆盖率 ≥ 95%**(branch coverage),不得下降。
|
||||
- **公共 API 优先测试**:用公共接口(`has`/`get`),不访问私有方法;
|
||||
故障注入等场景可临时访问私有属性,docstring 注明原因。
|
||||
- **命名**:`test_<被测对象>_<场景>`。
|
||||
- **断言**:原生 `assert x == 1`,禁用 `self.assertEqual`;`pytest.raises` 必填 `match=`。
|
||||
- **Mock 优先级**:`monkeypatch` > 内联 stub > `unittest.mock` > `pytest-mock`。
|
||||
禁用 `@patch` 装饰器、`mock.patch.object` 上下文、`pytest-mock` 的 `mocker` fixture。
|
||||
- **fixture**:`tmp_path`/`monkeypatch`/`capsys` 优先;autouse 仅全局必需时用。
|
||||
- **slow 标记**:耗时测试加 `@pytest.mark.slow`,CI 可 `-m "not slow"` 跳过。
|
||||
- **测试代码也跑 ruff**:`tests/**` 忽略 `ARG001`/`ARG002`。
|
||||
|
||||
## 代码风格
|
||||
|
||||
- **行宽 120**(ruff formatter 处理)。
|
||||
- **docstring**:公共 API 必须有;中文叙述 + 中文注释是本项目既有风格。
|
||||
- **打印和日志**:使用中文打印和日志,避免使用英文。
|
||||
- **命名**:`snake_case` 函数/变量,`PascalCase` 类,`UPPER_SNAKE` 常量,`_leading_underscore` 私有。
|
||||
- **字符串引号**:ruff 默认双引号。
|
||||
- **末尾单 `\n`**、**无尾随空格**(pre-commit 强制)。
|
||||
- **不用 emoji**:除非用户明确要求。
|
||||
|
||||
## Pythonic 风格
|
||||
|
||||
- **`is` 比较 `None`/`True`/`False`**:单例用 `is`,值用 `==`(PEP 8 E711/E712)。
|
||||
- **EAFP 优于 LBYL**:先尝试再处理异常,而非先检查再执行(避免竞态窗口)。
|
||||
- **truthiness**:`if items:` 优于 `if len(items) > 0:`。
|
||||
- **字符串格式化**:首选 f-string;`%` 仅用于 `logging` 延迟格式化。
|
||||
- **推导式**优于 `map`+`filter`;> 2 层拆为显式循环。
|
||||
- **`enumerate`** 替代 `range(len())`;**`zip`** 并行迭代(3.10+ 用 `strict=True`)。
|
||||
- **解包** `a, b = pair` 优于索引访问;忽略值用 `_`。
|
||||
- **海象运算符 `:=`**(3.8+):赋值+判断合一,但不滥用。
|
||||
|
||||
## 日志
|
||||
|
||||
- **`logging.getLogger(__name__)`**:每模块独立 logger,禁用 `print` 调试残留。
|
||||
- **结构化上下文**:`extra={...}` 传字段;`logger.warning("task %r failed: %s", name, exc)` 优于 f-string(延迟格式化)。
|
||||
- **日志级别**:`DEBUG` 诊断 / `INFO` 关键流程 / `WARNING` 可恢复异常 / `ERROR` 需人工介入。
|
||||
- **禁止日志密码/密钥**:脱敏后再记录。
|
||||
|
||||
## 路径与资源
|
||||
|
||||
- **优先 `pathlib.Path`**:`Path("a") / "b"` 而非 `os.path.join`(ruff `PTH` 强制);
|
||||
禁止字符串拼接路径。类型注解用 `Path`,边界 `str` 立即包装。
|
||||
- **`with` 语句**:文件、锁、连接、临时目录一律用 `with` 或 `contextlib.contextmanager`;
|
||||
多资源用 `contextlib.ExitStack`。
|
||||
- **显式关闭**:长生命周期对象(连接池、线程池)实现 `close()`,但优先 `with`。
|
||||
- **批量操作**:循环内多次 acquire/release 改为批量一次。
|
||||
|
||||
## 安全
|
||||
|
||||
- **禁用 `eval`/`exec`**:处理不可信输入时绝不使用;用 `ast.literal_eval` 或专用解析器。
|
||||
- **`subprocess`**:禁用 `shell=True` 除非命令完全可信;优先 `list[str]` 形式。
|
||||
- **凭证不入仓**:密钥/token/密码放 `.env` 或环境变量,`.gitignore` 必须包含 `.env`。
|
||||
- **日志脱敏**:记录请求/响应时移除 `Authorization`、`password` 等字段。
|
||||
- **依赖审计**:`uv lock` 后审阅新增依赖,避免引入已知 CVE 的包。
|
||||
|
||||
## 性能要点
|
||||
|
||||
- **避免重复计算**:循环内查询应缓存或预构建映射(如 `{name: spec}`)。
|
||||
- **避免双重查找**:`has(k)` + `get(k)` 改为单次 `get(k)` + `KeyError` 回退。
|
||||
- **统一校验**:入口校验一次,下游路径不重复(如 `run()` 统一 `validate()`,`layers()` 不再重复)。
|
||||
- **事件 emit**:任务生命周期必须 emit `RUNNING` → `SUCCESS`/`FAILED`/`SKIPPED`,
|
||||
不要留死分支(`# pragma: no cover` 是清理信号,应激活或删除)。
|
||||
|
||||
## Git 与提交
|
||||
|
||||
- **自动提交**:任务完成后自动 `git add`(按文件名)+ `git commit` + `git push`(仅当分支已跟踪远程时执行 push;新分支跳过 push 并在总结中说明)。
|
||||
- **不修改 git config**。
|
||||
- **不运行破坏性命令**(`push --force`/`reset --hard`/`clean -f`)除非用户明确要求。
|
||||
- **staging**:按文件名添加,不用 `git add -A`/`git add .`,避免误加敏感文件。
|
||||
- **commit message**:简洁,聚焦"为什么"而非"是什么";遵循仓库既有风格。
|
||||
@@ -0,0 +1,151 @@
|
||||
---
|
||||
alwaysApply: true
|
||||
---
|
||||
|
||||
# 自驱动开发规则
|
||||
|
||||
本规则定义一种"目标驱动、闭环执行"的工作模式:仅在任务开始时与用户确认一次目标与边界,后续由 Agent 自主完成"计划 → 编码 → 测试 → 文档 → 验证"的迭代循环,直到用户目标达成。
|
||||
|
||||
## 核心原则
|
||||
|
||||
- **目标导向**:始终以用户最终目标为准绳,所有阶段产出都应服务于该目标。
|
||||
- **闭环执行**:每个子任务必须走完"计划 → 实现 → 测试 → 文档 → 验证"五步;禁止跳步留半成品。
|
||||
- **自主决策**:初始确认之后,实现路径、API 形态、重构范围、文件命名、测试组织、错误修复策略等由 Agent 自行决断,不再逐项请示。**可逆操作(编辑文件、运行测试、修复 lint、调整实现)直接执行,不询问**;只有不可逆/高风险操作才暂停。
|
||||
- **透明沟通**:每个阶段开始前用一句话说明意图;关键节点(完成、阻塞、转向)给简短更新;不复述内部思考,**不在收尾时停下询问"是否继续"或"是否提交"**——直接输出总结并结束。
|
||||
- **安全边界**:仅在高风险、不可逆操作或真正阻塞时才暂停找用户。
|
||||
|
||||
## 初始确认(一次性,仅在最开始)
|
||||
|
||||
任务启动时,用 `AskUserQuestion` 一次性确认以下信息(已由项目规范覆盖的不必重复确认):
|
||||
|
||||
1. **目标与范围**:要解决什么问题?交付物是什么?显式列出不在范围内的内容。
|
||||
2. **验收标准**:怎样算"完成"?可观测的判定条件(功能、性能、覆盖率阈值)。
|
||||
3. **特殊约束**:除 `python-standards.md` 之外的约束(兼容性、依赖限制、API 兼容策略等)。
|
||||
4. **测试要求**:覆盖率门槛(项目默认 ≥95%,branch);是否需要新增 `slow` 标记。
|
||||
|
||||
**git commit/push 不在确认范围内**:任务完成后自动 commit + push(仅当分支已跟踪远程时执行 push;新分支跳过 push 并在总结中说明),遵循 `.trae/rules/git-commit-message.md` 风格。仅 force-push、reset --hard、clean -f、修改 git config 等真正破坏性操作才需暂停确认。
|
||||
|
||||
确认后,将目标与验收标准固化进 `TaskCreate` 任务列表,后续不再就同一信息反复询问。
|
||||
|
||||
## 迭代循环
|
||||
|
||||
下列五个阶段构成一个完整闭环。未达验收标准时,回到「计划」开启下一轮;达标准时,进入「收尾」。
|
||||
|
||||
### 1. 计划(Plan)
|
||||
|
||||
- 用 Explore/Glob/Grep 研究相关代码与既有模式,避免凭空设计。
|
||||
- 用 `TaskCreate` 把目标拆为可独立验证的子任务;每完成一项立即 `TaskUpdate` 为 completed。
|
||||
- 优先复用现有抽象;不为本轮假想需求设计接口。
|
||||
- 不过早抽象:三处相似才考虑提取,否则就地写。
|
||||
|
||||
### 2. 实现(Code)
|
||||
|
||||
- 严格遵守 `.trae/rules/python-standards.md` 与既有代码风格。
|
||||
- 优先 Edit 现有文件;新增文件需有明确职责边界。
|
||||
- 不引入运行时依赖(项目零依赖原则);确需引入须在计划阶段说明。
|
||||
- 公共 API 必须有完整类型注解与中文 docstring。
|
||||
- 不写未被要求的功能、不为未来场景预留扩展点。
|
||||
|
||||
### 3. 测试(Test)
|
||||
|
||||
- 新增/修改的公共 API 必须配套测试;优先通过公共接口测试,故障注入可访问私有属性并在 docstring 注明。
|
||||
- Mock 优先级:`monkeypatch` > 内联 stub > `unittest.mock` > `pytest-mock`;禁用 `@patch` 装饰器。
|
||||
- 必跑校验(每次修改后):
|
||||
|
||||
```bash
|
||||
uvx --from pyflowx pymake tc
|
||||
uvx --from pyflowx pymake cov
|
||||
```
|
||||
|
||||
- 测试失败时定位根因再修复,不通过放宽断言或 `# pragma: no cover` 绕过。
|
||||
- 覆盖率不得低于上一次的值(项目门槛 95%,branch)。
|
||||
|
||||
### 4. 文档(Docs)
|
||||
|
||||
- 同步更新 docstring、README、模块结构说明。
|
||||
- 行为变更须同步更新 `.agents/skills/pyflowx-development/SKILL.md` 中的对应章节。
|
||||
- 跨会话有价值的设计决策、约束、陷阱,追加到 memory(`project_memory.md` 或对应 `topics.md`)。
|
||||
- 不主动新建 `*.md` 文档;除非用户明确要求。
|
||||
|
||||
### 5. 验证(Verify)
|
||||
|
||||
- 逐条对照初始确认的「验收标准」核验;未满足则回到「计划」继续下一轮。
|
||||
- 全套门禁通过:ruff、pyrefly、pytest、coverage。
|
||||
- 给出本轮变更清单(改了哪些文件、为什么)。
|
||||
|
||||
## 多阶段项目
|
||||
|
||||
当项目按阶段(P0/P1/P2/...)划分时,初始确认的目标是**整个项目**,而非单个阶段。
|
||||
|
||||
- **阶段是里程碑,不是边界**:确认"推进 P1"只是指定下一步重点,不缩小整体目标范围;
|
||||
也不意味着 P1 完成后就可以停下。整体目标 = 所有阶段交付完毕。
|
||||
- **阶段完成不是停止条件**:某阶段验收标准全部满足后,**自动进入下一阶段的「计划」步骤**,
|
||||
不停下来输出"是否继续"或"是否扩展范围"的询问。仅在阶段切换时用一句话说明
|
||||
"进入 Pn,重点:…",然后继续自驱动迭代。
|
||||
- **「收尾」仅适用于整个项目完成**:只有所有阶段(含最后一个)都交付后,才执行
|
||||
「收尾」输出最终总结。单阶段完成 → 进入下一阶段的「计划」,不触发收尾。
|
||||
- **跨阶段依赖**:若下一阶段依赖外部资源(如新依赖审批、环境配置),属"不可恢复的失败"
|
||||
暂停条件;否则一律连续推进。
|
||||
- **阶段范围超出预期**:执行中发现某阶段需要显著扩大范围或改变方向(如 P2 本来只做
|
||||
表格抽取,发现必须先重写 IR),属"超出初始确认范围"暂停条件,需找用户确认。
|
||||
|
||||
## 暂停条件(仅在以下情况中断自驱动找用户)
|
||||
|
||||
1. **歧义无法自决**:需求存在多种合理解读且无既有约定可循。
|
||||
2. **高风险/不可逆操作**:删除非临时文件、`git push --force`、`reset --hard`、删表、修改 CI 配置、修改 git config、卸载依赖等。**普通 `git commit`/`push` 不属于此类**(任务完成后自动执行)。
|
||||
3. **不可恢复的失败**:根因不在本仓库、需外部环境/权限配合、或经两轮尝试仍无法定位。
|
||||
4. **超出初始确认范围**:用户目标在执行中发现需要显著扩大范围或改变方向。
|
||||
5. **用户主动询问**:用户在对话中提出新问题或要求澄清。
|
||||
|
||||
**注意**:"目标已达成"**不是**暂停条件——但"目标已达成"指**整个项目目标**(所有阶段交付完毕),而非单个阶段验收满足。单阶段验收满足后应自动进入下一阶段(见「多阶段项目」),不触发收尾。
|
||||
|
||||
非以上情况,一律继续自驱动,不要为"求确认"而暂停。
|
||||
|
||||
## 决策判据:该问还是自决
|
||||
|
||||
遇到不确定时,按以下顺序判断:
|
||||
|
||||
1. **是否不可逆/高风险?** 是 → 暂停确认(如删除文件、`push --force`、修改 CI 配置、卸载依赖)。否 → 继续。
|
||||
2. **是否在初始确认范围内?** 是 → 按确认执行,不询问。否 → 视为"超出初始确认范围",暂停。
|
||||
3. **是否有既有约定可循?** 是 → 按约定执行(参考 `python-standards.md`、`project_memory.md`)。否 → 视为"歧义无法自决",暂停。
|
||||
4. **是否可逆?** 是 → 直接执行,即使结果可能不完美(可在后续迭代修正)。否 → 暂停。
|
||||
|
||||
**可直接自决(不询问)的典型情况**:
|
||||
|
||||
- 测试失败、覆盖率不达标、lint/类型检查报错 → 定位根因并修复。
|
||||
- 代码风格选择(命名、模块划分、参数顺序)→ 自决。
|
||||
- 文件编辑、运行测试、运行校验命令 → 直接执行。
|
||||
- 任务完成后输出收尾总结 → 直接输出,不询问下一步。
|
||||
- 显式指定 `name` 参数以保持测试兼容性 → 自决。
|
||||
- 重命名局部变量以避免遮蔽 → 自决。
|
||||
|
||||
**必须暂停询问的典型情况**:
|
||||
|
||||
- 删除非临时文件、重命名公共模块/包。
|
||||
- `git push --force`、`reset --hard`、`clean -f`、修改 git config(普通 commit/push 自动执行,无需询问)。
|
||||
- 引入新的运行时依赖(违反项目零依赖原则)。
|
||||
- 修改 CI 配置、pre-commit 钩子、pyproject.toml 的工具链配置。
|
||||
- 卸载或降级既有依赖。
|
||||
|
||||
## 沟通风格
|
||||
|
||||
- 阶段切换时一句话说明即可;不要把内部推理写给用户看。
|
||||
- 完成子任务后用一两句总结改了什么、下一步做什么。
|
||||
- 遇到阻塞时直接说明:卡在哪、试了什么、需要用户做什么。
|
||||
- **不在收尾时询问"是否需要提交"或"是否扩展范围"**——直接输出总结并结束。用户后续若有新需求,由用户主动提出。
|
||||
- 不使用 emoji,除非用户明确要求。
|
||||
|
||||
## 工具使用
|
||||
|
||||
- 独立操作尽量并行调用(多个 Read/Grep/Glob 一批发出)。
|
||||
- 用 `TaskCreate`/`TaskUpdate` 维护进度,不批量推迟标记。
|
||||
- 长命令用后台运行(`run_in_background`),完成会自动通知。
|
||||
- 文件操作一律用专用工具:Read/Edit/Write/Glob/Grep,不用 `cat`/`sed`/`grep`/`find`。
|
||||
|
||||
## 收尾
|
||||
|
||||
- **仅当整个项目所有阶段都交付后**,才执行收尾:直接输出最终总结并结束任务(交付物、关键决策、遗留事项)。
|
||||
- 单阶段完成**不属于收尾**——见「多阶段项目」,应自动进入下一阶段的「计划」。
|
||||
- **自动提交**:收尾时自动 `git add`(按文件名)+ `git commit`(遵循 `.trae/rules/git-commit-message.md` 风格)+ `git push`(仅当分支已跟踪远程时执行;新分支跳过 push 并在总结中说明);**不询问**"是否需要提交"或"是否扩展范围"。
|
||||
- 若验收标准未全部满足,回到「计划」继续下一轮,不停下询问。
|
||||
- 将本次会话的关键产出与决策更新到 memory,便于后续会话续接。
|
||||
@@ -0,0 +1,967 @@
|
||||
---
|
||||
name: "pyflowx-development"
|
||||
description: "PyFlowX 项目(DAG 任务调度库)的开发知识库。归档迭代 06-20 中可复用的架构模式、踩坑总结与设计决策,并同步迭代 21(P9)、迭代 22(P10 图片处理)、迭代 23(P11 文件操作底层 API)、迭代 24(P12 通用 DAG 构造器)与迭代 25(执行热路径性能优化)的行为变更。在设计与调度器、CLI、YAML 编排、取消机制、序列化、观测性、错误诊断、状态后端、性能优化、任务编排、检查点恢复、监控导出、图片处理、文件操作底层 API、通用 DAG 构造器相关的功能时参考。"
|
||||
---
|
||||
|
||||
# PyFlowX 开发知识库
|
||||
|
||||
本技能归档自迭代 06-20 的过程记录,并同步迭代 21(P9 功能扩展)、
|
||||
迭代 22(P10 图片处理)、迭代 23(P11 文件操作底层 API)、迭代 24
|
||||
(P12 通用 DAG 构造器)与迭代 25(执行热路径性能优化)的行为变更。
|
||||
按主题分类整理可复用知识。
|
||||
过程性细节(覆盖率数字、命令输出)已剔除,仅保留架构模式、设计依据
|
||||
与陷阱总结。
|
||||
|
||||
## 一、兼容性与代码清理
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **`typing_extensions` 版本守卫不可移除**:`override`(3.12+)与
|
||||
`TypeVar(default=)`(3.13+)在 3.10/3.11/3.12 上仍需 `typing_extensions`
|
||||
回退。守卫是前向兼容代码,不是 3.8 死代码。
|
||||
`pyproject.toml` 声明 `typing-extensions>=4.13.2; python_version < '3.13'`。
|
||||
- **`from __future__ import annotations` 保留**:PEP 563 在 3.10+ 仍提供
|
||||
前向引用与延迟求值价值。统一移除需逐文件审计前向引用、自引用类型,
|
||||
风险高收益低。
|
||||
- **`rules/` 修改必须先获用户授权**:`.trae/rules/` 下文件修改受
|
||||
`dev-workflow.md` 约束,未授权前不得擅自变动,即便描述已过时
|
||||
(如 "最低 Python 3.8" 实际已升至 >=3.10)。
|
||||
|
||||
### 踩坑总结
|
||||
|
||||
- **不要凭"3.8 兼容代码"假设做清理**:调研后才发现 `typing_extensions`
|
||||
守卫实际是 3.12/3.13 前向兼容,删除会破坏 3.10-3.12 上的 `override`
|
||||
与 `TypeVar(default=)`。清理前必须验证"过期代码"的真实用途。
|
||||
- **README/文档与代码状态必须同步**:未实现的特性声明(如未落地的
|
||||
YAML 编排章节)、过期的覆盖率徽章、错误的依赖描述都会误导用户。
|
||||
清理时优先对齐 README 与 `pyproject.toml`。
|
||||
|
||||
## 二、YAML 任务编排(GitHub Actions 风格)
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **Matrix 笛卡尔积展开**:多键矩阵用 `itertools.product` 展开为笛卡尔积,
|
||||
命名 `{job_id}_{key}-{value}`(多键用 `_` 连接)。
|
||||
如 `version: ["3.10","3.11"]` × `os: ["linux","macos"]` 展开 4 个任务
|
||||
`test_version-3.10_os-linux` 等。
|
||||
- **矩阵依赖自动展开**:若 job A `needs: [B]`,B 是矩阵 job,A 依赖 B 的
|
||||
所有变体(无需手动列出每个变体)。
|
||||
- **`${{ matrix.key }}` 占位符替换**:在 cmd/run/cwd/env 的字符串值中
|
||||
统一替换矩阵占位符。
|
||||
- **字段名 hyphen/underscore 兼容**:YAML 中 `continue-on-error` 与
|
||||
`continue_on_error` 等价,读取时统一规范化。
|
||||
- **cmd vs run 区分**:
|
||||
- `cmd: [...]` → list 形式(无 shell,直接 exec)
|
||||
- `run: "..."` → shell 字符串形式
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **条件解析映射到内置 Conditions**:
|
||||
- `success()` / `always()` → 无条件(空 conditions)
|
||||
- `env.VAR` → `ENV_VAR_EXISTS("VAR")`
|
||||
- `env.VAR == 'x'` → `ENV_VAR_EQUALS("VAR", "x")`
|
||||
- `env.VAR != 'x'` → `NOT(ENV_VAR_EQUALS("VAR", "x"))`
|
||||
- `failure()` → 不支持,抛 `ValueError`(PyFlowX 不支持 failure 触发)
|
||||
- **yamlrun 作为特殊命令而非 `@px.tool`**:CLI 路由
|
||||
`pf yamlrun <file> [--strategy S] [--dry-run] [--list] [--quiet] [--progress]`
|
||||
作为 `PfApp` 的 builtin 特殊命令注册,与 `pf graph` 同级,不通过 `@px.tool` 装饰器。
|
||||
- **Graph.from_yaml 委托 yaml_loader**:`Graph.from_yaml(path)` classmethod
|
||||
委托给 `yaml_loader.load_yaml(path)` 函数式 API,保持 Graph 类职责单一。
|
||||
|
||||
### Matrix include/exclude 语义(iter-19)
|
||||
|
||||
- **exclude**: 组合的所有 key-value 对都匹配则剔除(`combo.items() >= exclude.items()`)。
|
||||
如 `exclude: [{os: macos, version: "3.8"}]` 剔除 macos+3.8 组合。
|
||||
- **include**: 追加额外组合,可引入基础矩阵之外的键。
|
||||
如基础矩阵 `version: ["3.10"]`,`include: [{version: "3.11", experimental: true}]`
|
||||
追加 3.11 实验组合。
|
||||
- **无基础矩阵键时返回空列表而非 `[{}]`**:避免产生空名任务(`job_`)。
|
||||
|
||||
### 条件依赖(iter-19)
|
||||
|
||||
- **`needs` 项支持 `{job: name, if: expr}` 格式**:条件不满足时跳过该依赖
|
||||
(不加入 `depends_on`)。
|
||||
- 复用 `_parse_condition` 解析 `if` 表达式;条件求值时传空上下文 `{}`
|
||||
(`env` 条件直接读 `os.environ`)。
|
||||
- **典型场景**:仅在特定矩阵变体下才需要的依赖,如
|
||||
`needs: [{job: setup, if: env.DEPLOY == "true"}]`。
|
||||
|
||||
### outputs 字段(iter-19)
|
||||
|
||||
- `TaskSpec` 新增 `outputs: Mapping[str, str] | None` 作为元数据,
|
||||
仅携带不参与执行,可通过 `spec.outputs` 查询。
|
||||
- 支持 `${{ matrix.* }}` 占位符替换。
|
||||
- `to_dict` / `from_json` 往返保留;序列化向前兼容(旧 JSON 无此字段时回退 None)。
|
||||
|
||||
### 踩坑总结
|
||||
|
||||
- **YAML 加载失败统一包装**:`yaml_loader._safe_load` 应将 `yaml.YAMLError`
|
||||
包装为 `ValueError` 抛出,CLI 层统一捕获处理。否则 YAML 语法错误会
|
||||
以底层异常形式泄漏到用户面前。
|
||||
- **不可达防御代码应清理**:`_cartesian_product` 与 `_parse_condition` 中
|
||||
的防御性分支若逻辑上不可达,应删除而非保留 `# pragma: no cover`。
|
||||
|
||||
## 三、性能优化与基准套件
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **独立 `benchmarks/` 目录**(非 pytest 测试),用 `time.perf_counter` 计时,
|
||||
通过 `python -m benchmarks` CLI 入口运行。覆盖:
|
||||
- 图构建:10/100/500/1000 节点 DAG 的构建 + validate + layers
|
||||
- 任务执行:空 fn 任务 × 100/500,四种策略对比
|
||||
- 上下文注入:有/无依赖、有/无 Context 标注
|
||||
- 状态后端:MemoryBackend vs JSONBackend vs SQLiteBackend
|
||||
- **`layers()` 缓存模式**:Graph.layers() 每次 run() 都重算拓扑排序,
|
||||
应缓存。模式:
|
||||
- 添加 `_layers_cache` 私有字段
|
||||
- 首次调用计算并缓存
|
||||
- `add()` / `clear()` 等变更方法中失效缓存
|
||||
- 参照 `resolved_spec` 已有缓存模式
|
||||
- **cmd 任务快速路径**:cmd 任务的 `effective_fn` 是无参闭包 `_run()`,
|
||||
无需上下文注入。检测 `spec.fn is None and spec.cmd is not None` 时
|
||||
直接返回 `((), {})`,跳过 `build_call_args` 的签名内省与 dep_context 构建。
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **性能特征(500 空任务)**:
|
||||
- `sequential`:514 ops/s(最快,无并发开销)—— 小任务/无 I/O 场景首选
|
||||
- `thread`:93 ops/s(线程池开销)—— I/O 密集型场景
|
||||
- `async`:44 ops/s(事件循环开销)—— 高并发 I/O 场景
|
||||
- `dependency`:42 ops/s(最大并行度但调度开销高)—— 复杂依赖图
|
||||
- **状态后端性能特征**:
|
||||
- `MemoryBackend`:save/load 600万/447万 ops/s(首选,无持久化需求时)
|
||||
- `JSONBackend`:save(batch=10)/load 3913/11.9万 ops/s
|
||||
- `SQLiteBackend`:save(batch=10)/load 9657/1.5万 ops/s(save 更快,load 较慢)
|
||||
- **缓存优化收益**:`layers()` 缓存命中 ~1500万 ops/s(~50000x 加速);
|
||||
cmd 快速路径 1130万 ops/s vs fn 无依赖 144万 ops/s(~8x 加速)。
|
||||
|
||||
### 自实现 Kahn 算法拓扑排序(iter-20)
|
||||
|
||||
- **方案**:模块级 `_topological_layers(deps)` 函数用 dict + list 直接计算
|
||||
入度与反向邻接,按层 BFS 处理。返回 `(layers, cycle_nodes)`:
|
||||
- 无环时 `cycle_nodes = None`
|
||||
- 有环时 `cycle_nodes` 为入度仍 >0 的节点列表(非精确环路径,仅指示存在环)
|
||||
- **额外优化**:`validate()` 顺带填充 `_layers_cache`,使后续 `layers()`
|
||||
直接命中缓存,避免二次计算 `_topological_layers`。
|
||||
- **结果**:diamond(1000) 图构建约 2.21ms(~452 ops/s)。
|
||||
|
||||
### build_call_args 双快速路径(iter-17)
|
||||
|
||||
- **快速路径 1(cmd 任务)**:`spec.fn is None and spec.cmd is not None and
|
||||
not spec.args and not spec.kwargs` → 直接返回 `((), {})`。
|
||||
- **快速路径 2(fn 无依赖)**:`not spec.depends_on and not spec.soft_depends_on
|
||||
and not spec.args and not spec.kwargs` → 调用 `_fn_no_dep_injection(fn)`
|
||||
获取 `@lru_cache(maxsize=1024)` 缓存的注入计划 `(context_params, error_param)`。
|
||||
- `context_params`:Context 标注参数名元组;存在必填无默认值参数时返回 `None`
|
||||
(需走慢路径)。
|
||||
- `error_param`:第一个无默认值且非 Context 标注的参数名(用于错误信息)。
|
||||
- **不变量**:`error_param is None ⟺ context_params 非 None`,用 `assert`
|
||||
辅助 pyrefly 类型收窄。
|
||||
- **提取到 `_try_fast_path` 函数**:避免 `build_call_args` 分支数超 PLR0912
|
||||
阈值 12。
|
||||
- **不可哈希 fn 回退**:`_cached_signature` 抛 TypeError 时回退到直接内省。
|
||||
|
||||
### JSONBackend batch 延迟验证(iter-17)
|
||||
|
||||
- **问题**:`save` 原本每次都 `json.dumps(value)` 验证可序列化性,batch
|
||||
模式下 N 次 save 产生 N 次验证开销,但 `flush` 只做一次 `json.dump`。
|
||||
- **方案**:`_defer_flush=True` 时跳过 `json.dumps` 验证,让 `flush` 的
|
||||
`json.dump` 统一捕获 `TypeError`/`ValueError`。非 batch 模式仍即时验证
|
||||
以提供精确错误位置。
|
||||
- **依据**:非 batch 调用方期望即时错误反馈;batch 调用方接受延迟错误以换吞吐。
|
||||
|
||||
### from_specs 批量注册(iter-17)
|
||||
|
||||
- **问题**:`from_specs` 原本对每个 spec 调 `_register`,而 `_register`
|
||||
每次都 `_resolved_cache.clear()` + `_layers_cache = None`,N 个 spec
|
||||
产生 N 次缓存清空。
|
||||
- **方案**:直接写入 `graph.specs` / `graph.deps` 字典,重复名检测在循环内做。
|
||||
缓存失效由后续 `validate()` / `layers()` 首次调用自然触发(cache 为空时
|
||||
自动重算)。
|
||||
- **不扩展到 `add`/`chain`**:增量 API 需即时校验以快速失败;批量构造可
|
||||
接受延迟校验。
|
||||
|
||||
### orjson 可选依赖抽象层(iter-20)
|
||||
|
||||
- **方案**:`_json.py` 模块 `try: import orjson except ImportError: import json`
|
||||
回退。导出 `dumps`/`loads`/`dump`/`load`/`JSONDecodeError`,签名与 stdlib
|
||||
兼容。
|
||||
- **安装**:`pip install pyflowx[fast]` 启用 orjson(`[project.optional-dependencies]
|
||||
fast = ["orjson>=3.10.0"]`)。
|
||||
- **orjson 兼容性差异**:
|
||||
- `dumps` 返回 `bytes` → wrapper 中 `.decode("utf-8")` 统一为 `str`
|
||||
- 无 `ensure_ascii` 参数 → wrapper 接受并忽略(`# noqa: ARG001`),orjson 始终 UTF-8
|
||||
- 无 `indent` 参数 → 映射为 `OPT_INDENT_2`
|
||||
- 紧凑模式无空格(`[1,2,3]` vs stdlib `[1, 2, 3]`)→ 测试断言改为格式无关
|
||||
- `JSONDecodeError` 直接复用 orjson 的(是 stdlib `json.JSONDecodeError` 子类)
|
||||
- **pyrefly 兼容**:`import orjson` 在 try 块内,所有 `orjson.xxx` 引用加
|
||||
`# type: ignore[possibly-unbound]`;orjson 未安装时 `# type: ignore[import-not-found]`。
|
||||
|
||||
### slots=True 内存优化(iter-20)
|
||||
|
||||
- **方案**:所有 dataclass 添加 `slots=True`,消除 per-instance `__dict__`
|
||||
开销(约 100-150 bytes/instance)。
|
||||
- **应用范围**:`Graph`/`GraphDefaults`/`TaskSpec`/`TaskResult`/`RetryPolicy`/
|
||||
`TaskHooks`/`TaskEvent`。
|
||||
- **冲突处理**:`frozen=True` + `slots=True` + `cached_property` 不兼容
|
||||
(无 `__dict__` 存缓存,frozen 禁止设 slot)→ 将 `TaskSpec.effective_fn`
|
||||
从 `@cached_property` 改为 `@property`。闭包创建极轻量(仅捕获 `self`),
|
||||
重复访问开销可忽略。
|
||||
- **结果**:10k TaskSpec 929 → 785 bytes/instance(15.5% 减少,1.38 MB 节省)。
|
||||
|
||||
### 线程池跨层复用(iter-20)
|
||||
|
||||
- **问题**:`thread` 策略每层创建新 `ThreadPoolExecutor`,N 层图创建/销毁
|
||||
N 次线程池。
|
||||
- **方案**:`_drive_threaded` 创建一个池,跨所有层复用;`max_workers` 取
|
||||
最大层宽 `max((len(layer) for layer in layers), default=1)`,上限 32。
|
||||
- **API 变更**:`ThreadedLayerRunner.execute` 签名 `max_workers: int` →
|
||||
`pool: concurrent.futures.ThreadPoolExecutor`,调用方负责池生命周期。
|
||||
- **额外**:`asyncio.get_event_loop()` → `asyncio.get_running_loop()`,
|
||||
兼容 Python 3.12+ 弃用警告(调用点均在 `asyncio.run()` 内的协程中)。
|
||||
|
||||
### 执行热路径 4 项优化(iter-25)
|
||||
|
||||
sequential(500) 从 764 → 930 ops/s(+22%),thread(500) 从 108 → 119 ops/s(+10%)。
|
||||
|
||||
- **env_context 快速路径**:`SyncTaskRunner.run` 与 `_submit_sync_task.fn_call`
|
||||
内联 `if spec.env is None and spec.cwd is None: effective_fn(...)`,跳过
|
||||
`with spec.env_context():` 的 `_GeneratorContextManager` 对象创建。即使
|
||||
`_env_and_cwd` 内部已有 `if not env and cwd is None: yield` 快速路径,
|
||||
contextlib 包装器的对象创建 + `__enter__`/`__exit__` 协议开销仍存在,
|
||||
内联判断完全消除这层开销。对 fn 任务(最常见场景)每任务省一次上下文管理器。
|
||||
- **should_execute 早退**:`not self.conditions and not self.skip_if_missing`
|
||||
时直接 `return True, None`,避免空 list 分配 + 空序列迭代。
|
||||
- **resolved_spec 去重**:`_filter_and_sort` 返回 `(to_run, specs)` 元组,
|
||||
specs dict 透传给 Layer runner;`_build_semaphores` 签名 `graph: Graph` →
|
||||
`specs: Mapping[str, TaskSpec]`。消除 Sequential/Threaded/AsyncLayerRunner
|
||||
内的重复 `graph.resolved_spec(name)` 调用。`DependencyRunner._run_task` 仍
|
||||
调 `resolved_spec`(动态任务不在初始 specs 中)。
|
||||
- **storage_key 内联**:`_apply_cached`/`_store_result` 内联
|
||||
`spec.name if spec.cache_key is None else spec.storage_key(ctx)`,避免无
|
||||
cache_key 场景(最常见)的函数调用开销。
|
||||
- **适用边界**:async/dependency 策略对 noop 任务的固有并发开销(事件循环 +
|
||||
线程池提交)主导,per-task 优化相对影响小;真实 I/O 任务才能体现并行优势。
|
||||
|
||||
### 事件驱动优先级调度(iter-27)
|
||||
|
||||
DependencyRunner 从"一次性创建所有 asyncio Task"重写为"事件驱动按需创建":
|
||||
仅当任务依赖全部完成时才创建 asyncio Task,多个就绪任务按 `priority` 降序
|
||||
创建,高优先级任务优先获取事件循环与线程池资源。
|
||||
|
||||
- **`completed` / `in_flight` / `remaining` 三集合**:跟踪任务生命周期。
|
||||
`completed` = 已完成(含 SUCCESS/SKIPPED/FAILED);`in_flight` = asyncio
|
||||
Task 已创建未完成;`remaining` = 尚未调度。
|
||||
- **`concurrency_key=1` 串行化验证优先级**:测试中用 `concurrency_key="g"`
|
||||
+ `concurrency_limits={"g": 1}` 使无依赖任务串行执行,优先级顺序可观测。
|
||||
无 concurrency_key 时任务真并行,优先级仅影响 asyncio Task 创建顺序。
|
||||
- **与层策略优先级的差异**:层策略(sequential/thread/async)仅在同层内
|
||||
按优先级排序;dependency 策略跨层,按就绪时刻排序——依赖完成后立即就绪,
|
||||
不等同层其他任务。
|
||||
|
||||
### 增量就绪集优化(iter-27)
|
||||
|
||||
DependencyRunner 主循环每轮调用 `_find_ready()` 扫描全部 `remaining` 任务
|
||||
检查依赖是否满足,O(N*D) per round、O(N²·D) total。10k 任务链式图
|
||||
(10k 轮调度)下成为瓶颈。
|
||||
|
||||
- **`_build_dependency_index` 模块级函数**:预计算 `in_degree[name]`
|
||||
(尚未完成的硬+软依赖数)+ `dependents[d]`(依赖 d 的任务列表,反向邻接表)
|
||||
+ `ready`(in_degree=0 的初始就绪集)。复杂度从 O(N²·D) 降至 O(N·D)。
|
||||
- **`_on_complete(name)`**:任务完成后遍历 `dependents[name]`,逐个递减
|
||||
`in_degree`;降为 0 时加入 `ready` 集合。每完成一个任务仅 O(D_out) 操作。
|
||||
- **`_register_dynamic(raw_spec, spawner)`**:动态生成任务接入增量结构。
|
||||
spawner 此时未标记 completed(仍在 in_flight),故计入 unsatisfied;
|
||||
spawner 完成后 `_on_complete` 会递减并触发就绪。
|
||||
- **基准验证**:chain(10000) ~950ms、diamond(10000) ~740ms、wide(10000)
|
||||
~790ms。chain 线性扩展(1k:87ms → 5k:459ms → 10k:948ms),确认 O(N) 调度。
|
||||
|
||||
### 踩坑总结
|
||||
|
||||
- **`lru_cache` 对签名内省有 dict lookup 开销**:即便 `functools.lru_cache`
|
||||
缓存了 `_signature(fn)`,每次仍有 dict lookup。对热路径(如 cmd 任务)
|
||||
应在更外层短路,避免进入 `build_call_args`。
|
||||
- **`frozen=True` + `slots=True` + `cached_property` 三者不兼容**:选其中
|
||||
两个。若需 slots 内存优化,将 `cached_property` 改为 `@property`(轻量
|
||||
闭包场景)或外部 dict 缓存(重计算场景)。
|
||||
- **orjson 紧凑格式与 stdlib 不同**:跨后端测试需用格式无关断言
|
||||
(`assert "[1,2,3]" in s or "[1, 2, 3]" in s`),不能硬编码空格。
|
||||
- **monkeypatch 目标需跟随 import 路径**:当模块从 `import json` 改为
|
||||
`from ._json import dump`,测试中 `monkeypatch.setattr(json, "dump", ...)`
|
||||
失效,需改为 `monkeypatch.setattr(storage_mod, "dump", ...)`。
|
||||
|
||||
## 四、任务取消与优雅停止
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **`CancelToken` 基于 `threading.Event`**:线程安全取消令牌,可跨线程/协程使用。
|
||||
接口:
|
||||
- `cancel()` 设置取消状态
|
||||
- `is_cancelled` 属性检查
|
||||
- `wait(timeout)` 等待取消信号
|
||||
- **统一取消检查**:`_is_cancelled(effective_cancel) or internal_cancel.is_cancelled`
|
||||
统一外部取消与 KeyboardInterrupt 的内部取消,避免各处写两段检查逻辑。
|
||||
- **各策略取消点设计**:
|
||||
- `sequential`:每个任务执行前检查
|
||||
- `thread`:每层提交前检查;已提交任务等待完成(不强制中断)
|
||||
- `async`:每个 task 创建前检查;用 `asyncio.CancelledError`
|
||||
- `dependency`:每个 `_run_task` 创建前检查
|
||||
- **KeyInterrupt 优雅处理模式**:
|
||||
```python
|
||||
try:
|
||||
execute(...)
|
||||
except KeyboardInterrupt:
|
||||
cancel_event.set()
|
||||
# 等待运行中任务完成(最多 5 秒)
|
||||
# 标记未完成任务为 SKIPPED
|
||||
report.success = False
|
||||
finally:
|
||||
cleanup(...) # 进程池、监控器、通知器
|
||||
```
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **取消后待运行任务标记 SKIPPED,运行中任务等待完成**:不强制中断运行中
|
||||
的任务,避免资源泄漏与状态不一致。这是"优雅停止"的核心定义。
|
||||
- **`run()` 接受可选 `cancel_event`**:调用方未提供时内部创建。这让
|
||||
外部代码可以预先持有 cancel_event 引用,在任意时刻触发取消。
|
||||
- **提取 `_dispatch_strategy` 辅助函数控制分支数**:`run()` 函数中
|
||||
四种策略的分发若用 if/elif 链会导致 PLR0912(分支过多)。提取辅助
|
||||
函数将 `run()` 的 return 数量控制在 12 以内。
|
||||
|
||||
### 踩坑总结
|
||||
|
||||
- **资源清理必须在 finally 块**:进程池、监控器、通知器若在 try 块内
|
||||
清理,KeyboardInterrupt 会跳过清理导致泄漏。
|
||||
- **`PLR0912`/`PLR0911` 分支/返回过多**:复杂调度逻辑容易触发。优先
|
||||
提取辅助函数(如 `_dispatch_strategy`、`_print_diagnostics`),
|
||||
而非用 `# noqa` 抑制。合并语义相同的分支(如 no-args 与 --help
|
||||
合并)也是有效手段。
|
||||
|
||||
## 五、子图执行
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **`Graph.subgraph_with_deps(names)` 传递闭包**:从 `names` 出发,
|
||||
沿 `depends_on` + `soft_depends_on` 向上遍历,收集所有必须先完成的任务。
|
||||
返回新的 Graph 或任务名集合。
|
||||
- **`run(only, tags)` 参数组合**:
|
||||
- `only`:任务名列表,运行这些任务及其传递依赖
|
||||
- `tags`:标签列表,运行匹配任意标签的任务及其传递依赖
|
||||
- 两者可组合:并集后计算传递闭包
|
||||
- 都为 None 时运行全图(保持现有行为不变)
|
||||
- **CLI 选项设计**:`pf yamlrun pipeline.yaml --only task1,task2 --tags ingest,report`
|
||||
逗号分隔,与执行过滤解耦。
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **传递闭包同时包含软依赖**:`subgraph_with_deps` 沿 `depends_on` 与
|
||||
`soft_depends_on` 双向遍历。软依赖也是必须先完成的任务,不可遗漏。
|
||||
|
||||
## 六、结果序列化
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **值序列化默认策略**(处理任意 Python 对象):
|
||||
1. `None` 直接返回 `None`
|
||||
2. 可 JSON 序列化的原样返回
|
||||
3. 不可序列化的回退到 `repr(value)`
|
||||
4. 调用方可通过 `value_serializer` 自定义
|
||||
- **`to_dict` 结构用列表保序**:`results` 为列表(非 dict),便于
|
||||
CSV/JSON 数组导出,保留执行顺序。
|
||||
字段:`name/status/value/error/attempts/started_at/finished_at/duration_seconds/reason/tags/depends_on`。
|
||||
- **CSV 导出省略 value 列**:`include_value=False` 时省略 value 列,
|
||||
因为值可能含逗号/换行符干扰 CSV 解析。
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **`from_json` 仅供查询,不能重新执行**:反序列化只重建可序列化字段,
|
||||
`TaskSpec` 的 `fn`/`cmd` 等不可恢复,使用 noop 占位。重建后的 report
|
||||
仅供查询/分析,不能用于重新执行。这是明确的契约边界。
|
||||
- **`datetime` 通过 `fromisoformat` 还原**:ISO 8601 字符串往返转换,
|
||||
避免自定义时间格式。
|
||||
|
||||
### HTML 报告导出设计(iter-19)
|
||||
|
||||
- **自包含 HTML5 文档**:内联 CSS,无外部依赖,便于邮件附件/嵌入式展示。
|
||||
- **XSS 防护**:所有用户内容(任务名/错误/原因/值)经 `html.escape` 转义。
|
||||
- **状态徽章着色**:success 绿 / failed 红 / skipped 黄 / running 蓝,
|
||||
通过 CSS class 区分。
|
||||
- **`_render_summary_card` 静态方法**:渲染汇总卡片(总数/成功/失败/跳过/
|
||||
耗时),避免在 `to_html` 主流程中重复 HTML 拼接。
|
||||
- **value 列渲染**:尝试 `dumps(value)` 序列化后转义;失败回退 `repr(value)`。
|
||||
|
||||
### 踩坑总结
|
||||
|
||||
- **`_noop_fn` 占位函数体无法覆盖**:与 `task.py:_task_noop` 同模式,
|
||||
占位函数的函数体在测试中无法触发,属于可接受的覆盖率缺口。
|
||||
- **orjson 紧凑格式与 stdlib 不同**:HTML 报告中 value 列断言需格式无关
|
||||
(`"[1,2,3]" in s or "[1, 2, 3]" in s`),不能硬编码空格。
|
||||
|
||||
## 七、CLI 可视化与体验
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **`pf graph` 多格式输出**:
|
||||
- `--format ascii`(默认):`rich.tree.Tree` 按层分组 + `rich.table.Table` 显示依赖/标签
|
||||
- `--format mermaid`:复用 `Graph.to_mermaid()` 直接打印(不加颜色,便于复制到 mermaid.live)
|
||||
- `--format list`:仅列任务名(与 `yamlrun --list` 一致)
|
||||
- `--format deps`:纯依赖关系表
|
||||
- `--orientation`:Mermaid 方向(TD/TB/BT/LR/RL)
|
||||
- **`--color-by tag` 着色模式**:维护 `tag_colors` 字典(标签 → rich 颜色名),
|
||||
首次见到标签时从调色板循环分配。无标签任务回退默认色。`color_by="none"`
|
||||
时统一颜色,不显示图例。
|
||||
- **`--list` 过滤与执行过滤解耦**:
|
||||
- `--only`/`--tags` 控制执行范围
|
||||
- `--list-tag`/`--list-name` 仅控制 `--list` 模式的显示
|
||||
- 名称匹配大小写不敏感(子串),标签匹配精确,两者可组合(交集)
|
||||
- **`pf info [tool]` 工具详情**:
|
||||
- 无参数 → 所有工具简表(工具/别名/子命令数/描述)
|
||||
- 带参数 → 工具详情(别名、描述、子命令表)
|
||||
- 支持别名解析(`pf info clearscreen` 等价于 `pf info clr`)
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **shell 补全不依赖第三方库**:直接生成静态脚本
|
||||
- bash: `complete -F _pf_complete pf` + `compgen -W "..."`
|
||||
- zsh: `#compdef pf` + `_describe 'command' cmds`
|
||||
- fish: 每个命令一行 `complete -c pf -n "__fish_use_subcommand" -a "..."`
|
||||
工具名来自 `_TOOL_ALIASES` 规范名集合,二级补全交给子命令的 `--help`。
|
||||
- **`--progress` 与 `--quiet` 互斥**:quiet 时无进度条,避免输出冲突。
|
||||
|
||||
### 踩坑总结
|
||||
|
||||
- **闭包内可变计数器用 `counter[0]` 单元素 list**:pyrefly 在闭包内对
|
||||
父作用域变量的读-写支持不佳,`nonlocal` 会触发类型检查告警。改用
|
||||
`counter = [0]` 单元素 list,闭包内 `counter[0] += 1` 修改的是 list
|
||||
内容(非绑定),pyrefly 接受。
|
||||
- **`_TOOL_REGISTRY` 的 key 类型为 `str | None`**:排序时必须过滤 `None`,
|
||||
否则 `sorted()` 会因类型比较失败抛错。
|
||||
|
||||
## 八、错误诊断
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **诊断数据结构(frozen dataclass)**:
|
||||
- `FailureCluster`:`pattern`(异常类型+消息前缀)/`tasks`/`sample_error`
|
||||
- `DependencyChain`:`failed_task`/`root_cause`/`chain`/`broken_at`
|
||||
- `DiagnosticReport`:`failed_tasks`/`skipped_due_to_failure`/`root_causes`/`dependency_chains`/`failure_clusters`/`hints`
|
||||
- **根因识别算法**:
|
||||
- **根因任务**:FAILED 状态且其所有硬依赖都是 SUCCESS(或无依赖)
|
||||
- **依赖链**:从根因到失败任务的路径,用 BFS 反向遍历依赖图
|
||||
- **broken_at**:链中第一个非 SUCCESS 的任务(通常是根因本身)
|
||||
- **相似失败聚类**:按 `(异常类型名, 消息前 50 字符)` 聚类,
|
||||
便于发现批量失败模式。
|
||||
- **根因提示表**:识别常见异常类型并给出建议
|
||||
- `FileNotFoundError` → 检查路径配置
|
||||
- `ImportError`/`ModuleNotFoundError` → 检查依赖安装
|
||||
- `TimeoutError`/`TaskTimeoutError` → 考虑增加 timeout 或优化性能
|
||||
- `ConnectionError` → 检查网络连通性
|
||||
- `PermissionError` → 检查权限
|
||||
- `KeyError` → 检查上下文注入参数
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **`TaskFailedError` 携带 `report` 字段**:失败时 `_finalize_failure`
|
||||
传入 report,CLI 层可从异常对象直接获取诊断信息,无需重复构造。
|
||||
`report` 字段可选(`Optional[RunReport]`),向后兼容。
|
||||
- **`report.diagnose()` 成功时返回 `None`**:明确区分"无诊断"与"空诊断",
|
||||
避免调用方误判。
|
||||
- **CLI 失败时自动打印诊断摘要到 stderr**:`pf yamlrun` 失败时
|
||||
(`report.success == False`)自动调用 `_print_diagnostics` 辅助方法,
|
||||
无需用户额外参数。
|
||||
|
||||
### 踩坑总结
|
||||
|
||||
- **CLI 异常处理合并同类**:`TaskFailedError` 与 `PyFlowXError` 的
|
||||
处理逻辑相似,应合并到同一 except 块,提取 `_print_diagnostics`
|
||||
辅助方法。重复的 except 分支会增加 PLR0911 风险。
|
||||
|
||||
## 九、观测性
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **`run_id` 短码生成**:`uuid4().hex[:8]` 生成 8 字符十六进制短码,
|
||||
足以区分并发运行,便于人工阅读。`RunReport.run_id` 字段默认自动生成,
|
||||
显式指定时尊重调用方。
|
||||
- **结构化日志 extra 字段标准化**(不用 `LoggerAdapter`):
|
||||
- `run_id` — 运行 ID(所有日志)
|
||||
- `task_name` — 任务名(任务级日志)
|
||||
- `status` — 任务状态(success/failed/skipped/cancelled)
|
||||
- `attempts` — 尝试次数(失败日志)
|
||||
- `error_type` — 异常类型名(失败日志)
|
||||
- `strategy` / `total_tasks` — run() 入口/结束日志
|
||||
- `run_id` 从 `report` 参数读取,无 report 上下文时填 `"-"`
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **放弃 `LoggerAdapter` 方案**:会强制改写所有 `logger.xxx` 调用点,
|
||||
且对模块级函数(`_apply_cached`/`_handle_failure`)签名侵入大。
|
||||
改用直接在每条日志显式 `extra={...}`,散点 extra 已足够,无需过度抽象。
|
||||
- **`profile()` 便捷方法委托 ProfileReport**:
|
||||
```python
|
||||
def profile(self, graph: Graph) -> ProfileReport:
|
||||
from .profiling import ProfileReport
|
||||
|
||||
return ProfileReport.from_report(self, graph)
|
||||
```
|
||||
`Graph` / `ProfileReport` 仅在 `TYPE_CHECKING` 块导入,避免循环依赖;
|
||||
`from __future__ import annotations` 使注解延迟求值。
|
||||
- **序列化向前兼容**:`from_json()` 优先恢复 `run_id`,缺失时自动生成
|
||||
新 ID。旧版 JSON(无 `run_id`)可正常加载。
|
||||
|
||||
### 通知器平台扩展(iter-19)
|
||||
|
||||
- **复用 `WebhookNotifier` 基类**:Slack/Discord/Telegram/Feishu/DingTalk/
|
||||
WeChatNotifier 均继承 `WebhookNotifier`,仅覆盖 `_build_payload(payload: dict) -> dict`
|
||||
适配平台格式。零新依赖(stdlib `urllib.request`)。
|
||||
- **TelegramNotifier 覆盖 `__init__`**:额外需要 `chat_id` 参数,
|
||||
URL 中拼接 `bot_token`。
|
||||
- **`Notifier` Protocol 生命周期**:`notify(level, payload)` + `close()`。
|
||||
`NotificationLevel` 枚举(RUNNING/SUCCESS/FAILED/SKIPPED)+ `ALL_LEVELS`
|
||||
frozenset;`levels` 参数内部过滤。
|
||||
- **WebhookNotifier 通用字段**:`url`/`secret`/`levels`/`timeout`,
|
||||
`_send` 用 `urllib.request.urlopen` 发送 JSON POST,`_build_payload`
|
||||
默认透传(子类按平台格式重写)。
|
||||
|
||||
### 踩坑总结
|
||||
|
||||
- **循环依赖用 `TYPE_CHECKING` 守卫 + 延迟导入**:`report.py` 依赖
|
||||
`profiling.py` 的 `ProfileReport` 与 `graph.py` 的 `Graph`,但运行时
|
||||
不需要。`TYPE_CHECKING` 块导入 + `from __future__ import annotations`
|
||||
注解延迟求值,是打破循环依赖的标准模式。
|
||||
- **不要为未来场景预留扩展点**:`LoggerAdapter` + `extra` 的统一封装
|
||||
属于"未来可考虑"项,当前散点 extra 已足够,不应预先抽象。
|
||||
|
||||
### 监控导出(iter-21,P9.4)
|
||||
|
||||
- **`MetricsCollector` 作为 `on_event` 回调**:收集任务生命周期事件,
|
||||
聚合为 Prometheus 文本格式。线程安全(多线程策略下回调可并发调用)。
|
||||
- 指标:`pyflowx_task_total{task,status}` (counter)、
|
||||
`pyflowx_task_duration_seconds{task}` (gauge,最近值)、
|
||||
`pyflowx_task_duration_seconds_sum{task}` (counter)、
|
||||
`pyflowx_task_retries_total{task}` (counter)、
|
||||
`pyflowx_run_total{status}` (counter)
|
||||
- `record_run(report)` 单独记录运行级指标;`reset()` 清空全部
|
||||
- **`metrics_text()` 按 section 条件化输出**:每个指标 section 仅在
|
||||
有数据时才输出 `# HELP` / `# TYPE` 行。避免 `reset()` 后仍残留
|
||||
HELP/TYPE 行误导调用方判断"是否还有数据"。
|
||||
- **`health_check(report)` 状态判定**:
|
||||
- `healthy` — 全部 SUCCESS
|
||||
- `degraded` — 部分(非全部)FAILED
|
||||
- `unhealthy` — 全部 FAILED
|
||||
- `unknown` — 无任务结果
|
||||
返回 dict 含 `status`/`total`/`success`/`failed`/`skipped`/`duration`。
|
||||
- **`start_metrics_server` 用 stdlib `http.server`**:零依赖暴露
|
||||
Prometheus 端点(`/metrics` 路径)。返回 `stop()` 函数关闭服务器。
|
||||
生产环境可叠加反向代理或替换为 `prometheus_client`。
|
||||
- `@override` 装饰 `log_message` 静默访问日志,按 Python 版本条件导入
|
||||
(`typing` 3.12+ / `typing_extensions` <3.12)
|
||||
|
||||
## 十、API 设计通用模式
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **classmethod 委托函数式 API**:如 `Graph.from_yaml(path)` 委托
|
||||
`yaml_loader.load_yaml(path)`,保持类职责单一,函数式 API 可独立复用。
|
||||
- **frozen dataclass 表达不可变结果**:`FailureCluster`/
|
||||
`DependencyChain`/`DiagnosticReport` 等只读结果用 `@dataclass(frozen=True)`,
|
||||
自动获得 `__hash__` 与 `__eq__`,适合作为字典 key 或集合元素。
|
||||
- **列表保序而非 dict**:当顺序有意义(如执行顺序、依赖链路径)时,
|
||||
用 list 而非 dict 存储结果,便于序列化数组导出。
|
||||
- **可选参数 None 哨兵**:`run(only=None, tags=None)`,None 表示
|
||||
"不限制",与空列表 `[]` 语义区分(空列表通常表示"无匹配")。
|
||||
- **提取辅助函数控制分支数**:`run()`、CLI 命令分发等复杂函数容易
|
||||
触发 PLR0911/PLR0912。优先提取 `_dispatch_strategy`/
|
||||
`_print_diagnostics`/`_print_task_list` 等辅助函数,而非用 noqa 抑制
|
||||
或合并语义不同的分支。
|
||||
- **depends_on 自动推断(iter-26)**:`depends_on` 为空的纯 fn 任务,
|
||||
从必需参数名(无默认值、非 Context 标注、非 `*args`/`**kwargs`)
|
||||
匹配图中任务名自动推断依赖。安全规则:仅当 `depends_on` 为空时触发
|
||||
(显式声明优先);已在 `soft_depends_on` 中的参数名不重复加入硬依赖;
|
||||
cmd 任务与无 fn 任务不推断。`Graph.add()` 和 `Graph.from_specs()` 均
|
||||
调用 `_auto_infer_deps_single`,前者用图中已有任务名,后者用全部任务名。
|
||||
- **`px.graph(*specs)` 快捷构造(iter-26)**:等价于 `Graph.from_specs`
|
||||
但接受可变参数而非列表,减少 `[...]` 样板。透传 `defaults`/`namespace`。
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **公共 API 必须有完整类型注解与中文 docstring**:项目既有风格,
|
||||
`python-standards.md` 硬约束。
|
||||
- **零运行时依赖原则(YAML 例外)**:除 PyYAML(YAML 编排必需)、
|
||||
rich + typer(CLI 必需)、typing-extensions(前向兼容)、orjson(可选
|
||||
加速,`pip install pyflowx[fast]`)外,不引入运行时依赖。新增依赖须
|
||||
审慎,优先用标准库。
|
||||
|
||||
## 十一、状态后端与缓存
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **`StateBackend` 抽象基类**:`get`/`put`/`has`/`iter`/`clear` +
|
||||
`invalidate(key) -> bool`(iter-19 新增)。`invalidate` 返回是否实际
|
||||
删除,便于调用方判断缓存命中。
|
||||
- **`_TTLStateBackendMixin` 4 原语**:`_get_raw`/`_put_raw`/`_iter_raw`/
|
||||
`_clear_raw`/`_delete_raw`(iter-19 新增)。子类只需实现这 5 个原语,
|
||||
`get`/`put`/`has`/`iter`/`clear`/`invalidate` 由 mixin 统一组合 TTL 判断。
|
||||
- **`MemoryBackend` LRU 驱逐(iter-19)**:
|
||||
- 用 `OrderedDict` 替代普通 dict
|
||||
- `_get_raw` 命中时 `move_to_end(key)` 更新访问顺序
|
||||
- `_put_raw` 超限时 `popitem(last=False)` 驱逐头部(最旧)
|
||||
- `maxsize=None` 时不驱逐(默认行为,向后兼容)
|
||||
- **`SQLiteBackend` WAL 模式 + RLock**:线程安全;`batch()` 通过
|
||||
`_in_batch` 标志延迟 commit,不持锁。
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **`JSONBackend.invalidate` 覆盖**:非 batch 模式下需 `flush()` 落盘,
|
||||
否则删除只在内存中生效,进程崩溃后状态丢失。batch 模式下标记
|
||||
`_dirty=True`,由 `flush()` 统一处理。
|
||||
- **`SQLiteBackend._delete_raw` 用 `cursor.rowcount > 0`**:判断是否
|
||||
实际删除,避免先 `has` 后 `delete` 的双次查询。
|
||||
- **`maxsize` 默认 None 而非 0**:0 会立即驱逐所有项,等价于禁用缓存;
|
||||
None 表示无限制,与历史行为兼容。
|
||||
|
||||
## 十二、CLI 工具错误处理
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **`subprocess.run` 的 `check` 参数分级策略**(iter-16):
|
||||
- **失败需感知**(dockercmd/msdownload/sglang)→ `check=True` +
|
||||
`try/except subprocess.CalledProcessError` 打印 rich 错误
|
||||
- **失败可容忍但需检查**(taskkill)→ 保持 `check=False`,但检查
|
||||
`returncode` 打印结果(pkill 返回 1 表示无匹配进程,非错误)
|
||||
- **失败无关紧要**(clr/which)→ `check=False`,加注释说明原因
|
||||
- **错误路径统一处理模式**:
|
||||
```python
|
||||
try:
|
||||
subprocess.run(cmd, check=True, ...)
|
||||
except subprocess.CalledProcessError as e:
|
||||
_console.print(f"[red]命令失败 (returncode={e.returncode})[/]")
|
||||
except FileNotFoundError:
|
||||
_console.print(f"[red]命令未找到: {cmd[0]}[/]")
|
||||
```
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **rich Console 而非 print**:CLI 工具错误用 `rich.console.Console`
|
||||
打印带颜色输出,与项目其他 CLI 工具一致。
|
||||
- **`check=True` + try/except 优于 `check=False` + 手动检查**:除非
|
||||
命令返回非零是预期行为(如 taskkill pkill 无匹配、which 未找到命令),
|
||||
否则用 `check=True` 让 subprocess 抛异常,统一在 except 中处理。
|
||||
|
||||
### 踩坑总结
|
||||
|
||||
- **`@patch` 装饰器禁用**:项目 `python-standards.md` 禁用 `@patch`
|
||||
装饰器与 `mock.patch.object` 上下文。mock subprocess 异常用
|
||||
`monkeypatch.setattr(subprocess, "run", lambda *_, **__: (_ for _ in ()).throw(err))`
|
||||
—— 用生成器抛异常的技巧让 lambda 能抛出指定异常。
|
||||
- **mock 异常抛出技巧**:`lambda *_, **__: (_ for _ in ()).throw(err)`
|
||||
利用生成器仅在迭代时抛异常的特性,让无参 lambda 能抛出指定异常。
|
||||
|
||||
## 十三、测试与质量保障
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **`# pragma: no cover` 清理策略**(iter-18):按 `python-standards.md`
|
||||
"pragma: no cover 是清理信号,应激活或删除"原则,逐处分析:
|
||||
- **逻辑上不可达** → 删除(如 `if result is None` 在已过滤的列表后)
|
||||
- **防御性代码但实际可触发** → 激活:移除 pragma,写测试覆盖该分支
|
||||
- **调用方已保证的前置条件** → 保留 pragma,加注释说明依赖契约
|
||||
- **覆盖率缺口分析方法**:
|
||||
1. 跑 `pytest --cov=src/pyflowx --cov-report=term-missing`
|
||||
2. 找出 `Stmts`/`Miss` 列中 Miss > 0 的文件
|
||||
3. 看 `Missing` 列定位未覆盖行号
|
||||
4. 判断是"未触发分支"还是"不可达代码"
|
||||
5. 前者补测试,后者清 pragma 或删代码
|
||||
- **测试命名**:`test_<被测对象>_<场景>`,如 `test_json_backend_flush_type_error`。
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **公共 API 优先测试**:用 `has`/`get` 等公共接口测试,不访问私有方法。
|
||||
故障注入等场景可临时访问私有属性,docstring 注明原因。
|
||||
- **Mock 优先级**:`monkeypatch` > 内联 stub > `unittest.mock` >
|
||||
`pytest-mock`。禁用 `@patch` 装饰器、`mock.patch.object` 上下文、
|
||||
`pytest-mock` 的 `mocker` fixture。
|
||||
- **`slow` 标记**:耗时测试加 `@pytest.mark.slow`,CI 可 `-m "not slow"` 跳过。
|
||||
|
||||
### 踩坑总结
|
||||
|
||||
- **monkeypatch 目标需跟随 import 路径**:当模块从 `import json` 改为
|
||||
`from ._json import dump`,测试中 `monkeypatch.setattr(json, "dump", ...)`
|
||||
失效,需改为 `monkeypatch.setattr(storage_mod, "dump", ...)`。
|
||||
- **占位函数体无法覆盖**:`_noop_fn` 等占位函数的函数体在测试中无法触发
|
||||
(参数已校验),属可接受的覆盖率缺口,加 `# pragma: no cover` 并注释原因。
|
||||
- **不可哈希参数回退分支**:`lru_cache` 缓存的函数遇到不可哈希参数会抛
|
||||
TypeError,需 try/except 回退到慢路径。该回退分支需单独写测试(构造
|
||||
不可哈希 fn 触发)。
|
||||
|
||||
## 十四、任务编排与持久化(P9,iter-21)
|
||||
|
||||
### 任务分组(P9.1.2)
|
||||
|
||||
- **`Graph.group(name, tasks)` 逻辑分组**:`name` 不能与已注册任务名或
|
||||
已有组名冲突;`tasks` 必须是已注册任务。组名可作为其他任务的
|
||||
`depends_on` / `soft_depends_on` 引用,等价于依赖组内全部成员。
|
||||
- **统一依赖重写 `_rewrite_deps`**:`_rewrite_deps_for_loops` 改名为
|
||||
`_rewrite_deps`,合并 `_loop_groups` 与 `_groups` 字典后统一重写
|
||||
依赖。所有调用点(`_register` / `_expand_loop_to_specs` / `from_specs`)
|
||||
都改为调用新名。
|
||||
- **`from_specs` 立即校验引用**:在 `from_specs` 中引用尚未声明的组名
|
||||
会立即抛 `MissingDependencyError`。若需引用稍后声明的组,改用
|
||||
`graph.add()` 在 `graph.group()` 之后注册。
|
||||
|
||||
### 动态任务生成(P9.1.3)
|
||||
|
||||
- **`TaskSpec(dynamic=True)`**:`fn` 运行时可返回 `TaskSpec | list[TaskSpec]`,
|
||||
DependencyRunner 中途注册并调度派生任务。
|
||||
- **仅 `dependency` 策略支持**:层模型(sequential/thread/async)的层屏障
|
||||
与动态扩展语义冲突。`_dispatch_strategy` 检测 `has_dynamic` 后显式拒绝
|
||||
其他策略,抛 `ValueError`。
|
||||
- **派生任务的依赖自动补充**:若派生 spec 未显式声明对父任务的依赖,
|
||||
DependencyRunner 用 `dataclasses.replace` 自动追加父任务名到
|
||||
`depends_on`,确保派生任务在父任务之后执行。
|
||||
- **`asyncio.wait` + 手动异常传播**:动态任务场景下 `futures` 字典会随
|
||||
执行增长,必须用 while 循环 + `asyncio.wait(return_when=FIRST_COMPLETED)`
|
||||
增量调度。**陷阱**:`asyncio.wait` 不会自动 re-raise task 异常,需手动
|
||||
`f.exception()` 检查并显式 `raise`,同时 `cancel()` 未完成任务,才能
|
||||
与 `asyncio.gather` 的 fail-fast 行为一致。
|
||||
|
||||
### 数据流(P9.2)
|
||||
|
||||
- **`TaskSpec.outputs` 元数据字段**:`Mapping[str, str] | None`,仅作为
|
||||
元数据携带不参与执行。支持 `${{ matrix.* }}` 占位符替换(YAML 场景)。
|
||||
- **`RunReport.output_of(task_name, output_name)`**:按 spec.outputs 中
|
||||
声明的 path 从 `result.value` 提取命名输出。
|
||||
- path 语法:`"$"` 取整个 result;`"key"` 取 dict 键;`"a.b.c"` 嵌套取值
|
||||
- 未声明 output 或任务不在报告中抛 `KeyError`
|
||||
- **`Graph.pipeline(*specs)` 语义别名**:等价于 `chain()`,强调数据流
|
||||
上下文注入的语义场景。
|
||||
|
||||
### 检查点恢复(P9.3)
|
||||
|
||||
- **`run(resume_from=RunReport | str | Path)`**:预填充 `report.results`
|
||||
与 `context`,executor 跳过已 SUCCESS 的任务。
|
||||
- 仅恢复 `status == SUCCESS` 且名称在当前图中的任务
|
||||
- FAILED / SKIPPED 任务会重新执行
|
||||
- 接受 `RunReport` 对象、JSON 字符串路径、`Path` 对象
|
||||
- **`_apply_resume` 抽取为独立函数**:避免 `run()` 的 PLR0912 分支数
|
||||
(≤12)超限。`_load_resume_report` 处理 `RunReport | str | Path` 三种
|
||||
输入形态。
|
||||
- **`_filter_and_sort` + `_run_task` 双重跳过**:`_filter_and_sort` 在
|
||||
层模型中跳过已恢复任务;`_run_task` 在 DependencyRunner 中检查
|
||||
`name in report.results and status == SUCCESS` 直接返回。
|
||||
- **下游获取上游恢复值**:恢复的任务结果写入 `context`,下游任务通过
|
||||
上下文注入正常获取上游返回值,无需重新执行上游。
|
||||
|
||||
### RunHistory(P9.3)
|
||||
|
||||
- **文件系统历史管理**:`RunHistory(dir)` 按 `{run_id}.json` 命名存储
|
||||
`RunReport` JSON。
|
||||
- **API**:`save(report) -> Path` / `load(run_id) -> RunReport` /
|
||||
`list_runs() -> list[str]` / `latest() -> RunReport | None` /
|
||||
`delete(run_id) -> bool` / `__len__` / `__contains__`。
|
||||
- **目录不存在自动创建**:构造函数 `mkdir(parents=True, exist_ok=True)`。
|
||||
- **`latest()` 按 mtime 排序**:取最新 mtime 的 JSON 文件加载。
|
||||
- **未实现压缩/归档**:大量历史运行可能占用较多磁盘,后续可加
|
||||
`max_entries` 或滚动归档。
|
||||
|
||||
### 踩坑总结
|
||||
|
||||
- **`asyncio.wait` vs `gather` 异常传播**:从 `gather` 切换到 `wait` 后
|
||||
7 个测试失败(`TaskFailedError` 被吞)。根因是 `wait` 不自动 re-raise,
|
||||
必须手动检查 `f.exception()` 并显式 raise + cancel 剩余任务。
|
||||
- **检查点恢复测试设计**:测试 `test_resume_downstream_gets_restored_result`
|
||||
需让上游 a 第一次成功、下游 b 第一次失败;第二次恢复时只恢复 a,
|
||||
重新执行 b。若 a/b 都成功则都被恢复,b 不会重新执行,无法验证
|
||||
"下游获取上游恢复值"。
|
||||
- **`from_specs` 与 `group` 的声明顺序**:`from_specs` 立即校验依赖引用,
|
||||
组名必须在 `from_specs` 调用前已声明。否则需用 `add()` 在 `group()`
|
||||
之后注册引用组的任务。
|
||||
- **`metrics_text` 条件化输出**:最初实现总是输出 HELP/TYPE 行(即便无数据),
|
||||
导致 `reset()` 后文本仍含 `pyflowx_task_total`,违反测试预期。所有
|
||||
section 都应按数据存在性条件化,与 retries/run_total 一致。
|
||||
|
||||
## 十五、图片处理(P10,iter-22)
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **可选依赖 try/except + `_require_*` 守卫**:模块顶部
|
||||
`try: from PIL import Image, ImageDraw, ImageFont; HAS_PIL = True;
|
||||
except ImportError: HAS_PIL = False`,每个子命令入口
|
||||
`if not _require_pil(): return` 打印 `pip install pyflowx[office]`
|
||||
提示而非崩溃。与 `pdftool.py` 的 `pypdf` 守卫模式一致。
|
||||
- **`@px.tool` 装饰器注册子命令**:`@px.tool("imagetool", subcommand="r",
|
||||
help="调整尺寸")`,subcommand 短别名(r/c/ro/fl/cv/wm/cp/i/e/hi/co)
|
||||
供 CLI 快捷调用(如 `pf img r input.png out.png --width 100`)。
|
||||
- **`image_pipeline` DAG 构造器**:参照 `compose()` 函数式构造器,
|
||||
每个 step 生成 `TaskSpec`,`depends_on` 链式依赖形成单链 DAG;
|
||||
`operations` 字典映射操作名到 `imagetool.image_*` 函数,不重新实现操作。
|
||||
- **`naming` 模板占位符**:`{stem}`(累积 stem,每步更新为输出文件 stem)/
|
||||
`{step}`(操作名)/`{ext}`(当前扩展名,`convert` 步骤可能改变)。
|
||||
默认 `"{stem}_{step}{ext}"`(如 `input_resize.jpg`)。
|
||||
- **`_make_step_fn` 闭包工厂**:提取为独立函数
|
||||
`_make_step_fn(base_fn, input_path, output_path, params)` 返回 `_run`
|
||||
闭包,利用每次调用的独立作用域正确捕获每轮参数,避免循环变量捕获陷阱。
|
||||
- **`pytest.importorskip("PIL")`**:测试模块顶部跳过未安装 Pillow 的环境,
|
||||
避免在未安装 `[office]` extra 的 CI 上失败。
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **单一 `imagetool` + office extra**:不拆分为 `imageops`/`imagemeta`
|
||||
等多工具,所有图片操作集中在 `ops/files/imagetool.py`。Pillow 通过
|
||||
`pyproject.toml` 的 `[office]` extra 安装(`pillow>=10.4.0`),
|
||||
零运行时依赖原则不破坏。
|
||||
- **`_save_image` 统一保存逻辑**:所有需要保存图片的子命令委托
|
||||
`_save_image(img, output, fmt, quality)` 处理 JPG→JPEG 规范化 +
|
||||
JPEG 强制 RGB 转换 + JPEG/WEBP 应用 quality 参数,避免每个子命令
|
||||
重复处理格式与 alpha 通道问题。
|
||||
- **`image_pipeline` 委托 imagetool 函数**:不重新实现图片操作,
|
||||
`operations` 字典映射操作名到 `imagetool.image_*` 函数,保持单一职责
|
||||
与单一实现源。
|
||||
- **EXIF 用 `exif.tobytes()`**:Pillow 10+ 的 `Image.Exif` 对象
|
||||
`bytes(exif)` 抛 `ValueError`,必须用 `exif.tobytes()` 序列化。
|
||||
`_save_exif` 辅助函数封装此逻辑。
|
||||
- **PLR0912 分支数控制**:`image_exif` 函数最初 13 分支超过 12 限制,
|
||||
提取 `_print_exif`/`_apply_exif_modifications`/`_apply_single_exif_set`/
|
||||
`_save_exif` 4 个辅助函数降至合规。
|
||||
|
||||
### 踩坑总结
|
||||
|
||||
- **Pillow 10+ API 迁移**:`Image.FLIP_LEFT_RIGHT`(模块级常量)已弃用,
|
||||
pyrefly 无法解析 → 改用 `Image.Transpose.FLIP_LEFT_RIGHT`(枚举形式);
|
||||
`bytes(exif)` 抛 `ValueError: bytes must be in range(0, 256)` →
|
||||
改用 `exif.tobytes()`。
|
||||
- **`quantize(colors=N)` 实际色数可能 < N**:简单图片(如纯色)量化后
|
||||
调色板可能少于请求色数,直接按 `count` 遍历会 `IndexError`。必须用
|
||||
`actual_count = len(palette) // 3` 动态计算并 `min(count, actual_count)`。
|
||||
- **JPEG 不支持 alpha 通道**:RGBA 模式图片直接保存为 JPEG 抛
|
||||
`cannot write mode RGBA as JPEG`。`_save_image` 中 JPEG 分支强制
|
||||
`img.convert("RGB")` 去 alpha。
|
||||
- **JPG 不是 Pillow 合法格式名**:`img.save(format="JPG")` 抛
|
||||
`KeyError: 'JPG'`。`_save_image` 中 `save_fmt == "JPG"` 时改写为
|
||||
`"JPEG"` 规范化。
|
||||
- **闭包捕获循环变量陷阱**:`image_pipeline` 中若直接在 for 循环内
|
||||
定义闭包 `def _run(): base_fn(input_path, output_path, **params)`,
|
||||
所有任务的 `_run` 会捕获**最后一次循环**的变量。必须提取为独立函数
|
||||
`_make_step_fn(base_fn, input_path, output_path, params)` 利用每次
|
||||
调用的独立作用域。
|
||||
- **`draw.textbbox()` 返回 float**:`text_w = bbox[2] - bbox[0]` 是
|
||||
float,传给 `_resolve_position`(期望 int)会触发类型警告。用
|
||||
`int(bbox[2] - bbox[0])` 显式转换。
|
||||
|
||||
## 十六、文件操作底层 API(P11,iter-23)
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **纯函数模块 + 导出模块本身**:`fileops.py` 20 个函数全部纯函数
|
||||
(返回数据,不打印),不带 `@px.tool` 装饰器。`__init__.py` 用
|
||||
`from . import fileops` + `"fileops"` 加入 `__all__`,导出模块而非
|
||||
20 个函数(避免 `find`/`exists`/`size` 等通用名污染顶层命名空间)。
|
||||
用户用 `from pyflowx.fileops import find, read_text` 或
|
||||
`px.fileops.find(...)` 访问。
|
||||
- **与 CLI 工具分层**:CLI 工具(pdftool/imagetool)面向终端用户,函数
|
||||
返回 `None` 打印结果;底层 API(fileops)面向编程组合,函数返回数据
|
||||
可被 `px.task` 包装。
|
||||
- **DAG 组合调用模式**:上游任务 `TaskSpec("find", fn=..., outputs={"paths": "$"})`
|
||||
声明输出,下游任务**参数名匹配上游任务名**自动注入上游结果值。
|
||||
`report.output_of("find", "paths")` 按 `outputs` 路径提取(`"$"` 整个结果 /
|
||||
`"key"` dict 键 / `"a.b.c"` 嵌套)。
|
||||
- **`find` 的 `max_depth` 语义**:`None` 无限制 / `0` 仅 root 直接 glob /
|
||||
`1` 直接子项 / `2` 孙项。用辅助函数 `_glob_with_depth(root, pattern, max_depth)`
|
||||
递归实现,`max_depth < 0` 返回空。
|
||||
- **`walk_files` 返回生成器**:`Iterator[Path]` 惰性求值,仅返回文件不返回
|
||||
目录,不做 pattern 匹配。与 `find`(返回排序 `list[Path]`,支持 pattern)
|
||||
互补。
|
||||
- **`sha256` 流式读取**:`sha256(path, chunk_size=65536)` 分块读取,适用
|
||||
大文件。默认 64KB chunk 平衡内存与 IO。
|
||||
- **写操作返回字节数/路径**:`write_text`/`write_bytes`/`append_text`
|
||||
返回写入字节数;`copy`/`move` 返回目标 `Path`;`delete` 返回 `bool`。
|
||||
便于下游任务使用。
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **仅依赖标准库**:`pathlib` / `hashlib` / `shutil`,不引入新依赖,
|
||||
遵守项目零运行时依赖原则。
|
||||
- **EAFP 优于 LBYL**:文件不存在等错误由标准库异常表达
|
||||
(`FileNotFoundError` / `FileExistsError`),不预先检查。
|
||||
- **`copy` / `move` 的 `overwrite` 语义统一**:
|
||||
- `overwrite=False` 且 `dst` 是目录:`src` 移入/复制到 `dst / src.name`
|
||||
- `overwrite=False` 且 `dst` 是已存在文件:抛 `FileExistsError`
|
||||
- `overwrite=True` 且 `dst` 存在:**先删除 `dst`**,再把 `src` 放到
|
||||
`dst` 路径(不走 `is_dir` 分支)
|
||||
两个函数语义一致,`overwrite=True` 意为"替换目标"而非"移到内部"。
|
||||
- **父目录自动创建**:`write_text`/`write_bytes`/`append_text`/`copy`/`move`
|
||||
都用 `dst_path.parent.mkdir(parents=True, exist_ok=True)` 自动创建父目录,
|
||||
避免调用方预处理。
|
||||
|
||||
### 踩坑总结
|
||||
|
||||
- **`copy`/`move` 的 `overwrite=True` 与 `dst.is_dir()` 顺序陷阱**:
|
||||
初版先走 `dst_path.is_dir()` 分支(`dst_path = dst / src.name`),再检查
|
||||
`overwrite`。这导致 `overwrite=True` 时 `dst` 本身没被删除,而是处理
|
||||
`dst/src.name`。修复:`overwrite=True` 且 `dst` 存在时,**先删除 `dst`**,
|
||||
不走 `is_dir` 分支。
|
||||
- **DAG 数据流注入要求参数名匹配任务名**:下游任务参数名必须与上游任务名
|
||||
一致才能自动注入。如 `TaskSpec("summarize", ...)` 的下游任务参数名必须是
|
||||
`summarize`,写成 `count_and_summarize` 会抛 `InjectionError: parameter
|
||||
has no dependency`。
|
||||
- **批量复制到目录不应使用 `overwrite=True`**:`copy(p, dst_dir, overwrite=True)`
|
||||
会删除整个 `dst_dir` 目录,导致后续复制失败。批量复制应用默认
|
||||
`overwrite=False`(复制到 `dst_dir / p.name`)。
|
||||
|
||||
## 十七、通用 DAG 构造器集合(P12,iter-24)
|
||||
|
||||
### 可复用模式
|
||||
|
||||
- **函数式 DAG 构造器模式**:`pipelines.py` 提供 3 个构造器,输入函数/
|
||||
命令/数据,返回 `Graph`,无副作用。与 `imaging.image_pipeline` 同级,
|
||||
复用 `Graph.chain` / `Graph.map` 的依赖链接模式,但提供更高层次抽象。
|
||||
- **`data_pipeline` 用函数 `__name__` 作默认任务名**:下游函数参数名匹配
|
||||
前驱任务名自动注入返回值。`<lambda>` 回退为 `step`;重名追加索引
|
||||
(`extract_1`)。用户可通过 `names` 参数自定义命名,但需确保下游函数
|
||||
参数名匹配自定义名。
|
||||
- **`command_chain` 自动命名 `cmd_{i:02d}_{first_arg}`**:索引保证唯一性,
|
||||
首参数提供可读性。空 list/空字符串回退为 `cmd`。shell 字符串命令取首个
|
||||
token。支持 `cwd`/`env`/`continue_on_error` 透传。
|
||||
- **`fan_out_fan_in` 用 `Context` 标注实现 reduce 聚合**:reduce 包装函数
|
||||
标注 `ctx: Context` 接收完整依赖上下文,按 `worker_names` 顺序提取结果,
|
||||
传递给用户的 `reduce` 函数。`reduce=None` 时仅生成 fan-out(无聚合)。
|
||||
- **闭包工厂模式绑定参数**:`_make_worker_fn(worker, item)` 与
|
||||
`_make_reduce_fn(reduce, worker_names)` 参照 `imaging._make_step_fn` 模式,
|
||||
闭包捕获函数参数(非循环变量),避免 Python 闭包"延迟绑定"陷阱。
|
||||
- **`switch`/`branch` 条件分支构造器(iter-27)**:基于已有条件原语
|
||||
(`DEP_EQUALS`/`DEP_MATCHES`/`NOT`/`AND`)构建,不引入新条件类型。
|
||||
`switch(selector, cases, *, default=None)` 为每个 case 追加
|
||||
`DEP_EQUALS(selector.name, case_key)` 条件;`default` 追加
|
||||
`AND(NOT(DEP_EQUALS(..., k)) for k in cases)` 条件。
|
||||
`branch(selector, predicate, if_true, *, if_false=None)` 为 `if_true` 追加
|
||||
`DEP_MATCHES(selector.name, predicate)`,为 `if_false` 追加 `NOT(...)`。
|
||||
用 `dataclasses.replace` 在不修改原 spec 的前提下追加 `depends_on` 和
|
||||
`conditions`。
|
||||
|
||||
### 设计决策
|
||||
|
||||
- **新建 `pipelines.py` 而非扩展 `compose.py`**:`compose.py` 职责是多图
|
||||
引用解析(`GraphComposer`/`compose`),与单图 DAG 构造器正交。新建模块
|
||||
保持职责清晰:`imaging.py` 图片专用,`pipelines.py` 通用构造器。
|
||||
- **`reduce` 可选支持纯 fan-out**:`fan_out_fan_in` 的 `reduce` 参数为
|
||||
`Callable | None`。`None` 时仅生成 fan-out,适用于不需要聚合的并行批处理
|
||||
(如批量文件复制,每个 worker 独立写文件)。
|
||||
- **与 `Graph.chain`/`Graph.map` 差异化定位**:Graph 方法是方法式 API
|
||||
(需先创建 Graph + TaskSpec),本模块是函数式 API(直接传函数/命令/数据,
|
||||
返回 Graph),更高层抽象,减少样板代码。
|
||||
|
||||
### 踩坑总结
|
||||
|
||||
- **`data_pipeline` 自定义 names 后参数名必须匹配**:若用 `names=["alpha",
|
||||
"beta"]`,下游函数参数名必须是 `alpha`(不是函数 `__name__`),否则抛
|
||||
`InjectionError: parameter has no dependency`。
|
||||
- **`command_chain` + `continue_on_error` 的下游仍被 SKIPPED**:
|
||||
`continue_on_error=True` 时失败任务标记 FAILED 不抛异常,但其硬依赖下游
|
||||
仍被 SKIPPED(`reason="上游任务状态为 failed"`)。`continue_on_error` 只
|
||||
影响"是否中止整图",不影响"下游是否跳过"。`report.success` 在
|
||||
`continue_on_error=True` 时仍为 `True`(不抛异常即视为成功)。
|
||||
- **命令失败时 `px.run` 抛 `TaskFailedError` 而非返回失败 report**:
|
||||
`continue_on_error=False`(默认)时,命令失败会抛 `TaskFailedError`,
|
||||
需用 `pytest.raises(px.TaskFailedError)` 捕获测试,不能检查
|
||||
`report.success`。
|
||||
@@ -0,0 +1,135 @@
|
||||
---
|
||||
name: "pyflowx-testing"
|
||||
description: "PyFlowX 项目的测试编写规范与 mock 使用指南。在编写或审查测试、选择 mock 工具、设计 fixture、处理 asyncio 测试时调用。"
|
||||
---
|
||||
|
||||
# PyFlowX 测试规范
|
||||
|
||||
本技能是 `.trae/rules/python-standards.md` 测试章节的详细展开。
|
||||
规则文件仅保留硬约束指针,本文件提供完整操作指南。
|
||||
|
||||
## 总则
|
||||
|
||||
- **覆盖率 ≥ 95%**(branch coverage),不得下降。
|
||||
- **公共 API 优先测试**:测试用公共接口(`has`/`get`),不访问私有方法
|
||||
(如 `_expired`)。兼容旧测试的私有方法应删除并迁移测试。
|
||||
例外:`_store`/`_flush` 等内部状态在无法用公共 API 触发时(如模拟过期、
|
||||
故障注入),可临时访问私有属性,并在 docstring 注明原因。
|
||||
- **命名**:`test_<被测对象>_<场景>`,如 `test_storage_key_cache_key_exception_returns_name`。
|
||||
- **每个测试一个断言重点**;多个断言要语义相关。
|
||||
- **slow 标记**:耗时测试加 `@pytest.mark.slow`,CI 可 `-m "not slow"` 跳过。
|
||||
- **测试代码也跑 ruff**:`tests/**` 忽略 `ARG001`/`ARG002`(未用 fixture 参数)。
|
||||
- **断言风格**:用原生 `assert` + 比较运算符(`assert x == 1`),
|
||||
不用 `self.assertEqual`;pytest 会生成更清晰的 diff。
|
||||
|
||||
## Mock 工具选择(强制)
|
||||
|
||||
**优先级**:`monkeypatch` > 内联 stub > `unittest.mock` > `pytest-mock`。
|
||||
|
||||
| 场景 | 工具 | 示例 |
|
||||
|------|------|------|
|
||||
| 替换模块属性 / 环境变量 / 工作目录 | `monkeypatch` | `monkeypatch.setattr(subprocess, "run", fake_run)` |
|
||||
| `os.environ["KEY"]` 临时设置 | `monkeypatch.setenv` | `monkeypatch.setenv("LOCALAPPDATA", "C:\\...")` |
|
||||
| 切换 cwd | `monkeypatch.chdir` | `monkeypatch.chdir(tmp_path)` |
|
||||
| 一次性 stub 函数 | 内联 lambda / 闭包 | `ran = []; monkeypatch.setattr(subprocess, "run", lambda *c, **__: ran.append(c))` |
|
||||
| 复杂 spy(记录调用次数/参数/返回序列) | `unittest.mock.MagicMock` | 仅当 lambda 不足以表达时 |
|
||||
| `with patch(...)` 上下文 | **禁用**(用 monkeypatch) | monkeypatch 自动 teardown 更安全 |
|
||||
|
||||
**禁止**:
|
||||
- 不用 `pytest-mock` 的 `mocker` fixture(项目虽在 dev 依赖声明,但实际
|
||||
测试代码未使用;为保持风格统一,新代码继续用 `monkeypatch`)。
|
||||
- 不用 `unittest.mock.patch` 装饰器(`@patch("x.y")`),它隐藏依赖且
|
||||
与 pytest fixture 模式不兼容;用 `monkeypatch.setattr` 替代。
|
||||
- 不用 `mock.patch.object` 作为上下文管理器,除非被测代码本身就是
|
||||
contextmanager(此时用 `monkeypatch.setattr` 仍更简单)。
|
||||
|
||||
## monkeypatch 使用规范
|
||||
|
||||
- **类型注解**:fixture 参数标注 `monkeypatch: pytest.MonkeyPatch`。
|
||||
- **作用域**:monkeypatch 自动在测试结束时撤销,**禁止**手动
|
||||
`monkeypatch.setattr(x, "y", original)` 恢复(多余且容易遗漏)。
|
||||
例外:在单个测试内需要中途恢复时,用 `monkeypatch.undo()` 全量撤销。
|
||||
- **替换目标**:替换"被测代码看到的对象",而非全局对象本身。
|
||||
- 错误:`monkeypatch.setattr("os.path.exists", fake)` —— 替换全局,影响其他模块。
|
||||
- 正确:`monkeypatch.setattr(pyflowx.command.shutil, "which", fake)` ——
|
||||
替换被测模块引用的 `shutil.which`。
|
||||
- **属性 vs 字符串路径**:优先属性访问形式 `monkeypatch.setattr(obj, "attr", val)`
|
||||
而非字符串路径 `monkeypatch.setattr("pkg.mod.obj.attr", val)`,
|
||||
前者有 IDE 跳转与重构支持。
|
||||
- **记录调用**:用闭包 `ran: list[tuple] = []` + `lambda *a, **k: ran.append((a, k))`
|
||||
替代 `MagicMock`,可读性更好且无需导入。
|
||||
|
||||
## Stub 与 Spy 模式
|
||||
|
||||
- **轻量 stub**:内联定义 `class MockResult: returncode = 0; stdout = ""`,
|
||||
替代 `MagicMock(return_value=...)`,类型明确且不引入 mock 依赖。
|
||||
- **状态收集**:闭包 + list 比 `mock.call_args_list` 更易断言:
|
||||
```python
|
||||
calls: list[list[str]] = []
|
||||
|
||||
|
||||
def fake_run(cmd: list[str], **_: Any) -> MockResult:
|
||||
calls.append(cmd)
|
||||
return MockResult()
|
||||
|
||||
|
||||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||||
assert calls == [["clear"]]
|
||||
```
|
||||
- **副作用序列**:需要按调用次数返回不同值时,用 `itertools.cycle` 或
|
||||
手动计数器,而非 `side_effect=[...]`(mock 专有 API)。
|
||||
- **异常注入**:`def raise_oserror(*a, **k): raise OSError("...")`,
|
||||
用 `pytest.raises(OSError)` 验证,而非 `side_effect=OSError`。
|
||||
|
||||
## 异常断言
|
||||
|
||||
- **`pytest.raises`**:必填 `match=` 正则(除非异常消息完全不可预测),
|
||||
避免误捕获同类异常:
|
||||
```python
|
||||
with pytest.raises(StorageError, match="cannot write"):
|
||||
b.save("a", 1)
|
||||
```
|
||||
- **异常链**:验证 `__cause__` 时用 `exc_info.value.__cause__`,
|
||||
确认 `raise X from Y` 因果链完整。
|
||||
- **禁止** `try/except + assert False`:用 `pytest.raises` 替代。
|
||||
|
||||
## Fixture 规范
|
||||
|
||||
- **`tmp_path`**:处理临时文件,自动清理,禁止 `tempfile.mkdtemp()` 手动管理。
|
||||
- **`monkeypatch`**:环境变量、cwd、模块属性 mock(见上)。
|
||||
- **`capsys`/`capfd`**:捕获 stdout/stderr,验证日志或命令输出。
|
||||
- **autouse fixture**:仅在全局必需时用(如 `conftest.py` 的
|
||||
`packtool_tmp_workdir` 自动切到 tmp_path);否则显式声明参数。
|
||||
- **fixture 命名**:`snake_case`,描述"提供什么"而非"测试什么"
|
||||
(`sample_graph` 优于 `test_data`)。
|
||||
- **fixture 作用域**:默认 `function`;`module`/`session` 仅当构造昂贵且
|
||||
只读时,并加注释说明无副作用。
|
||||
|
||||
## asyncio 测试
|
||||
|
||||
- **fixture `loop_scope="function"`**(pyproject 已配置默认值)。
|
||||
- **async 测试**:`async def test_x():`,pytest-asyncio 自动驱动。
|
||||
- **await 检查**:测试异步函数必须 `await` 结果,禁止仅验证返回 coroutine 对象。
|
||||
- **异步 mock**:用 `AsyncMock`(3.8+ 在 `unittest.mock`)或
|
||||
`async def fake(): return value`,禁用 `MagicMock(return_value=coro)`。
|
||||
|
||||
## 参数化
|
||||
|
||||
- **`@pytest.mark.parametrize`**:用 `ids` 参数提供可读标识:
|
||||
```python
|
||||
@pytest.mark.parametrize(
|
||||
("strategy", "expected_workers"),
|
||||
[("sequential", 1), ("thread", 8), ("async", 1)],
|
||||
ids=["seq", "thread-8", "async"],
|
||||
)
|
||||
```
|
||||
- **参数命名**:参数元组用有意义名称,而非 `("a", "b")`。
|
||||
- **组合爆炸**:参数组合 > 20 时拆分测试,避免单个测试函数臃肿。
|
||||
|
||||
## 测试组织
|
||||
|
||||
- **文件命名**:`test_<被测模块>.py`(`test_storage.py` 对应 `storage.py`)。
|
||||
- **类分组**:仅在测试逻辑强相关时用 `class TestXxx:` 分组;默认用模块级函数。
|
||||
- **docstring**:每个测试函数一句话说明"测试什么场景",复杂场景补充"为什么"。
|
||||
- **setup/teardown**:优先 fixture;`setup_method`/`teardown_method` 仅在
|
||||
无法用 fixture 表达时(罕见)。
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
# 使用国内镜像源拉取基础镜像
|
||||
# 备选镜像源前缀:docker.1ms.run / dockerpull.com / docker.xuanyuan.me
|
||||
FROM docker.m.daocloud.io/python:3.13-slim
|
||||
|
||||
# 国内镜像源(清华)
|
||||
ENV PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_TRUSTED_HOST=pypi.tuna.tsinghua.edu.cn
|
||||
ENV UV_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV UV_TRUSTED_HOST=pypi.tuna.tsinghua.edu.cn
|
||||
|
||||
# 环境变量:非交互 + 路径配置
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
LANG=C.UTF-8 \
|
||||
LC_ALL=C.UTF-8 \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_CACHE_DIR=/uv-cache \
|
||||
UV_PYTHON_INSTALL_DIR=/uv-python \
|
||||
UV_PROJECT_ENVIRONMENT=/opt/venv \
|
||||
PATH="/opt/venv/bin:${PATH}"
|
||||
|
||||
# 配置 apt 国内镜像(阿里云)并安装系统依赖
|
||||
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git \
|
||||
jq \
|
||||
build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 配置 pip 国内镜像(阿里云)
|
||||
RUN mkdir -p /etc/pip \
|
||||
&& printf '[global]\nindex-url = https://mirrors.aliyun.com/pypi/simple/\ntrusted-host = mirrors.aliyun.com\n' \
|
||||
> /etc/pip/pip.conf \
|
||||
&& mkdir -p /root/.config/pip \
|
||||
&& ln -sf /etc/pip/pip.conf /root/.config/pip/pip.conf
|
||||
|
||||
# 安装 uv 并预装 Python 3.8 / 3.13
|
||||
RUN pip install --no-cache-dir uv -i https://mirrors.aliyun.com/pypi/simple/ \
|
||||
&& uv python install 3.8 3.13
|
||||
|
||||
# 安装 Node.js 20.x(actions/checkout 需要)
|
||||
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
|
||||
apt-get install -y nodejs && \
|
||||
node --version
|
||||
|
||||
# 预装项目 dev 依赖(仅复制依赖描述文件,利用 Docker 层缓存)
|
||||
WORKDIR /workspace
|
||||
COPY pyproject.toml tox.ini README.md ./
|
||||
COPY src/ ./src/
|
||||
|
||||
# 同步依赖到 /opt/venv(CI 时直接复用)
|
||||
RUN uv sync --frozen --no-install-project 2>/dev/null || uv sync --no-install-project
|
||||
|
||||
# 预装 tox 环境(py38 + py313)
|
||||
RUN uvx tox run -e py38,py313 --notest 2>/dev/null || true
|
||||
|
||||
# 持久化 uv 缓存目录(CI 可挂载到宿主机加速)
|
||||
VOLUME ["/uv-cache"]
|
||||
|
||||
# 默认入口
|
||||
CMD ["/bin/bash"]
|
||||
@@ -5,7 +5,8 @@
|
||||
[](https://github.com/gookeryoung/pyflowx/actions/workflows/ci.yml)
|
||||
[](https://pypi.org/project/pyflowx/)
|
||||
[](https://pypi.org/project/pyflowx/)
|
||||
[](https://github.com/gookeryoung/pyflowx)
|
||||
[](https://pyflowx.readthedocs.io/zh/latest/)
|
||||
[](https://github.com/gookeryoung/pyflowx)
|
||||
[](https://github.com/gookeryoung/pyflowx/blob/main/LICENSE)
|
||||
|
||||
PyFlowX 把"任务依赖"这件事做到极致简单:**参数名就是依赖声明**。无需装饰器、
|
||||
@@ -14,18 +15,37 @@ PyFlowX 把"任务依赖"这件事做到极致简单:**参数名就是依赖
|
||||
## 特性
|
||||
|
||||
- **零样板** —— 参数名即依赖,框架自动注入上游结果
|
||||
- **三种执行策略** —— `sequential`(调试)/ `thread`(I/O 密集同步)/ `async`(I/O 密集异步)
|
||||
- **四种执行策略** —— `sequential`(调试)/ `thread`(I/O 密集同步)/ `async`(I/O 密集异步)/ `dependency`(依赖驱动,最大化并行)
|
||||
- **类型安全** —— `TaskSpec[T]` 把返回类型一路传到 `RunReport`,mypy strict 通过
|
||||
- **DAG 校验** —— 构建时即时校验重名、缺失依赖、环
|
||||
- **自动分层** —— Kahn 算法分组,同层任务可并行
|
||||
- **重试与超时** —— 每个任务独立配置 `retries` 与 `timeout`
|
||||
- **断点续跑** —— `MemoryBackend` / `JSONBackend`,成功结果可缓存复用
|
||||
- **重试与超时** —— 每个任务独立配置 `RetryPolicy`(max_attempts/delay/backoff/jitter/retry_on)与 `timeout`
|
||||
- **软依赖** —— `soft_depends_on` 仅用于上下文注入,不参与拓扑分层
|
||||
- **并发限制** —— `concurrency_key` + `concurrency_limits` 按组限流
|
||||
- **任务钩子** —— `TaskHooks`(pre_run/post_run/on_failure)生命周期回调
|
||||
- **断点续跑** —— `MemoryBackend` / `JSONBackend`,成功结果可缓存复用;`batch()` 批量落盘;`invalidate(key)` 单键失效
|
||||
- **缓存键** —— `cache_key` 函数基于输入计算稳定键,使不同输入产生独立缓存
|
||||
- **缓存驱逐** —— `MemoryBackend(maxsize=N)` LRU 驱逐最旧条目;TTL 过期自动忽略
|
||||
- **命令任务** —— `cmd` 参数直接执行外部命令,支持列表/shell/可调用对象
|
||||
- **条件执行** —— `conditions` 参数按平台、环境变量、应用安装等条件跳过任务
|
||||
- **图组合** —— `compose` / `GraphComposer` 编程式展开多图字符串引用
|
||||
- **任务模板** —— `task_template` 工厂批量生成相似 TaskSpec
|
||||
- **图级默认值** —— `GraphDefaults` 统一配置 retry/timeout/concurrency 等
|
||||
- **CLI 运行器** —— `CliRunner` 把多个图映射为命令行子命令,替代 Makefile
|
||||
- **可观测** —— `on_event` 回调、`dry_run` 预览、`verbose` 生命周期日志、Mermaid 可视化
|
||||
- **零运行时依赖** —— 仅依赖标准库(3.8 需 `graphlib_backport`)
|
||||
- **95% 测试覆盖** —— 分支覆盖率>= 95%
|
||||
- **可观测** —— `on_event` 回调(RUNNING/SUCCESS/FAILED/SKIPPED)、`dry_run` 预览、`verbose` 生命周期日志、Mermaid 可视化
|
||||
- **进度监控** —— `run(progress=True)` 开箱即用的 rich 进度条;`ProgressCallback` 协议供程序化集成
|
||||
- **任务通知** —— `Notifier` 协议支持飞书/钉钉/企业微信/Slack/Discord/Telegram webhook 与 Python 回调,全生命周期事件可配置
|
||||
- **流式获取** —— `run_iter()` 生成器逐个 yield 任务结果,适用于大量任务逐个处理
|
||||
- **任务取消** —— `CancelToken` 线程安全取消令牌,支持外部取消与 `KeyboardInterrupt` 优雅停止
|
||||
- **子图执行** —— `run(only=[...], tags=[...])` 按名称或标签运行图的子集,自动包含传递依赖
|
||||
- **结果序列化** —— `RunReport.to_json()` / `to_dict()` / `to_csv()` / `to_html()` 多格式导出运行结果,`from_json()` 支持反序列化重建;HTML 报告自包含内联 CSS,适合浏览器直接打开或邮件分享
|
||||
- **CLI 可视化** —— `pf graph <yaml>` 终端 DAG 渲染(ASCII 分层树按标签着色 / Mermaid / 依赖表),`pf yamlrun --progress` rich 进度条,`pf info [tool]` 工具详情,`pf completion bash|zsh|fish` shell 补全
|
||||
- **失败诊断** —— `RunReport.diagnose()` 自动分析根因、依赖链回溯、相似失败聚类与可操作建议;CLI 失败时自动打印诊断摘要
|
||||
- **可观测性** —— `run_id` 贯穿日志/序列化/诊断;结构化日志 `extra` 含 `task_name`/`status`/`attempts`/`error_type`;`profile(graph)` 离线性能分析(关键路径/并行度/瓶颈)
|
||||
- **状态后端** —— `MemoryBackend` / `JSONBackend` / `SQLiteBackend`(WAL 模式,适合大规模任务)
|
||||
- **YAML 任务编排** —— GitHub Actions 风格 `jobs`/`needs`/`strategy.matrix`/`if` 条件,`pf yamlrun pipeline.yaml` 一键执行
|
||||
- **最小依赖** —— `rich` + `typer` + `typing-extensions`(3.13 以下)+ `pyyaml`;安装 `orjson` 可加速 JSON 序列化(`pip install pyflowx[fast]`)
|
||||
- **97% 测试覆盖** —— 分支覆盖率 >= 95%
|
||||
|
||||
## 安装
|
||||
|
||||
@@ -67,23 +87,31 @@ print(report["double"]) # [2, 4, 6]
|
||||
|
||||
### TaskSpec —— 任务描述
|
||||
|
||||
`TaskSpec` 是不可变的任务描述符,是唯一需要配置的东西:
|
||||
`TaskSpec` 是不可变的任务描述符(`Generic[T]`,返回类型一路传到 `RunReport`),是唯一需要配置的东西:
|
||||
|
||||
```python
|
||||
px.TaskSpec(
|
||||
name="fetch_user", # 唯一标识
|
||||
fn=fetch_user, # 同步或异步函数
|
||||
cmd=["curl", "..."], # 或: 执行命令(覆盖 fn)
|
||||
depends_on=("auth",), # 依赖的任务名
|
||||
depends_on=("auth",), # 硬依赖(参与拓扑分层)
|
||||
soft_depends_on=("cache",), # 软依赖(仅注入,不参与分层)
|
||||
args=(uid,), # 静态位置参数(追加在注入参数后)
|
||||
kwargs={"timeout": 30}, # 静态关键字参数
|
||||
retries=3, # 失败重试次数(0 = 仅一次)
|
||||
retry=px.RetryPolicy(max_attempts=3, delay=1.0, backoff=2.0), # 重试策略
|
||||
timeout=30.0, # 超时秒数(None = 不限制)
|
||||
tags=("api", "user"), # 自由标签,用于子图过滤
|
||||
conditions=(is_prod,), # 条件函数列表(全部为 True 才执行)
|
||||
priority=10, # 同层内优先级(高优先执行,默认 0)
|
||||
concurrency_key="db", # 并发分组键(配合 concurrency_limits 限流)
|
||||
cache_key=lambda ctx: str(ctx.get("uid")), # 缓存键函数(不同输入独立缓存)
|
||||
hooks=px.TaskHooks(pre_run=..., post_run=..., on_failure=...), # 生命周期钩子
|
||||
cwd=Path("/tmp"), # 命令工作目录(仅 cmd 模式)
|
||||
env={"DEBUG": "1"}, # 环境变量覆盖(fn 与 cmd 模式均生效)
|
||||
verbose=True, # 打印命令输出(仅 cmd 模式)
|
||||
skip_if_missing=True, # 命令不存在时自动跳过(仅 list[str] cmd)
|
||||
allow_upstream_skip=False, # 上游 SKIPPED/FAILED 时是否仍执行
|
||||
continue_on_error=False, # 本任务失败是否不中断整体
|
||||
)
|
||||
```
|
||||
|
||||
@@ -97,18 +125,134 @@ px.TaskSpec(
|
||||
### Graph —— DAG 构建
|
||||
|
||||
```python
|
||||
graph = px.Graph.from_specs([...]) # 整批校验(推荐)
|
||||
# 图级默认值:TaskSpec 字段为 None 时回退
|
||||
defaults = px.GraphDefaults(retry=px.RetryPolicy(max_attempts=2), timeout=60.0)
|
||||
|
||||
graph = px.Graph.from_specs([...], defaults=defaults) # 整批校验(推荐)
|
||||
# 或增量构建
|
||||
graph = px.Graph()
|
||||
graph = px.Graph(defaults=defaults)
|
||||
graph.add(px.TaskSpec("a", fn_a))
|
||||
graph.add(px.TaskSpec("b", fn_b, ("a",)))
|
||||
|
||||
graph.validate() # 显式校验(环检测)
|
||||
graph.layers() # 拓扑分层
|
||||
graph.layers() # 拓扑分层(run() 入口已统一校验,直接调用需自行先 validate)
|
||||
graph.to_mermaid() # Mermaid 可视化
|
||||
graph.describe() # 人类可读摘要
|
||||
graph.subgraph(("api",)) # 按标签切片
|
||||
graph.subgraph_by_names(("a", "b")) # 按名称切片
|
||||
graph.map("fetch", [1, 2, 3], lambda i: TaskSpec(f"fetch_{i}", ...)) # 批量 fan-out
|
||||
```
|
||||
|
||||
### 图组合 —— compose
|
||||
|
||||
`compose` / `GraphComposer` 把带字符串引用的多个图展开为纯 `Graph`:
|
||||
|
||||
```python
|
||||
graphs = {
|
||||
"build": px.Graph.from_specs([px.TaskSpec("b", cmd=["echo", "b"])]),
|
||||
"all": px.Graph.from_specs(["build", px.TaskSpec("t", cmd=["echo", "t"])]),
|
||||
}
|
||||
resolved = px.compose(graphs) # "all" 图中的 "build" 引用被展开
|
||||
```
|
||||
|
||||
引用格式:`"command_name"`(整个图)或 `"command_name.task_name"`(特定任务)。
|
||||
`CliRunner` 内部自动调用 `compose`。
|
||||
|
||||
### YAML 任务编排
|
||||
|
||||
GitHub Actions 风格的 YAML 文件直接加载为 `Graph`,支持 `jobs`/`needs`/`strategy.matrix`/`if`/`continue-on-error`/`env`/`defaults`,以及 `matrix.include`/`exclude` 过滤、条件依赖(`needs: [{job: ..., if: ...}]`)、`outputs` 声明:
|
||||
|
||||
```yaml
|
||||
# pipeline.yaml
|
||||
strategy: thread
|
||||
defaults:
|
||||
retry: {max_attempts: 3}
|
||||
env: {CI: "true"}
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
cmd: ["echo", "setup"]
|
||||
runs-on: linux
|
||||
|
||||
build:
|
||||
needs: [setup]
|
||||
cmd: ["echo", "build-${{ matrix.version }}"]
|
||||
strategy:
|
||||
matrix:
|
||||
version: ["3.10", "3.11", "3.12"]
|
||||
os: ["linux", "macos"]
|
||||
exclude:
|
||||
- version: "3.10"
|
||||
os: "macos"
|
||||
include:
|
||||
- version: "3.13"
|
||||
os: "linux"
|
||||
if: "env.CI"
|
||||
outputs:
|
||||
version: "${{ matrix.version }}"
|
||||
|
||||
deploy:
|
||||
needs:
|
||||
- job: build
|
||||
if: "env.BRANCH == 'main'"
|
||||
cmd: ["echo", "deploy"]
|
||||
|
||||
test:
|
||||
needs: [build]
|
||||
cmd: ["echo", "test"]
|
||||
continue-on-error: true
|
||||
```
|
||||
|
||||
```python
|
||||
import pyflowx as px
|
||||
|
||||
graph = px.Graph.from_yaml("pipeline.yaml")
|
||||
# 或从字符串解析:graph = px.parse_yaml_string("...")
|
||||
report = px.run(graph, strategy="thread")
|
||||
```
|
||||
|
||||
CLI 一键执行:
|
||||
|
||||
```bash
|
||||
pf yamlrun pipeline.yaml # 执行
|
||||
pf yamlrun pipeline.yaml --dry-run # 仅预览,不执行
|
||||
pf yamlrun pipeline.yaml --list # 列出所有任务
|
||||
pf yamlrun pipeline.yaml --quiet # 静默模式
|
||||
pf yamlrun pipeline.yaml --strategy sequential # 覆盖策略
|
||||
```
|
||||
|
||||
**字段映射**:
|
||||
|
||||
| YAML 字段 | TaskSpec 字段 | 说明 |
|
||||
|-----------|---------------|------|
|
||||
| `cmd` / `run` | `cmd` | `cmd` 为列表,`run` 为 shell 字符串 |
|
||||
| `needs` | `depends_on` | 矩阵 job 依赖自动展开为所有变体 |
|
||||
| `if` | `conditions` | `success()`/`always()`/`env.VAR`/`env.VAR == 'x'`/`env.VAR != 'x'` |
|
||||
| `strategy.matrix` | 矩阵扇出 | 笛卡尔积,命名 `{job}_{key}-{value}` |
|
||||
| `strategy.matrix.include` | 矩阵追加 | 额外组合(可引入新键) |
|
||||
| `strategy.matrix.exclude` | 矩阵剔除 | 匹配所有 key-value 的组合被移除 |
|
||||
| `${{ matrix.key }}` | 占位符 | `cmd`/`run`/`cwd`/`env`/`outputs` 中替换 |
|
||||
| `outputs` | `outputs` | 任务声明的输出键值映射(元数据,不参与执行) |
|
||||
| `needs: [{job: ..., if: ...}]` | 条件依赖 | `if` 不满足时跳过该依赖 |
|
||||
| `continue-on-error` | `continue_on_error` | 字段名支持 hyphen/underscore |
|
||||
| `runs-on` | `tags` | 追加到 tags 元组 |
|
||||
| `defaults` | `GraphDefaults` | 图级默认值(retry/timeout/env/cwd 等) |
|
||||
|
||||
### 任务模板 —— task_template
|
||||
|
||||
`task_template` 工厂批量生成相似 TaskSpec:
|
||||
|
||||
```python
|
||||
fetch = px.task_template(
|
||||
fn=fetch_url,
|
||||
retry=px.RetryPolicy(max_attempts=5),
|
||||
timeout=30.0,
|
||||
tags=("api",),
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
fetch("users", url="https://api.example.com/users"),
|
||||
fetch("posts", url="https://api.example.com/posts"),
|
||||
])
|
||||
```
|
||||
|
||||
### run —— 执行
|
||||
@@ -116,12 +260,14 @@ graph.subgraph_by_names(("a", "b")) # 按名称切片
|
||||
```python
|
||||
report = px.run(
|
||||
graph,
|
||||
strategy="async", # sequential | thread | async
|
||||
strategy="async", # sequential | thread | async | dependency
|
||||
max_workers=8, # thread 策略的线程池大小
|
||||
concurrency_limits={"db": 2}, # 按 concurrency_key 限流
|
||||
dry_run=False, # True = 仅打印计划
|
||||
verbose=False, # True = 打印任务生命周期日志
|
||||
on_event=callback, # 状态转换回调
|
||||
on_event=callback, # 状态转换回调(RUNNING/SUCCESS/FAILED/SKIPPED)
|
||||
state=px.JSONBackend("state.json"), # 断点续跑后端
|
||||
continue_on_error=False, # True = 单任务失败不中断整体
|
||||
)
|
||||
```
|
||||
|
||||
@@ -141,7 +287,7 @@ report.describe() # 人类可读报告
|
||||
按顺序求值:
|
||||
|
||||
1. **标注为 `Context`** 的参数 → 接收完整上游结果映射
|
||||
2. **名称匹配依赖** 的参数 → 接收该依赖的结果
|
||||
2. **名称匹配依赖** 的参数 → 接收该依赖的结果(含软依赖,缺失时注入默认值)
|
||||
3. **`**kwargs`** 参数 → 接收所有依赖结果(dict)
|
||||
4. **`TaskSpec.args` / `kwargs`** → 为非依赖参数提供静态值
|
||||
|
||||
@@ -170,8 +316,11 @@ def fetch_user(uid: int) -> dict: # uid 来自 TaskSpec.args
|
||||
| `sequential` | 串行 | 调试、CPU 密集 | 直接调用 | 事件循环 |
|
||||
| `thread` | 线程池 | I/O 密集同步 | 线程池 | 不支持 |
|
||||
| `async` | 事件循环 | I/O 密集异步 | 卸载到线程池 | 事件循环 |
|
||||
| `dependency` | 依赖驱动 | 最大化并行度 | 卸载到线程池 | 事件循环 |
|
||||
|
||||
所有策略都遵循 `retries`、`timeout`、上下文注入、状态后端,并发出 `TaskEvent`。
|
||||
所有策略都遵循 `RetryPolicy`、`timeout`、上下文注入、状态后端、`concurrency_limits`,
|
||||
并发出 `TaskEvent`(RUNNING/SUCCESS/FAILED/SKIPPED)。`dependency` 策略无层屏障:
|
||||
任务在其所有硬依赖完成后立即启动。
|
||||
|
||||
## 命令任务
|
||||
|
||||
@@ -244,43 +393,56 @@ runner.run_cli() # 解析 sys.argv 并执行
|
||||
命令行用法:
|
||||
|
||||
```bash
|
||||
python build.py clean # 执行 clean 图
|
||||
python build.py build --strategy thread # 覆盖执行策略
|
||||
python build.py test --dry-run # 仅打印执行计划
|
||||
python build.py --list # 列出所有命令
|
||||
python build.py --quiet # 静默模式
|
||||
pf pymake clean # 执行 clean 图
|
||||
pf pymake build --strategy thread # 覆盖执行策略
|
||||
pf pymake test --dry-run # 仅打印执行计划
|
||||
pf pymake --list # 列出所有命令
|
||||
pf pymake --quiet # 静默模式
|
||||
```
|
||||
|
||||
`verbose=True`(默认)时打印任务生命周期(开始/成功/失败/跳过)与命令输出;`--quiet` 关闭。
|
||||
|
||||
## 示例
|
||||
|
||||
仓库 `examples/` 目录包含完整示例:
|
||||
|
||||
- [`etl_pipeline.py`](examples/etl_pipeline.py) —— ETL 流水线(sequential)
|
||||
- [`parallel_run.py`](examples/parallel_run.py) —— 并行执行对比(thread vs sequential)
|
||||
- [`async_aggregation.py`](examples/async_aggregation.py) —— 异步聚合 + Context 注入
|
||||
|
||||
运行:
|
||||
|
||||
```bash
|
||||
python examples/etl_pipeline.py
|
||||
python examples/parallel_run.py
|
||||
python examples/async_aggregation.py
|
||||
```
|
||||
|
||||
## 断点续跑
|
||||
|
||||
```python
|
||||
from pyflowx import JSONBackend
|
||||
|
||||
# 第一次运行:成功结果写入 state.json
|
||||
backend = JSONBackend("state.json")
|
||||
backend = JSONBackend("state.json", ttl=3600) # ttl 秒数,过期条目自动忽略
|
||||
report = px.run(graph, strategy="sequential", state=backend)
|
||||
|
||||
# 第二次运行:已缓存任务自动跳过
|
||||
# 第二次运行:已缓存任务自动跳过(状态为 SKIPPED)
|
||||
report = px.run(graph, strategy="sequential", state=backend)
|
||||
# report.results 中缓存任务状态为 SKIPPED
|
||||
```
|
||||
|
||||
`run()` 内部以 `backend.batch()` 包裹整个执行:所有 `save` 延迟到运行结束时统一落盘一次
|
||||
(`JSONBackend` 从 O(N²) 降为 O(N) 磁盘写入;`MemoryBackend` 为 no-op)。
|
||||
|
||||
**缓存键**:默认存储键为任务名。配置 `cache_key` 函数后,键为 `"name:cache_key_value"`,
|
||||
使不同输入产生独立缓存条目:
|
||||
|
||||
```python
|
||||
px.TaskSpec(
|
||||
"fetch_user",
|
||||
fn=fetch_user,
|
||||
cache_key=lambda ctx: str(ctx.get("uid")), # 不同 uid 独立缓存
|
||||
)
|
||||
```
|
||||
|
||||
**LRU 驱逐**:`MemoryBackend(maxsize=N)` 限制最大条目数,超限时按 LRU(最近最少使用)
|
||||
策略驱逐最旧条目。`get`/`has` 命中会将条目移到最近使用位置:
|
||||
|
||||
```python
|
||||
from pyflowx import MemoryBackend
|
||||
|
||||
backend = MemoryBackend(maxsize=100) # 最多 100 条,超出驱逐最旧
|
||||
```
|
||||
|
||||
**手动失效**:`invalidate(key)` 删除单个键的缓存,`clear()` 清除全部:
|
||||
|
||||
```python
|
||||
backend.invalidate("fetch_user") # 删除单个键,返回是否已删除
|
||||
backend.clear() # 清除全部
|
||||
```
|
||||
|
||||
## 错误处理
|
||||
@@ -312,7 +474,7 @@ except px.PyFlowXError:
|
||||
| 特性 | PyFlowX | Airflow | Prefect | Dask |
|
||||
|------|---------|---------|---------|------|
|
||||
| 零样板 | 参数名即依赖 | 装饰器 + XCom | 装饰器 | 装饰器 |
|
||||
| 运行时依赖 | 仅标准库 | 重型 | 中型 | 中型 |
|
||||
| 运行时依赖 | rich + typer | 重型 | 中型 | 中型 |
|
||||
| 类型安全 | mypy strict | 弱 | 中 | 中 |
|
||||
| 异步原生 | 是 | 否 | 部分 | 否 |
|
||||
| 断点续跑 | 内置 | 需配置 | 需配置 | 需配置 |
|
||||
@@ -321,23 +483,339 @@ except px.PyFlowXError:
|
||||
|
||||
PyFlowX 专注于**单机 DAG 调度**的极致简洁,适合 ETL、数据处理、CI 流水线等场景。
|
||||
|
||||
## 高级特性
|
||||
|
||||
### 并发限制
|
||||
|
||||
按 `concurrency_key` 分组限流,避免压垮下游资源:
|
||||
|
||||
```python
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("q1", fn=query_db, concurrency_key="db"),
|
||||
px.TaskSpec("q2", fn=query_db, concurrency_key="db"),
|
||||
px.TaskSpec("q3", fn=query_db, concurrency_key="db"),
|
||||
])
|
||||
# 同一时刻最多 2 个 "db" 组任务运行
|
||||
px.run(graph, strategy="async", concurrency_limits={"db": 2})
|
||||
```
|
||||
|
||||
### 任务钩子
|
||||
|
||||
`TaskHooks` 在任务生命周期触发(异常仅记录,不影响任务状态):
|
||||
|
||||
```python
|
||||
hooks = px.TaskHooks(
|
||||
pre_run=lambda spec: print(f"start {spec.name}"),
|
||||
post_run=lambda spec, value: print(f"done {spec.name}"),
|
||||
on_failure=lambda spec, exc: alert(spec.name, exc),
|
||||
)
|
||||
px.TaskSpec("task", fn=work, hooks=hooks)
|
||||
```
|
||||
|
||||
### 优先级
|
||||
|
||||
同层内按 `priority` 降序执行(稳定排序):
|
||||
|
||||
```python
|
||||
px.TaskSpec("low", fn=work, priority=0)
|
||||
px.TaskSpec("high", fn=work, priority=10) # 同层内先执行
|
||||
```
|
||||
|
||||
### 任务取消
|
||||
|
||||
`CancelToken` 提供线程安全的取消信号,传入 `run(cancel_event=...)` 即可外部取消正在执行的图。
|
||||
`KeyboardInterrupt`(Ctrl+C)也会触发相同的优雅取消流程:
|
||||
|
||||
```python
|
||||
token = px.CancelToken()
|
||||
|
||||
# 在另一个线程中触发取消
|
||||
import threading
|
||||
|
||||
|
||||
def cancel_later():
|
||||
import time
|
||||
|
||||
time.sleep(2)
|
||||
token.cancel("超时取消")
|
||||
|
||||
|
||||
threading.Thread(target=cancel_later, daemon=True).start()
|
||||
|
||||
# 取消后:待运行任务标记 SKIPPED,运行中任务等待完成,report.success = False
|
||||
report = px.run(graph, strategy="dependency", cancel_event=token)
|
||||
for name, result in report.results.items():
|
||||
print(f"{name}: {result.status.value}")
|
||||
```
|
||||
|
||||
也支持标准 `threading.Event` 作为取消信号:
|
||||
|
||||
```python
|
||||
event = threading.Event()
|
||||
px.run(graph, cancel_event=event)
|
||||
```
|
||||
|
||||
### 子图执行
|
||||
|
||||
`run(only=...)` / `run(tags=...)` 只运行指定任务及其传递依赖,适用于调试单个任务或增量执行:
|
||||
|
||||
```python
|
||||
# 只运行 "report" 任务及其所有上游依赖
|
||||
report = px.run(graph, only=["report"])
|
||||
|
||||
# 按标签过滤:只运行带 "ingest" 标签的任务及其依赖
|
||||
report = px.run(graph, tags=["ingest"])
|
||||
|
||||
# 两者可组合(取并集)
|
||||
report = px.run(graph, only=["report"], tags=["ingest"])
|
||||
```
|
||||
|
||||
CLI 也支持 `--only` / `--tags`:
|
||||
|
||||
```bash
|
||||
pf yamlrun pipeline.yaml --only report,deploy
|
||||
pf yamlrun pipeline.yaml --tags ingest,report
|
||||
```
|
||||
|
||||
### 结果序列化
|
||||
|
||||
`RunReport` 支持将运行结果导出为 JSON / dict / CSV,便于持久化、跨进程传输或对接外部工具:
|
||||
|
||||
```python
|
||||
report = px.run(graph, strategy="sequential")
|
||||
|
||||
# JSON 字符串(含 success/summary/results)
|
||||
text = report.to_json(indent=2)
|
||||
|
||||
# JSON 兼容字典
|
||||
data = report.to_dict()
|
||||
|
||||
# CSV 表格导出(不含 value 列以避免特殊字符干扰)
|
||||
csv_text = report.to_csv(include_value=False)
|
||||
```
|
||||
|
||||
任务返回值可能是任意 Python 对象(不一定 JSON 可序列化)。默认策略:
|
||||
可序列化原样保留,不可序列化回退到 `repr(value)`;调用方可通过
|
||||
`value_serializer` 自定义:
|
||||
|
||||
```python
|
||||
# 自定义值序列化(如把 Path 转为字符串)
|
||||
text = report.to_json(value_serializer=str)
|
||||
```
|
||||
|
||||
`from_json()` 可从 JSON 字符串重建 `RunReport`,**仅供查询/分析**,不能重新
|
||||
执行(`TaskSpec` 的 `fn`/`cmd` 等不可序列化字段无法恢复):
|
||||
|
||||
```python
|
||||
restored = px.RunReport.from_json(text)
|
||||
print(restored["double"]) # 任务返回值
|
||||
print(restored.result_of("a").status) # TaskStatus
|
||||
```
|
||||
|
||||
### CLI 可视化
|
||||
|
||||
`pf graph <yaml>` 在终端可视化任务图 DAG,无需运行即可检查依赖关系:
|
||||
|
||||
```bash
|
||||
# 默认 ASCII 渲染:分层树(按标签着色) + 依赖关系表
|
||||
pf graph pipeline.yaml
|
||||
|
||||
# 输出 Mermaid 语法(可粘贴到 mermaid.live)
|
||||
pf graph pipeline.yaml --format mermaid
|
||||
|
||||
# 自定义方向
|
||||
pf graph pipeline.yaml --format mermaid --orientation LR
|
||||
|
||||
# 仅列出任务名
|
||||
pf graph pipeline.yaml --format list
|
||||
|
||||
# 纯依赖关系表
|
||||
pf graph pipeline.yaml --format deps
|
||||
|
||||
# 关闭标签着色(统一单色)
|
||||
pf graph pipeline.yaml --color-by none
|
||||
```
|
||||
|
||||
ASCII 模式默认 `--color-by tag`,按任务首标签选择 rich 颜色并在图下方
|
||||
显示标签颜色图例,便于视觉分组。
|
||||
|
||||
`pf yamlrun` 新增 `--progress` 选项,启用 rich 进度条显示运行中任务、完成
|
||||
数与耗时:
|
||||
|
||||
```bash
|
||||
pf yamlrun pipeline.yaml --progress
|
||||
```
|
||||
|
||||
### 任务列表过滤
|
||||
|
||||
`pf yamlrun --list` 支持 `--list-tag` / `--list-name` 过滤,便于大型 YAML
|
||||
中快速定位任务:
|
||||
|
||||
```bash
|
||||
# 列出所有任务
|
||||
pf yamlrun pipeline.yaml --list
|
||||
|
||||
# 只列出含 ingest 标签的任务
|
||||
pf yamlrun pipeline.yaml --list --list-tag ingest
|
||||
|
||||
# 按任务名子串过滤(大小写不敏感)
|
||||
pf yamlrun pipeline.yaml --list --list-name extract
|
||||
|
||||
# 组合过滤(交集)
|
||||
pf yamlrun pipeline.yaml --list --list-tag ingest --list-name extract
|
||||
```
|
||||
|
||||
### 工具详情与 Shell 补全
|
||||
|
||||
`pf info [tool]` 显示工具详情:
|
||||
|
||||
```bash
|
||||
# 列出所有工具的简表(含子命令数、描述)
|
||||
pf info
|
||||
|
||||
# 显示指定工具的子命令详情
|
||||
pf info clr
|
||||
pf info gittool
|
||||
```
|
||||
|
||||
`pf completion <shell>` 生成 shell 补全脚本,提升日常使用体验:
|
||||
|
||||
```bash
|
||||
# bash —— 添加到 ~/.bashrc
|
||||
pf completion bash >> ~/.bashrc
|
||||
|
||||
# zsh —— 添加到 ~/.zshrc
|
||||
pf completion zsh >> ~/.zshrc
|
||||
|
||||
# fish —— 保存到 ~/.config/fish/completions/pf.fish
|
||||
pf completion fish > ~/.config/fish/completions/pf.fish
|
||||
```
|
||||
|
||||
补全脚本覆盖所有内建命令(`yamlrun`/`graph`/`info`/`completion`)与
|
||||
`@px.tool` 注册的工具名;二级补全交给具体子命令的 `--help` 自描述。
|
||||
|
||||
### 失败诊断
|
||||
|
||||
运行失败时,`RunReport.diagnose()` 自动生成结构化诊断报告,包含:
|
||||
|
||||
- **根因任务**:FAILED 且所有硬依赖非 FAILED 的任务(依赖链中最先出问题的环节)
|
||||
- **依赖链回溯**:从根因到每个失败任务的路径
|
||||
- **相似失败聚类**:按 `(异常类型, 消息前缀)` 聚类,发现批量失败模式
|
||||
- **可操作建议**:识别 FileNotFoundError/ImportError/TimeoutError 等常见异常并给出提示
|
||||
|
||||
```python
|
||||
report = px.run(graph, strategy="sequential")
|
||||
if not report.success:
|
||||
diag = report.diagnose()
|
||||
if diag is not None:
|
||||
print(diag.describe())
|
||||
# 根因任务
|
||||
print(diag.root_causes)
|
||||
# 依赖链
|
||||
for chain in diag.dependency_chains:
|
||||
print(f"{chain.failed_task}: {' -> '.join(chain.chain)}")
|
||||
```
|
||||
|
||||
`pf yamlrun` 失败时自动打印诊断摘要到 stderr:
|
||||
|
||||
```bash
|
||||
pf yamlrun pipeline.yaml # 失败时自动输出诊断报告
|
||||
```
|
||||
|
||||
`report.describe()` 在失败时也会附诊断摘要。
|
||||
|
||||
### 可观测性
|
||||
|
||||
每次 `run()` 自动生成 8 字符十六进制 `run_id`,作为本次运行的唯一追踪 ID,
|
||||
贯穿日志、序列化结果与诊断报告:
|
||||
|
||||
```python
|
||||
report = px.run(graph)
|
||||
print(report.run_id) # 如 "a3f4b2c1"
|
||||
print(report.summary()["run_id"]) # summary 也含 run_id
|
||||
```
|
||||
|
||||
`run_id` 通过结构化日志的 `extra` 字段注入所有执行器日志,便于跨系统关联。
|
||||
日志格式遵循 `logging` 标准延迟格式化,`extra` 字段包括:
|
||||
|
||||
- `run_id` —— 本次运行 ID
|
||||
- `task_name` —— 任务名(任务级日志)
|
||||
- `status` —— 任务状态(`running`/`success`/`failed`/`skipped`)
|
||||
- `attempts` —— 尝试次数(失败日志)
|
||||
- `error_type` —— 异常类型名(失败日志)
|
||||
|
||||
```python
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
# 日志输出示例:
|
||||
# INFO 运行开始: run_id=a3f4b2c1 strategy=dependency tasks=3
|
||||
# INFO task 'a' skipped (cached)
|
||||
# WARNING task 'b' failed (attempt 1/3): RuntimeError('boom'); retrying
|
||||
# INFO 运行结束: run_id=a3f4b2c1 success=False tasks=3
|
||||
```
|
||||
|
||||
`to_json()` / `to_dict()` / `from_json()` 同步保留 `run_id`,反序列化时可还原:
|
||||
旧版 JSON(无 `run_id` 字段)会自动生成新 ID,向前兼容。
|
||||
|
||||
`report.profile(graph)` 返回 `ProfileReport`,基于 `started_at`/`finished_at`
|
||||
做离线性能分析(关键路径、并行度、Top 瓶颈),零运行时开销:
|
||||
|
||||
```python
|
||||
profile = report.profile(graph)
|
||||
print(profile.describe()) # 人类可读报告
|
||||
print(profile.critical_path) # 关键路径任务序列
|
||||
bottlenecks = profile.top_bottlenecks(5) # Top-5 瓶颈任务
|
||||
```
|
||||
|
||||
## 开发
|
||||
|
||||
```bash
|
||||
# 安装开发依赖
|
||||
uv sync --extra dev
|
||||
|
||||
# 运行测试(含覆盖率)
|
||||
uv run pytest --cov=pyflowx --cov-fail-under=100
|
||||
# 运行测试(含覆盖率,阈值 95%)
|
||||
uv run pytest --cov=pyflowx --cov-fail-under=95
|
||||
|
||||
# 类型检查
|
||||
uv run mypy
|
||||
uv run pyrefly check .
|
||||
|
||||
# 代码风格
|
||||
uv run ruff check src tests examples
|
||||
uv run ruff format --check src tests examples
|
||||
uv run ruff check .
|
||||
uv run ruff format --check .
|
||||
```
|
||||
|
||||
## 模块结构
|
||||
|
||||
### 核心
|
||||
|
||||
| 模块 | 职责 |
|
||||
|------|------|
|
||||
| `task.py` | 纯数据结构:`TaskSpec`、`RetryPolicy`、`TaskHooks`、`TaskStatus` |
|
||||
| `graph.py` | DAG 构建、校验、分层、可视化 |
|
||||
| `compose.py` | 多图组合:`GraphComposer` / `compose` |
|
||||
| `context.py` | 上下文注入:参数名→依赖解析 |
|
||||
| `command.py` | 命令执行:`run_command`(list/shell/Callable) |
|
||||
| `cancellation.py` | 任务取消:`CancelToken` 线程安全取消令牌 |
|
||||
| `conditions.py` | 条件执行:内置条件与组合器 |
|
||||
| `executors.py` | 执行器与 `run` 入口:四种策略共享模块级辅助;verbose 统一应用到 spec |
|
||||
| `storage.py` | 状态后端:`MemoryBackend` / `JSONBackend`(batch flush) |
|
||||
| `yaml_loader.py` | YAML 任务编排:`load_yaml` / `parse_yaml_string`(GitHub Actions 风格) |
|
||||
| `runner.py` | CLI 运行器:`CliRunner` |
|
||||
| `report.py` | 运行结果:`RunReport` / `TaskResult` |
|
||||
| `tools.py` | `@px.tool` 装饰器:`ToolSpec` / `run_tool` / `list_tools` / `list_subcommands` |
|
||||
| `profiling.py` | 性能分析:`Profiler` 任务耗时统计 |
|
||||
| `errors.py` | 错误家族:`PyFlowXError` 子类 |
|
||||
| `ops/` | CLI 工具模块集合(每个子模块用 `@px.tool` 注册一个工具) |
|
||||
|
||||
### CLI 工具
|
||||
|
||||
| 模块 | 职责 |
|
||||
|------|------|
|
||||
| `cli/pf.py` | 统一入口:`pf <tool> [command]`,按需导入 `ops/<tool>.py` 触发 `@px.tool` 注册 |
|
||||
| `cli/legacy/profiler.py` | 性能分析 CLI(legacy) |
|
||||
| `cli/legacy/emlmanager.py` | 邮件管理 CLI(legacy) |
|
||||
|
||||
## 许可证
|
||||
|
||||
MIT
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""PyFlowX 性能基准套件.
|
||||
|
||||
用法::
|
||||
|
||||
python -m benchmarks # 运行全部基准
|
||||
python -m benchmarks graph # 仅图构建基准
|
||||
python -m benchmarks execution # 仅执行基准
|
||||
python -m benchmarks context # 仅上下文注入基准
|
||||
python -m benchmarks storage # 仅状态后端基准
|
||||
python -m benchmarks advanced # 仅高级基准
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
__all__ = ["print_results", "time_it"]
|
||||
|
||||
|
||||
def time_it(fn: Callable[[], Any], iterations: int = 100, warmup: int = 5) -> tuple[float, float]:
|
||||
"""计时工具:返回 (平均耗时 ms, 吞吐 ops/sec)."""
|
||||
for _ in range(warmup):
|
||||
fn()
|
||||
times: list[float] = []
|
||||
for _ in range(iterations):
|
||||
t0 = time.perf_counter()
|
||||
fn()
|
||||
times.append(time.perf_counter() - t0)
|
||||
avg = sum(times) / len(times)
|
||||
return avg * 1000, 1.0 / avg if avg > 0 else float("inf")
|
||||
|
||||
|
||||
def print_results(title: str, results: list[tuple[str, int, float, float]]) -> None:
|
||||
"""打印格式化基准结果表."""
|
||||
console = Console()
|
||||
table = Table(title=title, show_header=True, header_style="bold")
|
||||
table.add_column("场景", style="cyan", no_wrap=True)
|
||||
table.add_column("迭代", justify="right")
|
||||
table.add_column("平均耗时", justify="right", style="yellow")
|
||||
table.add_column("吞吐", justify="right", style="green")
|
||||
for name, iters, ms, ops in results:
|
||||
ms_str = f"{ms:.3f} ms" if ms < 1 else f"{ms:.2f} ms"
|
||||
ops_str = f"{ops:.0f} ops/s" if ops > 1000 else f"{ops:.1f} ops/s"
|
||||
table.add_row(name, str(iters), ms_str, ops_str)
|
||||
console.print(table)
|
||||
console.print()
|
||||
@@ -0,0 +1,475 @@
|
||||
"""PyFlowX 性能基准套件.
|
||||
|
||||
用法::
|
||||
|
||||
python -m benchmarks # 运行全部基准
|
||||
python -m benchmarks graph # 仅图构建基准
|
||||
python -m benchmarks execution # 仅执行基准
|
||||
python -m benchmarks context # 仅上下文注入基准
|
||||
python -m benchmarks storage # 仅状态后端基准
|
||||
python -m benchmarks advanced # 仅高级基准
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
import pyflowx as px
|
||||
from benchmarks import print_results, time_it
|
||||
from benchmarks.bench_advanced import run_advanced
|
||||
from pyflowx import Graph, GraphDefaults, RetryPolicy, TaskSpec
|
||||
from pyflowx._json import _HAS_ORJSON
|
||||
from pyflowx._json import dumps as _dumps
|
||||
from pyflowx._json import loads as _loads
|
||||
from pyflowx.context import build_call_args
|
||||
from pyflowx.storage import JSONBackend, MemoryBackend, SQLiteBackend
|
||||
|
||||
# ============================================================================
|
||||
# 图生成工具
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def make_chain(n: int) -> list[TaskSpec]:
|
||||
"""生成 n 个任务的链式 DAG。"""
|
||||
specs = [TaskSpec(f"t{i}", cmd=["true"]) for i in range(n)]
|
||||
for i in range(1, n):
|
||||
specs[i] = TaskSpec(f"t{i}", cmd=["true"], depends_on=(f"t{i - 1}",))
|
||||
return specs
|
||||
|
||||
|
||||
def make_diamond(n: int) -> list[TaskSpec]:
|
||||
"""生成 n 个任务的菱形 DAG(每层宽度约 sqrt(n))。"""
|
||||
width = max(1, int(math.sqrt(n)))
|
||||
specs: list[TaskSpec] = []
|
||||
prev_layer: list[str] = []
|
||||
layer = 0
|
||||
count = 0
|
||||
while count < n:
|
||||
cur_layer: list[str] = []
|
||||
for j in range(width):
|
||||
if count >= n:
|
||||
break
|
||||
name = f"l{layer}_t{j}"
|
||||
deps = tuple(prev_layer) if prev_layer else ()
|
||||
specs.append(TaskSpec(name, cmd=["true"], depends_on=deps))
|
||||
cur_layer.append(name)
|
||||
count += 1
|
||||
prev_layer = cur_layer
|
||||
layer += 1
|
||||
return specs
|
||||
|
||||
|
||||
def make_wide(n: int) -> list[TaskSpec]:
|
||||
"""生成 n 个独立任务(无依赖,最大并行度)。"""
|
||||
return [TaskSpec(f"t{i}", cmd=["true"]) for i in range(n)]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 基准:图构建
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def bench_construction() -> None:
|
||||
"""图构建(from_specs + validate)基准。"""
|
||||
results = []
|
||||
for n in (10, 100, 500, 1000):
|
||||
specs = make_chain(n)
|
||||
ms, _ = time_it(lambda s=specs: Graph.from_specs(s), iterations=20)
|
||||
results.append((f"chain({n})", 20, ms, 1.0 / (ms / 1000) if ms > 0 else 0))
|
||||
|
||||
for n in (10, 100, 500, 1000):
|
||||
specs = make_diamond(n)
|
||||
ms, _ = time_it(lambda s=specs: Graph.from_specs(s), iterations=20)
|
||||
results.append((f"diamond({n})", 20, ms, 1.0 / (ms / 1000) if ms > 0 else 0))
|
||||
|
||||
print_results("图构建 (from_specs + validate)", results)
|
||||
|
||||
|
||||
def bench_layers() -> None:
|
||||
"""拓扑分层基准(冷启动 vs 缓存命中)。"""
|
||||
results = []
|
||||
for n in (100, 500, 1000):
|
||||
specs = make_diamond(n)
|
||||
graph = Graph.from_specs(specs)
|
||||
|
||||
def _cold(g: Graph = graph) -> None:
|
||||
g._layers_cache = None # type: ignore[attr-defined]
|
||||
g.layers()
|
||||
|
||||
ms_cold, ops_cold = time_it(_cold, iterations=50, warmup=5)
|
||||
results.append((f"layers(cold,{n})", 50, ms_cold, ops_cold))
|
||||
|
||||
ms_hot, ops_hot = time_it(lambda g=graph: g.layers(), iterations=200, warmup=10)
|
||||
results.append((f"layers(cached,{n})", 200, ms_hot, ops_hot))
|
||||
|
||||
print_results("拓扑分层 (layers)", results)
|
||||
|
||||
|
||||
def bench_resolved_spec() -> None:
|
||||
"""resolved_spec 缓存命中基准。"""
|
||||
results = []
|
||||
for n in (100, 500, 1000):
|
||||
specs = make_chain(n)
|
||||
defaults = GraphDefaults(retry=RetryPolicy(max_attempts=2))
|
||||
graph = Graph.from_specs(specs, defaults=defaults)
|
||||
name = f"t{n // 2}"
|
||||
ms, ops = time_it(lambda g=graph, nm=name: g.resolved_spec(nm), iterations=500, warmup=20)
|
||||
results.append((f"resolved_spec(cached,{n})", 500, ms, ops))
|
||||
|
||||
print_results("resolved_spec (缓存命中)", results)
|
||||
|
||||
|
||||
def run_graph() -> None:
|
||||
"""运行全部图基准。"""
|
||||
bench_construction()
|
||||
bench_layers()
|
||||
bench_resolved_spec()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 基准:任务执行
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def bench_sequential() -> None:
|
||||
"""sequential 策略执行基准。"""
|
||||
results = []
|
||||
|
||||
def noop() -> None:
|
||||
pass
|
||||
|
||||
for n in (50, 200, 500):
|
||||
specs = [TaskSpec(f"t{i}", fn=noop) for i in range(n)]
|
||||
graph = Graph.from_specs(specs)
|
||||
ms, ops = time_it(lambda g=graph: px.run(g, strategy="sequential"), iterations=10, warmup=2)
|
||||
results.append((f"sequential({n})", 10, ms, ops))
|
||||
|
||||
print_results("执行策略: sequential", results)
|
||||
|
||||
|
||||
def bench_thread() -> None:
|
||||
"""thread 策略执行基准。"""
|
||||
results = []
|
||||
|
||||
def noop() -> None:
|
||||
pass
|
||||
|
||||
for n in (50, 200, 500):
|
||||
specs = [TaskSpec(f"t{i}", fn=noop) for i in range(n)]
|
||||
graph = Graph.from_specs(specs)
|
||||
ms, ops = time_it(lambda g=graph: px.run(g, strategy="thread"), iterations=10, warmup=2)
|
||||
results.append((f"thread({n})", 10, ms, ops))
|
||||
|
||||
print_results("执行策略: thread", results)
|
||||
|
||||
|
||||
def bench_async() -> None:
|
||||
"""async 策略执行基准。"""
|
||||
results = []
|
||||
|
||||
def noop() -> None:
|
||||
pass
|
||||
|
||||
for n in (50, 200, 500):
|
||||
specs = [TaskSpec(f"t{i}", fn=noop) for i in range(n)]
|
||||
graph = Graph.from_specs(specs)
|
||||
ms, ops = time_it(lambda g=graph: px.run(g, strategy="async"), iterations=10, warmup=2)
|
||||
results.append((f"async({n})", 10, ms, ops))
|
||||
|
||||
print_results("执行策略: async", results)
|
||||
|
||||
|
||||
def bench_dependency() -> None:
|
||||
"""dependency 策略执行基准。"""
|
||||
results = []
|
||||
|
||||
def noop() -> None:
|
||||
pass
|
||||
|
||||
for n in (50, 200, 500):
|
||||
specs = [TaskSpec(f"t{i}", fn=noop) for i in range(n)]
|
||||
graph = Graph.from_specs(specs)
|
||||
ms, ops = time_it(lambda g=graph: px.run(g, strategy="dependency"), iterations=10, warmup=2)
|
||||
results.append((f"dependency({n})", 10, ms, ops))
|
||||
|
||||
print_results("执行策略: dependency", results)
|
||||
|
||||
|
||||
def bench_large_graph() -> None:
|
||||
"""大图调度基准:验证增量就绪集优化效果(1k–10k 任务)。
|
||||
|
||||
覆盖三种图形状:
|
||||
* chain —— 深链(每轮仅 1 个就绪,调度轮数 = N)
|
||||
* wide —— 完全并行(首轮全部就绪,仅 1 轮调度)
|
||||
* diamond —— 菱形(多层,每层多任务)
|
||||
"""
|
||||
results = []
|
||||
|
||||
def noop() -> None:
|
||||
pass
|
||||
|
||||
for shape, maker in (("chain", make_chain), ("diamond", make_diamond), ("wide", make_wide)):
|
||||
for n in (1000, 5000, 10000):
|
||||
specs = maker(n)
|
||||
# 替换为 noop fn(避免子进程开销,纯测调度性能)
|
||||
specs = [TaskSpec(s.name, fn=noop, depends_on=s.depends_on) for s in specs]
|
||||
graph = Graph.from_specs(specs)
|
||||
ms, ops = time_it(lambda g=graph: px.run(g, strategy="dependency"), iterations=3, warmup=1)
|
||||
results.append((f"dependency-{shape}({n})", 3, ms, ops))
|
||||
|
||||
print_results("大图调度 (dependency, 增量就绪集)", results)
|
||||
|
||||
|
||||
def bench_cmd_execution() -> None:
|
||||
"""cmd 任务执行基准(真实子进程)。"""
|
||||
results = []
|
||||
for n in (10, 50, 100):
|
||||
specs = [TaskSpec(f"t{i}", cmd=["true"]) for i in range(n)]
|
||||
graph = Graph.from_specs(specs)
|
||||
ms, ops = time_it(lambda g=graph: px.run(g, strategy="sequential"), iterations=5, warmup=1)
|
||||
results.append((f"cmd-sequential({n})", 5, ms, ops))
|
||||
|
||||
for n in (10, 50, 100):
|
||||
specs = [TaskSpec(f"t{i}", cmd=["true"]) for i in range(n)]
|
||||
graph = Graph.from_specs(specs)
|
||||
ms, ops = time_it(lambda g=graph: px.run(g, strategy="thread", max_workers=8), iterations=5, warmup=1)
|
||||
results.append((f"cmd-thread({n})", 5, ms, ops))
|
||||
|
||||
print_results("cmd 任务执行 (['true'])", results)
|
||||
|
||||
|
||||
def run_execution() -> None:
|
||||
"""运行全部执行基准。"""
|
||||
bench_sequential()
|
||||
bench_thread()
|
||||
bench_async()
|
||||
bench_dependency()
|
||||
bench_large_graph()
|
||||
bench_cmd_execution()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 基准:上下文注入
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def bench_context_no_deps() -> None:
|
||||
"""无依赖 fn 任务的上下文注入基准。"""
|
||||
results = []
|
||||
|
||||
def noop() -> None:
|
||||
pass
|
||||
|
||||
spec = TaskSpec("noop", fn=noop)
|
||||
context: dict[str, Any] = {}
|
||||
ms, ops = time_it(lambda s=spec, c=context: build_call_args(s, c), iterations=2000, warmup=100)
|
||||
results.append(("fn(no-deps)", 2000, ms, ops))
|
||||
|
||||
# cmd 任务快速路径
|
||||
spec_cmd = TaskSpec("cmd", cmd=["true"])
|
||||
ms, ops = time_it(lambda s=spec_cmd, c=context: build_call_args(s, c), iterations=2000, warmup=100)
|
||||
results.append(("cmd(fast-path)", 2000, ms, ops))
|
||||
|
||||
print_results("上下文注入 (build_call_args)", results)
|
||||
|
||||
|
||||
def bench_context_with_deps() -> None:
|
||||
"""有依赖 fn 任务的上下文注入基准。"""
|
||||
results = []
|
||||
|
||||
def consumer(a: int, b: int) -> int:
|
||||
return a + b
|
||||
|
||||
spec = TaskSpec("consumer", fn=consumer, depends_on=("a", "b"))
|
||||
context = {"a": 1, "b": 2, "c": 3}
|
||||
ms, ops = time_it(lambda s=spec, c=context: build_call_args(s, c), iterations=2000, warmup=100)
|
||||
results.append(("fn(2-deps)", 2000, ms, ops))
|
||||
|
||||
# Context 标注
|
||||
from pyflowx.task import Context
|
||||
|
||||
def ctx_fn(ctx: Context) -> int:
|
||||
return sum(ctx.values())
|
||||
|
||||
spec_ctx = TaskSpec("ctx", fn=ctx_fn, depends_on=("a", "b"))
|
||||
ms, ops = time_it(lambda s=spec_ctx, c=context: build_call_args(s, c), iterations=2000, warmup=100)
|
||||
results.append(("fn(Context-annotated)", 2000, ms, ops))
|
||||
|
||||
# **kwargs
|
||||
def kwargs_fn(**kwargs: int) -> int:
|
||||
return sum(kwargs.values())
|
||||
|
||||
spec_kw = TaskSpec("kw", fn=kwargs_fn, depends_on=("a", "b"))
|
||||
ms, ops = time_it(lambda s=spec_kw, c=context: build_call_args(s, c), iterations=2000, warmup=100)
|
||||
results.append(("fn(**kwargs)", 2000, ms, ops))
|
||||
|
||||
print_results("上下文注入 (有依赖)", results)
|
||||
|
||||
|
||||
def run_context() -> None:
|
||||
"""运行全部上下文注入基准。"""
|
||||
bench_context_no_deps()
|
||||
bench_context_with_deps()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 基准:状态后端
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def bench_storage() -> None:
|
||||
"""状态后端 save/load 基准。"""
|
||||
results = []
|
||||
|
||||
# MemoryBackend
|
||||
mem = MemoryBackend()
|
||||
ms, ops = time_it(lambda: mem.save("key", "value"), iterations=1000, warmup=50)
|
||||
results.append(("MemoryBackend.save", 1000, ms, ops))
|
||||
|
||||
ms, ops = time_it(mem.load, iterations=1000, warmup=50)
|
||||
results.append(("MemoryBackend.load", 1000, ms, ops))
|
||||
|
||||
# JSONBackend(batch 模式)
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
json_path = str(Path(tmp_dir) / "state.json")
|
||||
|
||||
json_backend = JSONBackend(json_path)
|
||||
with json_backend.batch():
|
||||
for i in range(100):
|
||||
json_backend.save(f"task_{i}", f"result_{i}")
|
||||
|
||||
def _json_save() -> None:
|
||||
jb = JSONBackend(json_path)
|
||||
with jb.batch():
|
||||
for i in range(10):
|
||||
jb.save(f"bench_{i}", f"val_{i}")
|
||||
|
||||
ms, ops = time_it(_json_save, iterations=50, warmup=5)
|
||||
results.append(("JSONBackend.save(batch=10)", 50, ms, ops))
|
||||
|
||||
# 复杂 value(嵌套 dict)—— 展示 batch 模式延迟验证的优化效果
|
||||
complex_value = {
|
||||
"output": {"path": "/data/result.json", "size": 1024, "checksum": "abc123"},
|
||||
"metrics": {"accuracy": 0.95, "latency_ms": 120, "samples": 1000},
|
||||
"artifacts": [f"artifact_{j}.bin" for j in range(10)],
|
||||
"metadata": {"version": "1.0", "tags": ["trained", "validated"], "created_at": "2026-07-07"},
|
||||
}
|
||||
|
||||
def _json_save_complex() -> None:
|
||||
jb = JSONBackend(json_path)
|
||||
with jb.batch():
|
||||
for i in range(10):
|
||||
jb.save(f"bench_complex_{i}", complex_value)
|
||||
|
||||
ms, ops = time_it(_json_save_complex, iterations=50, warmup=5)
|
||||
results.append(("JSONBackend.save(batch=10,complex)", 50, ms, ops))
|
||||
|
||||
ms, ops = time_it(json_backend.load, iterations=200, warmup=10)
|
||||
results.append(("JSONBackend.load", 200, ms, ops))
|
||||
|
||||
# SQLiteBackend
|
||||
db_path = str(Path(tmp_dir) / "state.db")
|
||||
|
||||
sqlite_backend = SQLiteBackend(db_path)
|
||||
with sqlite_backend.batch():
|
||||
for i in range(100):
|
||||
sqlite_backend.save(f"task_{i}", f"result_{i}")
|
||||
|
||||
def _sqlite_save() -> None:
|
||||
sb = SQLiteBackend(db_path)
|
||||
with sb.batch():
|
||||
for i in range(10):
|
||||
sb.save(f"bench_{i}", f"val_{i}")
|
||||
|
||||
ms, ops = time_it(_sqlite_save, iterations=50, warmup=5)
|
||||
results.append(("SQLiteBackend.save(batch=10)", 50, ms, ops))
|
||||
|
||||
ms, ops = time_it(sqlite_backend.load, iterations=200, warmup=10)
|
||||
results.append(("SQLiteBackend.load", 200, ms, ops))
|
||||
|
||||
print_results("状态后端 (save/load)", results)
|
||||
|
||||
# 序列化基准:_json 抽象层 dumps/loads
|
||||
ser_results = []
|
||||
report_like = {
|
||||
"run_id": "abc12345",
|
||||
"success": True,
|
||||
"results": [
|
||||
{
|
||||
"name": f"task_{i}",
|
||||
"status": "success",
|
||||
"attempts": 1,
|
||||
"duration_seconds": 0.123 + i * 0.001,
|
||||
"value": {"output": [i, i + 1, i + 2], "meta": {"tag": "api"}},
|
||||
}
|
||||
for i in range(100)
|
||||
],
|
||||
}
|
||||
serialized = _dumps(report_like)
|
||||
|
||||
ms, ops = time_it(lambda: _dumps(report_like), iterations=200, warmup=10)
|
||||
ser_results.append(("dumps(report-100)", 200, ms, ops))
|
||||
|
||||
ms, ops = time_it(lambda: _loads(serialized), iterations=200, warmup=10)
|
||||
ser_results.append(("loads(report-100)", 200, ms, ops))
|
||||
|
||||
backend_name = "orjson" if _HAS_ORJSON else "stdlib"
|
||||
print_results(f"JSON 序列化 (后端={backend_name})", ser_results)
|
||||
|
||||
# 清理临时目录
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def run_storage() -> None:
|
||||
"""运行全部状态后端基准。"""
|
||||
bench_storage()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 主入口
|
||||
# ============================================================================
|
||||
|
||||
|
||||
BENCH_MODULES: dict[str, Callable[[], None]] = {
|
||||
"graph": run_graph,
|
||||
"execution": run_execution,
|
||||
"context": run_context,
|
||||
"storage": run_storage,
|
||||
"advanced": run_advanced,
|
||||
}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""CLI 入口。"""
|
||||
args = argv if argv is not None else sys.argv[1:]
|
||||
console = Console()
|
||||
console.print("[bold cyan]PyFlowX 性能基准套件[/bold cyan]\n")
|
||||
|
||||
if not args or args[0] in ("--all", "-a"):
|
||||
for name, fn in BENCH_MODULES.items():
|
||||
console.print(f"[bold]运行: {name}[/bold]")
|
||||
fn()
|
||||
elif args[0] in BENCH_MODULES:
|
||||
BENCH_MODULES[args[0]]()
|
||||
elif args[0] in ("--help", "-h"):
|
||||
console.print("用法: python -m benchmarks [graph|execution|context|storage]")
|
||||
console.print(" 无参数 = 运行全部基准")
|
||||
else:
|
||||
console.print(f"[red]未知基准模块: {args[0]}[/red]")
|
||||
console.print(f"可用: {', '.join(BENCH_MODULES)}")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,309 @@
|
||||
"""高级基准:条件评估/YAML 加载/通知器/run_iter/子图/取消/CPU+I/O 密集型任务.
|
||||
|
||||
这些基准覆盖核心调度引擎的扩展路径,与 bench_graph/bench_execution/bench_context/
|
||||
bench_storage 互补,构成完整的性能度量体系。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pyflowx as px
|
||||
from benchmarks import print_results, time_it
|
||||
from pyflowx import Graph, TaskSpec
|
||||
from pyflowx.cancellation import CancelToken
|
||||
from pyflowx.conditions import BuiltinConditions
|
||||
from pyflowx.notification import CallbackNotifier, NotificationLevel
|
||||
|
||||
# ============================================================================
|
||||
# 基准:条件评估
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def bench_conditions() -> None:
|
||||
"""条件评估基准(静态/上下文/复合)."""
|
||||
results: list[tuple[str, int, float, float]] = []
|
||||
|
||||
# 静态条件(IS_LINUX)
|
||||
cond_static = BuiltinConditions.IS_LINUX()
|
||||
ctx: dict[str, Any] = {}
|
||||
ms, ops = time_it(lambda c=cond_static, x=ctx: c(x), iterations=10000, warmup=500)
|
||||
results.append(("static(IS_LINUX)", 10000, ms, ops))
|
||||
|
||||
# 上下文条件(DEP_EQUALS)
|
||||
cond_dep = BuiltinConditions.DEP_EQUALS("a", 1)
|
||||
ctx_dep = {"a": 1, "b": 2}
|
||||
ms, ops = time_it(lambda c=cond_dep, x=ctx_dep: c(x), iterations=10000, warmup=500)
|
||||
results.append(("DEP_EQUALS", 10000, ms, ops))
|
||||
|
||||
# 复合条件(AND(OR(NOT, DEP_TRUTHY), DEP_PRESENT))
|
||||
cond_complex = BuiltinConditions.AND(
|
||||
BuiltinConditions.OR(
|
||||
BuiltinConditions.NOT(BuiltinConditions.IS_WINDOWS()),
|
||||
BuiltinConditions.DEP_TRUTHY("a"),
|
||||
),
|
||||
BuiltinConditions.DEP_PRESENT("b"),
|
||||
)
|
||||
ms, ops = time_it(lambda c=cond_complex, x=ctx_dep: c(x), iterations=10000, warmup=500)
|
||||
results.append(("AND(OR(NOT,DEP_TRUTHY),DEP_PRESENT)", 10000, ms, ops))
|
||||
|
||||
print_results("条件评估", results)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 基准:YAML 加载
|
||||
# ============================================================================
|
||||
|
||||
|
||||
_YAML_TEMPLATE = """\
|
||||
jobs:
|
||||
{jobs}
|
||||
defaults:
|
||||
retry:
|
||||
max_attempts: 2
|
||||
backoff: 0.0
|
||||
"""
|
||||
|
||||
|
||||
def _make_yaml(n: int) -> str:
|
||||
"""生成 n 个任务的 YAML(链式依赖)."""
|
||||
lines = []
|
||||
for i in range(n):
|
||||
deps = f"needs: [t{i - 1}]" if i > 0 else ""
|
||||
lines.append(f" t{i}:\n cmd: ['true']\n {deps}".rstrip())
|
||||
return _YAML_TEMPLATE.format(jobs="\n".join(lines))
|
||||
|
||||
|
||||
def bench_yaml_load() -> None:
|
||||
"""YAML 加载基准(解析 + Graph 构建)."""
|
||||
results: list[tuple[str, int, float, float]] = []
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
|
||||
for n in (10, 50, 100):
|
||||
yaml_text = _make_yaml(n)
|
||||
yaml_path = Path(tmp_dir) / f"bench_{n}.yaml"
|
||||
yaml_path.write_text(yaml_text, encoding="utf-8")
|
||||
ms, _ops = time_it(lambda p=yaml_path: Graph.from_yaml(p), iterations=20, warmup=2)
|
||||
results.append((f"yaml_load({n})", 20, ms, 1.0 / (ms / 1000) if ms > 0 else 0))
|
||||
|
||||
print_results("YAML 加载 (from_yaml)", results)
|
||||
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 基准:通知器
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def bench_notifiers() -> None:
|
||||
"""通知器 notify 调用开销基准."""
|
||||
results: list[tuple[str, int, float, float]] = []
|
||||
|
||||
# 构造一个真实 TaskEvent
|
||||
from pyflowx.task import TaskEvent, TaskStatus
|
||||
|
||||
event = TaskEvent(task="t0", status=TaskStatus.SUCCESS, attempts=1)
|
||||
|
||||
# CallbackNotifier(无级别过滤)
|
||||
calls: list[int] = [0]
|
||||
|
||||
def _cb(_e: Any) -> None:
|
||||
calls[0] += 1
|
||||
|
||||
notifier = CallbackNotifier(_cb)
|
||||
ms, ops = time_it(lambda e=event, n=notifier: n.notify(e), iterations=50000, warmup=1000)
|
||||
results.append(("CallbackNotifier.notify", 50000, ms, ops))
|
||||
|
||||
# 级别过滤(仅 SUCCESS)
|
||||
notifier_filtered = CallbackNotifier(_cb, levels={NotificationLevel.SUCCESS})
|
||||
ms, ops = time_it(lambda e=event, n=notifier_filtered: n.notify(e), iterations=50000, warmup=1000)
|
||||
results.append(("CallbackNotifier(filtered).notify", 50000, ms, ops))
|
||||
|
||||
print_results("通知器 notify", results)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 基准:run_iter 流式 API
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def bench_run_iter() -> None:
|
||||
"""run_iter 流式执行基准."""
|
||||
results: list[tuple[str, int, float, float]] = []
|
||||
|
||||
def noop() -> None:
|
||||
pass
|
||||
|
||||
for n in (50, 200, 500):
|
||||
specs = [TaskSpec(f"t{i}", fn=noop) for i in range(n)]
|
||||
graph = Graph.from_specs(specs)
|
||||
|
||||
def _run_iter(g: Graph = graph) -> None:
|
||||
list(px.run_iter(g, strategy="sequential"))
|
||||
|
||||
ms, ops = time_it(_run_iter, iterations=10, warmup=2)
|
||||
results.append((f"run_iter(sequential,{n})", 10, ms, ops))
|
||||
|
||||
print_results("run_iter 流式执行", results)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 基准:子图过滤
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def bench_subgraph() -> None:
|
||||
"""子图过滤基准(subgraph_with_deps 传递闭包计算)."""
|
||||
results: list[tuple[str, int, float, float]] = []
|
||||
|
||||
for n in (100, 500, 1000):
|
||||
specs = []
|
||||
for i in range(n):
|
||||
deps = (f"t{i - 1}",) if i > 0 else ()
|
||||
specs.append(TaskSpec(f"t{i}", cmd=["true"], depends_on=deps))
|
||||
graph = Graph.from_specs(specs)
|
||||
# 取中点任务,触发前半部分传递闭包
|
||||
target = f"t{n // 2}"
|
||||
ms, ops = time_it(
|
||||
lambda g=graph, t=target: g.subgraph_with_deps([t]),
|
||||
iterations=50,
|
||||
warmup=5,
|
||||
)
|
||||
results.append((f"subgraph_with_deps({n})", 50, ms, ops))
|
||||
|
||||
print_results("子图过滤 (subgraph_with_deps)", results)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 基准:取消机制
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def bench_cancellation() -> None:
|
||||
"""取消机制基准(cancel_event 触发到返回)."""
|
||||
results: list[tuple[str, int, float, float]] = []
|
||||
|
||||
def slow() -> None:
|
||||
time.sleep(0.5)
|
||||
|
||||
for n in (50, 200):
|
||||
specs = [TaskSpec(f"t{i}", fn=slow) for i in range(n)]
|
||||
graph = Graph.from_specs(specs)
|
||||
|
||||
def _cancel_after(g: Graph = graph) -> None:
|
||||
token = CancelToken()
|
||||
# 立即取消,让所有任务变 SKIPPED
|
||||
token.cancel()
|
||||
with contextlib.suppress(Exception):
|
||||
px.run(g, strategy="sequential", cancel_event=token)
|
||||
|
||||
ms, ops = time_it(_cancel_after, iterations=5, warmup=1)
|
||||
results.append((f"cancel(immediate,{n})", 5, ms, ops))
|
||||
|
||||
print_results("取消机制 (cancel_event)", results)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 基准:CPU 密集型任务
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _fib(n: int) -> int:
|
||||
"""递归斐波那契(CPU 密集型)."""
|
||||
if n < 2:
|
||||
return n
|
||||
return _fib(n - 1) + _fib(n - 2)
|
||||
|
||||
|
||||
def bench_cpu_intensive() -> None:
|
||||
"""CPU 密集型任务基准(递归斐波那契)."""
|
||||
results: list[tuple[str, int, float, float]] = []
|
||||
|
||||
for n_tasks, fib_n in ((10, 20), (20, 20), (10, 25)):
|
||||
specs = [TaskSpec(f"t{i}", fn=_fib, args=(fib_n,)) for i in range(n_tasks)]
|
||||
graph = Graph.from_specs(specs)
|
||||
|
||||
# sequential
|
||||
ms, ops = time_it(
|
||||
lambda g=graph: px.run(g, strategy="sequential"),
|
||||
iterations=3,
|
||||
warmup=1,
|
||||
)
|
||||
results.append((f"cpu-seq({n_tasks}x fib{fib_n})", 3, ms, ops))
|
||||
|
||||
# thread(CPU 密集型受 GIL 限制,验证是否退化)
|
||||
ms, ops = time_it(
|
||||
lambda g=graph: px.run(g, strategy="thread", max_workers=4),
|
||||
iterations=3,
|
||||
warmup=1,
|
||||
)
|
||||
results.append((f"cpu-thread({n_tasks}x fib{fib_n})", 3, ms, ops))
|
||||
|
||||
print_results("CPU 密集型 (递归斐波那契)", results)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 基准:I/O 密集型任务
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def bench_io_intensive() -> None:
|
||||
"""I/O 密集型任务基准(sleep 模拟)."""
|
||||
results: list[tuple[str, int, float, float]] = []
|
||||
|
||||
def io_sleep() -> None:
|
||||
time.sleep(0.01) # 10ms 模拟 I/O
|
||||
|
||||
for n in (20, 50):
|
||||
specs = [TaskSpec(f"t{i}", fn=io_sleep) for i in range(n)]
|
||||
graph = Graph.from_specs(specs)
|
||||
|
||||
# sequential(总耗时 ≈ n * 10ms)
|
||||
ms, ops = time_it(
|
||||
lambda g=graph: px.run(g, strategy="sequential"),
|
||||
iterations=3,
|
||||
warmup=1,
|
||||
)
|
||||
results.append((f"io-seq({n})", 3, ms, ops))
|
||||
|
||||
# thread(I/O 密集型应能并行,总耗时 ≈ 10ms)
|
||||
ms, ops = time_it(
|
||||
lambda g=graph: px.run(g, strategy="thread", max_workers=8),
|
||||
iterations=3,
|
||||
warmup=1,
|
||||
)
|
||||
results.append((f"io-thread({n})", 3, ms, ops))
|
||||
|
||||
# async(I/O 密集型应能并行)
|
||||
ms, ops = time_it(
|
||||
lambda g=graph: px.run(g, strategy="async"),
|
||||
iterations=3,
|
||||
warmup=1,
|
||||
)
|
||||
results.append((f"io-async({n})", 3, ms, ops))
|
||||
|
||||
print_results("I/O 密集型 (sleep 10ms)", results)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 主入口
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def run_advanced() -> None:
|
||||
"""运行全部高级基准."""
|
||||
bench_conditions()
|
||||
bench_yaml_load()
|
||||
bench_notifiers()
|
||||
bench_run_iter()
|
||||
bench_subgraph()
|
||||
bench_cancellation()
|
||||
bench_cpu_intensive()
|
||||
bench_io_intensive()
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
API 参考
|
||||
========
|
||||
|
||||
任务描述
|
||||
--------
|
||||
|
||||
.. autoclass:: pyflowx.TaskSpec
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
:exclude-members: args, kwargs
|
||||
|
||||
.. autoclass:: pyflowx.RetryPolicy
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.TaskHooks
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.TaskStatus
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
图构建
|
||||
------
|
||||
|
||||
.. autoclass:: pyflowx.Graph
|
||||
:members:
|
||||
:undoc-members:
|
||||
:exclude-members: from_specs, from_yaml
|
||||
|
||||
.. autoclass:: pyflowx.GraphDefaults
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autofunction:: pyflowx.compose
|
||||
.. autofunction:: pyflowx.task_template
|
||||
|
||||
执行
|
||||
----
|
||||
|
||||
.. autofunction:: pyflowx.run
|
||||
|
||||
.. autoclass:: pyflowx.RunReport
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.TaskResult
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
@px.tool 工具
|
||||
-------------
|
||||
|
||||
.. autoclass:: pyflowx.ToolSpec
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autofunction:: pyflowx.tool
|
||||
.. autofunction:: pyflowx.run_tool
|
||||
.. autofunction:: pyflowx.list_tools
|
||||
.. autofunction:: pyflowx.list_subcommands
|
||||
|
||||
命令执行
|
||||
--------
|
||||
|
||||
.. autofunction:: pyflowx.run_command
|
||||
|
||||
CLI 运行器
|
||||
----------
|
||||
|
||||
.. autoclass:: pyflowx.CliRunner
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
状态后端
|
||||
--------
|
||||
|
||||
.. autoclass:: pyflowx.StateBackend
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.MemoryBackend
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.JSONBackend
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
错误家族
|
||||
--------
|
||||
|
||||
.. autoexception:: pyflowx.PyFlowXError
|
||||
.. autoexception:: pyflowx.DuplicateTaskError
|
||||
.. autoexception:: pyflowx.MissingDependencyError
|
||||
.. autoexception:: pyflowx.CycleError
|
||||
.. autoexception:: pyflowx.TaskFailedError
|
||||
.. autoexception:: pyflowx.TaskTimeoutError
|
||||
.. autoexception:: pyflowx.InjectionError
|
||||
.. autoexception:: pyflowx.StorageError
|
||||
@@ -0,0 +1,45 @@
|
||||
变更日志
|
||||
========
|
||||
|
||||
0.4.5
|
||||
-----
|
||||
|
||||
CLI 重构
|
||||
~~~~~~~~
|
||||
|
||||
- 新增 ``pf`` 统一入口:通过 ``pf <tool> [command] [options]`` 调用所有工具
|
||||
- 13 个工具迁移到 YAML 配置(filedate/filelevel/folderback/folderzip/screenshot/sshcopyid/lscalc/bumpversion/autofmt/piptool/packtool/pdftool/gittool)
|
||||
- YAML 配置支持 ``cli:`` 段声明命令行参数 schema,由 ``build_cli_parser`` 自动生成 argparse
|
||||
- 删除 13 个冗余 ``.py`` 入口脚本,统一通过 ``pf`` 调用
|
||||
- ``run()`` 在 ``verbose=True`` 时自动把 verbose 标记应用到所有 spec
|
||||
- 全局选项 ``--verbose`` 改为 ``--quiet``(默认显示执行过程)
|
||||
- ``cmd`` 任务成功时打印 stdout(此前被静默丢弃)
|
||||
- ``gittool`` 用 ``CLEAN_EXCLUDES`` 数组变量配置 ``git clean -e`` 参数
|
||||
|
||||
YAML 任务编排
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- 支持 ``variables`` 变量定义,``${VAR}`` 在 cmd/env/cwd 中替换
|
||||
- 列表变量展开为 cmd 数组多个元素
|
||||
- ``cli:`` 段支持 subcommands/positional/options 三级 schema
|
||||
- 支持 ``type: path`` 自动转为 ``pathlib.Path``
|
||||
|
||||
文档
|
||||
~~~~
|
||||
|
||||
- 搭建 Sphinx 文档,发布到 ReadTheDocs
|
||||
- 更新 README:CLI 示例改为 ``pf`` 统一入口,模块结构表补全
|
||||
|
||||
0.3.x
|
||||
-----
|
||||
|
||||
- 新增 YAML 任务编排(GitHub Actions 风格 schema)
|
||||
- 新增 ``fn:`` 函数引用与 ``register_fn`` / ``get_fn`` 注册中心
|
||||
- 新增 ``compose`` / ``GraphComposer`` 多图组合
|
||||
- 新增 ``task_template`` 任务模板工厂
|
||||
- 新增 ``concurrency_key`` + ``concurrency_limits`` 并发限制
|
||||
- 新增 ``JSONBackend`` 断点续跑与 ``batch()`` 批量落盘
|
||||
- 新增 ``cache_key`` 缓存键函数
|
||||
- 新增条件执行(``IS_WINDOWS`` / ``HAS_INSTALLED`` / ``ENV_VAR_EQUALS`` 等)
|
||||
- 四种执行策略:``sequential`` / ``thread`` / ``async`` / ``dependency``
|
||||
- 参数名即依赖的上下文注入机制
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Sphinx 配置.
|
||||
|
||||
ReadTheDocs 构建 PyFlowX 文档站。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 确保 src/ 在 sys.path 中, autodoc 能导入 pyflowx
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
|
||||
|
||||
from pyflowx import __version__
|
||||
|
||||
# -- 项目信息 --------------------------------------------------------------
|
||||
project = "PyFlowX"
|
||||
author = "pyflowx"
|
||||
copyright = "2024, pyflowx"
|
||||
release = __version__
|
||||
version = __version__
|
||||
|
||||
# -- Sphinx 配置 -----------------------------------------------------------
|
||||
extensions = [
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.napoleon",
|
||||
"sphinx.ext.viewcode",
|
||||
"sphinx.ext.intersphinx",
|
||||
"myst_parser",
|
||||
]
|
||||
|
||||
# -- 主题 ------------------------------------------------------------------
|
||||
html_theme = "sphinx_rtd_theme"
|
||||
html_static_path = ["_static"]
|
||||
|
||||
# -- autodoc 配置 ----------------------------------------------------------
|
||||
autodoc_default_options = {
|
||||
"members": True,
|
||||
"undoc-members": True,
|
||||
"show-inheritance": True,
|
||||
"member-order": "bysource",
|
||||
}
|
||||
autodoc_type_hints = "description"
|
||||
autodoc_typehints_format = "short"
|
||||
|
||||
# -- napoleon 配置 (Google/NumPy docstring 兼容) --------------------------
|
||||
napoleon_google_docstring = True
|
||||
napoleon_numpy_docstring = True
|
||||
napoleon_include_init_with_doc = False
|
||||
napoleon_include_private_with_doc = False
|
||||
napoleon_include_special_with_doc = True
|
||||
|
||||
# -- intersphinx -----------------------------------------------------------
|
||||
intersphinx_mapping = {
|
||||
"python": ("https://docs.python.org/3", None),
|
||||
}
|
||||
|
||||
# -- 全局选项 ---------------------------------------------------------------
|
||||
language = "zh_CN"
|
||||
master_doc = "index"
|
||||
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
|
||||
source_suffix = {
|
||||
".rst": "restructuredtext",
|
||||
".md": "markdown",
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
pf 统一 CLI 入口
|
||||
================
|
||||
|
||||
所有工具通过 ``pf <tool> [command] [options]`` 调用。工具定义在 ``cli/configs/`` 目录下的 YAML 文件中。
|
||||
|
||||
基本用法
|
||||
--------
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pf # 列出所有可用工具
|
||||
pf filedate # 查看 filedate 工具帮助
|
||||
pf filedate add a.txt # 调用 filedate 的 add 子命令
|
||||
pf gitt c # 调用 gittool 的 c 子命令
|
||||
pf pymake b # 调用 pymake 的 b 别名
|
||||
|
||||
全局选项
|
||||
--------
|
||||
|
||||
所有 YAML 工具支持以下全局选项:
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 25 75
|
||||
|
||||
* - 选项
|
||||
- 说明
|
||||
* - ``--dry-run``
|
||||
- 仅打印执行计划,不执行
|
||||
* - ``--quiet`` / ``-q``
|
||||
- 减少输出,不显示执行过程
|
||||
* - ``--strategy``
|
||||
- 执行策略(``sequential`` / ``thread`` / ``async`` / ``dependency``)
|
||||
* - ``--list``
|
||||
- 列出所有任务名后退出
|
||||
|
||||
默认 ``verbose`` 开启,显示执行过程(任务开始/命令/返回码/任务成功)。``--quiet`` 关闭。
|
||||
|
||||
YAML 配置工具
|
||||
--------------
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 20 15 65
|
||||
|
||||
* - 工具
|
||||
- 别名
|
||||
- 说明
|
||||
* - ``filedate``
|
||||
- ``fd``
|
||||
- 文件日期处理
|
||||
* - ``filelevel``
|
||||
- ``fl``
|
||||
- 文件等级重命名
|
||||
* - ``folderback``
|
||||
- ``fb``
|
||||
- 文件夹备份
|
||||
* - ``folderzip``
|
||||
- ``fz``
|
||||
- 文件夹压缩
|
||||
* - ``gittool``
|
||||
- ``gitt``
|
||||
- Git 执行工具
|
||||
* - ``lscalc``
|
||||
- ``ls``
|
||||
- LS-DYNA 计算工具
|
||||
* - ``packtool``
|
||||
- ``pack``
|
||||
- Python 打包工具
|
||||
* - ``pdftool``
|
||||
- ``pdf``
|
||||
- PDF 文件工具集
|
||||
* - ``piptool``
|
||||
- ``pip``
|
||||
- pip 包管理工具
|
||||
* - ``screenshot``
|
||||
- ``ss``
|
||||
- 截图工具
|
||||
* - ``sshcopyid``
|
||||
- ``ssh``
|
||||
- SSH 密钥部署工具
|
||||
* - ``autofmt``
|
||||
- ``af``
|
||||
- 自动格式化工具
|
||||
* - ``bumpversion``
|
||||
- ``bump``
|
||||
- 版本号自动管理工具
|
||||
|
||||
传统工具
|
||||
--------
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 20 80
|
||||
|
||||
* - 工具
|
||||
- 说明
|
||||
* - ``pymake``
|
||||
- 构建工具(替代 Makefile),如 ``pf pymake b`` 构建
|
||||
* - ``yamlrun``
|
||||
- YAML pipeline 执行器,``pf yamlrun pipeline.yaml``
|
||||
* - ``profiler``
|
||||
- 性能分析
|
||||
* - ``emlman``
|
||||
- 邮件管理
|
||||
* - ``reseticon``
|
||||
- 重置图标缓存
|
||||
|
||||
自定义工具
|
||||
----------
|
||||
|
||||
在 ``cli/configs/`` 目录新建 ``<tool>.yaml`` 即可被 ``pf`` 自动发现:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
# cli/configs/mytool.yaml
|
||||
strategy: sequential
|
||||
variables:
|
||||
MSG: "hello"
|
||||
cli:
|
||||
description: "我的工具"
|
||||
usage: "pf mytool [command]"
|
||||
subcommands:
|
||||
greet:
|
||||
help: "打招呼"
|
||||
jobs:
|
||||
greet:
|
||||
cmd: ["echo", "${MSG}"]
|
||||
|
||||
执行::
|
||||
|
||||
pf mytool greet
|
||||
|
||||
CliRunner(编程式)
|
||||
-------------------
|
||||
|
||||
``CliRunner`` 把多个 Graph 映射为命令行子命令,适合构建项目专属构建工具:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
runner = px.CliRunner(
|
||||
strategy="sequential",
|
||||
description="My Build Tool",
|
||||
graphs={
|
||||
"clean": clean_graph,
|
||||
"build": build_graph,
|
||||
"test": test_graph,
|
||||
},
|
||||
)
|
||||
runner.run_cli() # 解析 sys.argv 并执行
|
||||
|
||||
命令行::
|
||||
|
||||
pf pymake clean
|
||||
pf pymake build --strategy thread
|
||||
pf pymake test --dry-run
|
||||
pf pymake --list
|
||||
pf pymake --quiet
|
||||
@@ -0,0 +1,93 @@
|
||||
执行策略与 run()
|
||||
=================
|
||||
|
||||
``run()`` 是执行入口,支持四种策略:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
report = px.run(
|
||||
graph,
|
||||
strategy="async", # sequential | thread | async | dependency
|
||||
max_workers=8, # thread 策略的线程池大小
|
||||
concurrency_limits={"db": 2}, # 按 concurrency_key 限流
|
||||
dry_run=False, # True = 仅打印计划
|
||||
verbose=True, # True = 打印执行过程
|
||||
on_event=callback, # 状态转换回调
|
||||
state=px.JSONBackend("state.json"), # 断点续跑后端
|
||||
continue_on_error=False, # True = 单任务失败不中断整体
|
||||
)
|
||||
|
||||
策略对比
|
||||
--------
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 18 18 30 16 18
|
||||
|
||||
* - 策略
|
||||
- 并发模型
|
||||
- 适用场景
|
||||
- 同步任务
|
||||
- 异步任务
|
||||
* - ``sequential``
|
||||
- 串行
|
||||
- 调试、CPU 密集
|
||||
- 直接调用
|
||||
- 事件循环
|
||||
* - ``thread``
|
||||
- 线程池
|
||||
- I/O 密集同步
|
||||
- 线程池
|
||||
- 不支持
|
||||
* - ``async``
|
||||
- 事件循环
|
||||
- I/O 密集异步
|
||||
- 卸载到线程池
|
||||
- 事件循环
|
||||
* - ``dependency``
|
||||
- 依赖驱动
|
||||
- 最大化并行度
|
||||
- 卸载到线程池
|
||||
- 事件循环
|
||||
|
||||
所有策略都遵循 ``RetryPolicy``、``timeout``、上下文注入、状态后端、``concurrency_limits``,
|
||||
并发出 ``TaskEvent``(RUNNING/SUCCESS/FAILED/SKIPPED)。``dependency`` 策略无层屏障:
|
||||
任务在其所有硬依赖完成后立即启动。
|
||||
|
||||
上下文注入规则
|
||||
--------------
|
||||
|
||||
按顺序求值:
|
||||
|
||||
1. **标注为 ``Context``** 的参数 → 接收完整上游结果映射
|
||||
2. **名称匹配依赖** 的参数 → 接收该依赖的结果(含软依赖,缺失时注入默认值)
|
||||
3. **``**kwargs``** 参数 → 接收所有依赖结果(dict)
|
||||
4. **``TaskSpec.args`` / ``kwargs``** → 为非依赖参数提供静态值
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
def aggregate(ctx: px.Context) -> Dict[str, Any]:
|
||||
"""ctx 包含所有 depends_on 任务的返回值。"""
|
||||
return dict(ctx)
|
||||
|
||||
def merge(fetch_a: str, fetch_b: str) -> str:
|
||||
"""fetch_a / fetch_b 自动注入。"""
|
||||
return fetch_a + fetch_b
|
||||
|
||||
断点续跑
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from pyflowx import JSONBackend
|
||||
|
||||
backend = JSONBackend("state.json", ttl=3600)
|
||||
report = px.run(graph, strategy="sequential", state=backend)
|
||||
|
||||
``run()`` 内部以 ``backend.batch()`` 包裹整个执行:所有 ``save`` 延迟到运行结束时统一落盘一次。
|
||||
|
||||
缓存键:默认存储键为任务名。配置 ``cache_key`` 函数后,键为 ``"name:cache_key_value"``。
|
||||
|
||||
完整 API 说明详见 :doc:`/api`。
|
||||
@@ -0,0 +1,50 @@
|
||||
Graph —— DAG 构建
|
||||
=================
|
||||
|
||||
``Graph`` 管理任务集合,提供建构建、校验、分层、可视化能力。
|
||||
|
||||
构建方式
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# 图级默认值:TaskSpec 字段为 None 时回退
|
||||
defaults = px.GraphDefaults(retry=px.RetryPolicy(max_attempts=2), timeout=60.0)
|
||||
|
||||
graph = px.Graph.from_specs([...], defaults=defaults) # 整批校验(推荐)
|
||||
|
||||
# 或增量构建
|
||||
graph = px.Graph(defaults=defaults)
|
||||
graph.add(px.TaskSpec("a", fn_a))
|
||||
graph.add(px.TaskSpec("b", fn_b, ("a",)))
|
||||
|
||||
常用方法
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
graph.validate() # 显式校验(环检测)
|
||||
graph.layers() # 拓扑分层(Kahn 算法)
|
||||
graph.to_mermaid() # Mermaid 可视化
|
||||
graph.describe() # 人类可读摘要
|
||||
graph.subgraph(("api",)) # 按标签切片
|
||||
graph.subgraph_by_names(("a", "b")) # 按名称切片
|
||||
graph.map("fetch", [1, 2, 3], lambda i: TaskSpec(f"fetch_{i}", ...)) # 批量 fan-out
|
||||
|
||||
图组合
|
||||
------
|
||||
|
||||
``compose`` / ``GraphComposer`` 把带字符串引用的多个图展开为纯 ``Graph``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
graphs = {
|
||||
"build": px.Graph.from_specs([px.TaskSpec("b", cmd=["echo", "b"])]),
|
||||
"all": px.Graph.from_specs(["build", px.TaskSpec("t", cmd=["echo", "t"])]),
|
||||
}
|
||||
resolved = px.compose(graphs) # "all" 图中的 "build" 引用被展开
|
||||
|
||||
引用格式:``"command_name"``(整个图)或 ``"command_name.task_name"``(特定任务)。
|
||||
``CliRunner`` 内部自动调用 ``compose``。
|
||||
|
||||
完整方法说明详见 :doc:`/api`。
|
||||
@@ -0,0 +1,89 @@
|
||||
TaskSpec —— 任务描述
|
||||
=====================
|
||||
|
||||
``TaskSpec`` 是不可变的任务描述符(``Generic[T]``,返回类型一路传到 ``RunReport``),是唯一需要配置的东西。
|
||||
|
||||
主要参数说明:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
px.TaskSpec(
|
||||
name="fetch_user", # 唯一标识
|
||||
fn=fetch_user, # 同步或异步函数
|
||||
cmd=["curl", "..."], # 或: 执行命令(覆盖 fn)
|
||||
depends_on=("auth",), # 硬依赖(参与拓扑分层)
|
||||
soft_depends_on=("cache",), # 软依赖(仅注入,不参与分层)
|
||||
args=(uid,), # 静态位置参数(追加在注入参数后)
|
||||
kwargs={"timeout": 30}, # 静态关键字参数
|
||||
retry=px.RetryPolicy(max_attempts=3, delay=1.0, backoff=2.0),
|
||||
timeout=30.0, # 超时秒数(None = 不限制)
|
||||
tags=("api", "user"), # 自由标签,用于子图过滤
|
||||
conditions=(is_prod,), # 条件函数列表(全部为 True 才执行)
|
||||
priority=10, # 同层内优先级(高优先执行,默认 0)
|
||||
concurrency_key="db", # 并发分组键(配合 concurrency_limits 限流)
|
||||
cache_key=lambda ctx: str(ctx.get("uid")), # 缓存键函数
|
||||
hooks=px.TaskHooks(pre_run=..., post_run=..., on_failure=...),
|
||||
cwd=Path("/tmp"), # 命令工作目录(仅 cmd 模式)
|
||||
env={"DEBUG": "1"}, # 环境变量覆盖
|
||||
verbose=True, # 打印命令输出(仅 cmd 模式)
|
||||
skip_if_missing=True, # 命令不存在时自动跳过(仅 list[str] cmd)
|
||||
allow_upstream_skip=False, # 上游 SKIPPED/FAILED 时是否仍执行
|
||||
continue_on_error=False, # 本任务失败是否不中断整体
|
||||
)
|
||||
|
||||
两种任务形态
|
||||
------------
|
||||
|
||||
- **函数任务**(``fn``):普通 Python 函数,参数名驱动自动注入
|
||||
- **命令任务**(``cmd``):执行外部命令,支持 ``list[str]``、``str``(shell)、``Callable`` 三种形态
|
||||
|
||||
``skip_if_missing=True`` 时,``list[str]`` 类型的 ``cmd`` 会通过 ``shutil.which`` 检查命令是否存在,不存在则跳过任务(标记为 ``SKIPPED``)而非失败。
|
||||
|
||||
重试策略
|
||||
--------
|
||||
|
||||
``RetryPolicy`` 配置重试次数、延迟、退避:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
retry = px.RetryPolicy(
|
||||
max_attempts=3, # 最大尝试次数
|
||||
delay=1.0, # 初始延迟秒数
|
||||
backoff=2.0, # 退避倍数
|
||||
jitter=0.1, # 随机抖动(避免惊群)
|
||||
retry_on=(ConnectionError,), # 仅对这些异常重试
|
||||
)
|
||||
|
||||
任务钩子
|
||||
--------
|
||||
|
||||
``TaskHooks`` 在任务生命周期触发(异常仅记录,不影响任务状态):
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
hooks = px.TaskHooks(
|
||||
pre_run=lambda spec: print(f"start {spec.name}"),
|
||||
post_run=lambda spec, value: print(f"done {spec.name}"),
|
||||
on_failure=lambda spec, exc: alert(spec.name, exc),
|
||||
)
|
||||
px.TaskSpec("task", fn=work, hooks=hooks)
|
||||
|
||||
任务模板
|
||||
--------
|
||||
|
||||
``task_template`` 工厂批量生成相似 TaskSpec:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
fetch = px.task_template(
|
||||
fn=fetch_url,
|
||||
retry=px.RetryPolicy(max_attempts=5),
|
||||
timeout=30.0,
|
||||
tags=("api",),
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
fetch("users", url="https://api.example.com/users"),
|
||||
fetch("posts", url="https://api.example.com/posts"),
|
||||
])
|
||||
|
||||
完整字段说明详见 :doc:`/api`。
|
||||
@@ -0,0 +1,164 @@
|
||||
YAML 任务编排
|
||||
=============
|
||||
|
||||
PyFlowX 支持 GitHub Actions 风格的声明式 YAML 任务编排,从 YAML 文件直接加载任务图。
|
||||
|
||||
编程式 API
|
||||
----------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# 从 YAML 文件加载任务图
|
||||
graph = px.Graph.from_yaml("pipeline.yaml")
|
||||
report = px.run(graph, strategy="thread")
|
||||
|
||||
# 或用函数式 API
|
||||
graph = px.load_yaml("pipeline.yaml")
|
||||
|
||||
# 从字符串解析
|
||||
graph = px.parse_yaml_string("""
|
||||
jobs:
|
||||
hello:
|
||||
cmd: ["echo", "hello"]
|
||||
""")
|
||||
|
||||
YAML Schema
|
||||
-----------
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
strategy: thread # 图级默认策略
|
||||
defaults: # 图级默认值
|
||||
retry: {max_attempts: 3}
|
||||
verbose: true
|
||||
env: {CI: "true"}
|
||||
|
||||
variables: # 变量定义 (可在 cmd/env 中 ${VAR} 引用)
|
||||
OUTPUT: "dist"
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
cmd: ["git", "clone", "..."]
|
||||
runs-on: linux
|
||||
|
||||
build:
|
||||
needs: [setup] # 依赖列表
|
||||
cmd: ["python", "-m", "build"]
|
||||
timeout: 300
|
||||
retry: {max_attempts: 2, delay: 1.0}
|
||||
|
||||
test:
|
||||
needs: [build]
|
||||
cmd: ["python${{ matrix.version }}", "-m", "pytest"]
|
||||
strategy:
|
||||
matrix: # 笛卡尔积展开为 6 个任务
|
||||
version: ["3.8", "3.9", "3.10"]
|
||||
os: ["linux", "macos"]
|
||||
if: "env.CI" # 条件: 环境变量存在
|
||||
|
||||
lint:
|
||||
needs: [build]
|
||||
cmd: ["ruff", "check"]
|
||||
if: "env.CI == 'true'"
|
||||
|
||||
deploy:
|
||||
needs: [test, lint] # 矩阵依赖自动展开
|
||||
cmd: ["twine", "upload"]
|
||||
if: "env.DEPLOY_TOKEN != ''"
|
||||
allow-upstream-skip: true
|
||||
concurrency-key: deploy_lock
|
||||
|
||||
字段映射
|
||||
--------
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 30 30 40
|
||||
|
||||
* - YAML 字段
|
||||
- TaskSpec 字段
|
||||
- 说明
|
||||
* - ``jobs.<id>``
|
||||
- ``name``
|
||||
- job ID 作为任务名
|
||||
* - ``cmd`` / ``run``
|
||||
- ``cmd``
|
||||
- ``cmd`` 为列表形式,``run`` 为 shell 字符串
|
||||
* - ``needs``
|
||||
- ``depends_on``
|
||||
- 依赖列表(矩阵任务自动展开)
|
||||
* - ``if``
|
||||
- ``conditions``
|
||||
- ``success()`` / ``always()`` / ``env.VAR`` / ``env.VAR == 'x'``
|
||||
* - ``strategy.matrix``
|
||||
- 矩阵扇出
|
||||
- 笛卡尔积展开为多个任务
|
||||
* - ``${{ matrix.key }}``
|
||||
- 占位符
|
||||
- 在 cmd/run/cwd/env 中替换
|
||||
* - ``timeout``
|
||||
- ``timeout``
|
||||
- 超时秒数
|
||||
* - ``retry``
|
||||
- ``retry``
|
||||
- ``{max_attempts, delay, backoff, jitter}``
|
||||
* - ``cwd``
|
||||
- ``cwd``
|
||||
- 工作目录
|
||||
* - ``env``
|
||||
- ``env``
|
||||
- 环境变量
|
||||
* - ``verbose``
|
||||
- ``verbose``
|
||||
- 详细输出
|
||||
* - ``continue-on-error``
|
||||
- ``continue_on_error``
|
||||
- 失败不中止整图
|
||||
* - ``skip-if-missing``
|
||||
- ``skip_if_missing``
|
||||
- 命令不存在时跳过
|
||||
* - ``allow-upstream-skip``
|
||||
- ``allow_upstream_skip``
|
||||
- 上游跳过时仍执行
|
||||
* - ``priority``
|
||||
- ``priority``
|
||||
- 同层优先级
|
||||
* - ``concurrency-key``
|
||||
- ``concurrency_key``
|
||||
- 并发限制键
|
||||
* - ``tags``
|
||||
- ``tags``
|
||||
- 自由标签
|
||||
* - ``runs-on``
|
||||
- ``tags``(追加)
|
||||
- 运行环境标签
|
||||
|
||||
CLI 配置段(``cli:``)
|
||||
----------------------
|
||||
|
||||
工具 YAML 还可定义 ``cli:`` 段,声明命令行参数 schema,由 ``pf`` 自动解析:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
cli:
|
||||
description: "FileDate - 文件日期处理工具"
|
||||
usage: "pf filedate <command> [files...]"
|
||||
subcommands:
|
||||
add:
|
||||
help: "添加日期前缀"
|
||||
positional:
|
||||
- name: FILES
|
||||
nargs: "+"
|
||||
type: path
|
||||
help: "文件路径"
|
||||
options:
|
||||
- name: CLEAR
|
||||
flag: "--clear"
|
||||
action: store_true
|
||||
help: "清除已有日期前缀"
|
||||
|
||||
支持的 ``type``:``str`` / ``int`` / ``float`` / ``path``。
|
||||
|
||||
完整 API 说明详见 :doc:`/api`。
|
||||
@@ -0,0 +1,54 @@
|
||||
PyFlowX 文档
|
||||
============
|
||||
|
||||
PyFlowX 是一个轻量、类型安全的 DAG 任务调度器:**参数名就是依赖声明**。
|
||||
无需装饰器、无需样板包装器,写一个普通函数,框架按参数名自动注入上游结果。
|
||||
|
||||
特性
|
||||
----
|
||||
|
||||
- **零样板** —— 参数名即依赖,框架自动注入上游结果
|
||||
- **四种执行策略** —— sequential(串行)、thread(线程池)、async(事件循环)、dependency(依赖驱动,最大化并行)
|
||||
- **类型安全** —— ``TaskSpec[T]`` 把返回类型一路传到 ``RunReport``
|
||||
- **DAG 校验** —— 构建时即时校验重名、缺失依赖、环
|
||||
- **自动分层** —— Kahn 算法分组,同层任务可并行
|
||||
- **重试与超时** —— 每个任务独立配置 ``RetryPolicy`` 与 ``timeout``
|
||||
- **并发限制** —— ``concurrency_key`` + ``concurrency_limits`` 按组限流
|
||||
- **断点续跑** —— ``MemoryBackend`` / ``JSONBackend``,成功结果可缓存复用
|
||||
- **命令任务** —— ``cmd`` 参数直接执行外部命令
|
||||
- **条件执行** —— ``conditions`` 按平台、环境变量等条件跳过任务
|
||||
- **pf 统一 CLI** —— ``pf <tool> [command]`` 调用所有工具
|
||||
- **最小依赖** —— ``rich`` + ``typer`` + ``typing-extensions``(3.13 以下)
|
||||
|
||||
文档导航
|
||||
--------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: 入门
|
||||
|
||||
installation
|
||||
quickstart
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: 用户指南
|
||||
|
||||
guide/task
|
||||
guide/graph
|
||||
guide/execution
|
||||
guide/cli
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: 参考
|
||||
|
||||
api
|
||||
changelog
|
||||
|
||||
索引
|
||||
----
|
||||
|
||||
* :ref:`genindex`
|
||||
* :ref:`modindex`
|
||||
* :ref:`search`
|
||||
@@ -0,0 +1,51 @@
|
||||
安装
|
||||
====
|
||||
|
||||
PyFlowX 支持 Python 3.10+,运行时依赖 ``rich``、``typer``、``pyyaml`` 与 ``typing-extensions``(3.13 以下)。
|
||||
|
||||
pip 安装
|
||||
--------
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install pyflowx
|
||||
|
||||
uv 安装
|
||||
-------
|
||||
|
||||
推荐使用 `uv <https://docs.astral.sh/uv/>`_:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
uv add pyflowx
|
||||
|
||||
可选依赖
|
||||
--------
|
||||
|
||||
``office`` —— PDF/图片处理(pdftool、screenshot 等工具需要):
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install pyflowx[office]
|
||||
|
||||
``dev`` —— 开发工具链(ruff、pyrefly、pytest、tox 等):
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install pyflowx[dev]
|
||||
|
||||
验证安装
|
||||
--------
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pf --version
|
||||
|
||||
输出示例::
|
||||
|
||||
PyFlowX 0.4.5
|
||||
|
||||
下一步
|
||||
------
|
||||
|
||||
前往 :doc:`quickstart` 开始使用。
|
||||
@@ -0,0 +1,87 @@
|
||||
快速上手
|
||||
========
|
||||
|
||||
核心思想:**参数名即依赖**。写一个普通函数,参数名匹配上游任务名,框架自动注入结果。
|
||||
|
||||
最小示例
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
def extract() -> list[int]:
|
||||
return [1, 2, 3]
|
||||
|
||||
# 参数名 extract 自动匹配上游任务名 → 自动注入
|
||||
def double(extract: list[int]) -> list[int]:
|
||||
return [x * 2 for x in extract]
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("extract", extract),
|
||||
px.TaskSpec("double", double, ("extract",)),
|
||||
])
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
print(report["double"]) # [2, 4, 6]
|
||||
|
||||
三种任务形态
|
||||
------------
|
||||
|
||||
1. **函数任务**(``fn``):普通 Python 函数,参数名驱动自动注入
|
||||
2. **命令任务**(``cmd``):执行外部命令,支持 ``list[str]`` / ``str``(shell)/ ``Callable``
|
||||
3. **YAML 声明式**:从 YAML 文件加载任务图
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("list", cmd=["ls", "-la"]),
|
||||
px.TaskSpec("greet", fn=lambda: "hello"),
|
||||
])
|
||||
|
||||
执行策略
|
||||
--------
|
||||
|
||||
PyFlowX 提供四种执行策略:
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 20 20 60
|
||||
|
||||
* - 策略
|
||||
- 并发模型
|
||||
- 适用场景
|
||||
* - ``sequential``
|
||||
- 串行
|
||||
- 调试、CPU 密集
|
||||
* - ``thread``
|
||||
- 线程池
|
||||
- I/O 密集同步
|
||||
* - ``async``
|
||||
- 事件循环
|
||||
- I/O 密集异步
|
||||
* - ``dependency``
|
||||
- 依赖驱动
|
||||
- 最大化并行度(默认推荐)
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
report = px.run(graph, strategy="dependency")
|
||||
|
||||
结果访问
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
report["task_name"] # 任务返回值
|
||||
report.result_of("task_name") # 完整 TaskResult
|
||||
report.success # 整体是否成功
|
||||
report.summary() # 统计字典
|
||||
report.failed_tasks() # 失败任务名列表
|
||||
|
||||
下一步
|
||||
------
|
||||
|
||||
- :doc:`guide/task` —— TaskSpec 详细配置
|
||||
- :doc:`guide/yaml` —— YAML 声明式任务编排
|
||||
- :doc:`guide/cli` —— ``pf`` 统一 CLI 入口
|
||||
+26
-43
@@ -6,42 +6,27 @@ classifiers = [
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.14",
|
||||
"Topic :: Software Development :: Libraries :: Application Frameworks",
|
||||
]
|
||||
dependencies = ["graphlib_backport >= 1.0.0; python_version < '3.9'"]
|
||||
dependencies = [
|
||||
"pyyaml>=6.0.1",
|
||||
"rich>=13.7.0",
|
||||
"typer>=0.24.0",
|
||||
"typing-extensions>=4.13.2; python_version < '3.13'",
|
||||
]
|
||||
description = "Lightweight, type-safe DAG task scheduler with multi-strategy execution."
|
||||
keywords = ["async", "dag", "scheduler", "task", "workflow"]
|
||||
license = { text = "MIT" }
|
||||
name = "pyflowx"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.8"
|
||||
version = "0.1.12"
|
||||
requires-python = ">=3.10"
|
||||
version = "0.4.9"
|
||||
|
||||
[project.scripts]
|
||||
autofmt = "pyflowx.cli.autofmt:main"
|
||||
bumpver = "pyflowx.cli.bumpversion:main"
|
||||
clr = "pyflowx.cli.clearscreen:main"
|
||||
emlman = "pyflowx.cli.emlmanager:main"
|
||||
envpy = "pyflowx.cli.envpy:main"
|
||||
envqt = "pyflowx.cli.envqt:main"
|
||||
envrs = "pyflowx.cli.envrs:main"
|
||||
filedate = "pyflowx.cli.filedate:main"
|
||||
filelvl = "pyflowx.cli.filelevel:main"
|
||||
foldback = "pyflowx.cli.folderback:main"
|
||||
foldzip = "pyflowx.cli.folderzip:main"
|
||||
gitt = "pyflowx.cli.gittool:main"
|
||||
hfdown = "pyflowx.cli.hfdownload:main"
|
||||
lscalc = "pyflowx.cli.lscalc:main"
|
||||
packtool = "pyflowx.cli.packtool:main"
|
||||
pdftool = "pyflowx.cli.pdftool:main"
|
||||
piptool = "pyflowx.cli.piptool:main"
|
||||
pymake = "pyflowx.cli.pymake:main"
|
||||
scrcap = "pyflowx.cli.screenshot:main"
|
||||
sshcopy = "pyflowx.cli.sshcopyid:main"
|
||||
taskk = "pyflowx.cli.taskkill:main"
|
||||
wch = "pyflowx.cli.which:main"
|
||||
emlman = "pyflowx.cli.legacy.emlmanager:main"
|
||||
pf = "pyflowx.cli.pf:main"
|
||||
pxp = "pyflowx.cli.legacy.profiler:main"
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
@@ -59,6 +44,8 @@ dev = [
|
||||
"tox-uv>=1.13.1",
|
||||
"tox>=4.25.0",
|
||||
]
|
||||
docs = ["myst-parser>=3.0", "sphinx-rtd-theme>=2.0", "sphinx>=7.0"]
|
||||
fast = ["orjson>=3.10.0"]
|
||||
office = [
|
||||
"pillow>=10.4.0",
|
||||
"pymupdf>=1.24.11",
|
||||
@@ -70,6 +57,9 @@ office = [
|
||||
build-backend = "hatchling.build"
|
||||
requires = ["hatchling"]
|
||||
|
||||
[tool.uv]
|
||||
required-version = ">=0.5.0"
|
||||
|
||||
[[tool.uv.index]]
|
||||
default = true
|
||||
url = "https://mirrors.aliyun.com/pypi/simple/"
|
||||
@@ -84,12 +74,12 @@ packages = ["src/pyflowx"]
|
||||
pyflowx = { workspace = true }
|
||||
|
||||
[dependency-groups]
|
||||
dev = ["pyflowx[dev,office]"]
|
||||
dev = ["pyflowx[dev,docs,fast,office]"]
|
||||
|
||||
[tool.coverage.run]
|
||||
branch = true
|
||||
concurrency = ["thread"]
|
||||
omit = ["src/pyflowx/examples/*", "tests/*"]
|
||||
omit = ["src/pyflowx/cli/*", "tests/*"]
|
||||
source = ["pyflowx"]
|
||||
|
||||
[tool.coverage.report]
|
||||
@@ -99,7 +89,7 @@ exclude_lines = [
|
||||
"pragma: no cover",
|
||||
"raise NotImplementedError",
|
||||
]
|
||||
fail_under = 80
|
||||
fail_under = 95
|
||||
show_missing = true
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
@@ -109,16 +99,7 @@ markers = ["slow: marks tests as slow (deselect with
|
||||
# Ruff 配置 - 与 .pre-commit-config.yaml 保持一致
|
||||
[tool.ruff]
|
||||
line-length = 120
|
||||
target-version = "py38"
|
||||
|
||||
[tool.ruff.format]
|
||||
# 使用双引号
|
||||
quote-style = "double"
|
||||
# 缩进使用空格
|
||||
indent-style = "space"
|
||||
# 保留尾随逗号
|
||||
skip-magic-trailing-comma = false
|
||||
# 行长度由 [tool.ruff] 中的 line-length 控制
|
||||
target-version = "py310"
|
||||
|
||||
[tool.ruff.lint]
|
||||
ignore = [
|
||||
@@ -151,9 +132,11 @@ select = [
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"**/tests/**" = ["ARG001", "ARG002"]
|
||||
"**/tests/**" = ["ARG001", "ARG002"]
|
||||
"src/pyflowx/tools.py" = ["PLR0911"]
|
||||
|
||||
[tool.pyrefly]
|
||||
preset = "basic"
|
||||
preset = "strict"
|
||||
project-includes = ["**/*.ipynb", "**/*.py*"]
|
||||
python-version = "3.8"
|
||||
python-version = "3.10"
|
||||
search-path = ["."]
|
||||
|
||||
+149
-52
@@ -2,25 +2,35 @@
|
||||
|
||||
公共 API
|
||||
--------
|
||||
* :func:`task` / :func:`cmd` —— 装饰器/工厂,从函数或命令快速创建 TaskSpec。
|
||||
* :func:`graph` —— 快捷构造图(等价于 ``Graph.from_specs``,接受可变参数)。
|
||||
* :class:`TaskSpec` —— 不可变任务描述符(唯一需要配置的东西)。
|
||||
* :class:`Graph` —— 由一组 spec 构建的 DAG;负责校验、分层、可视化。
|
||||
* :func:`run` —— 以 ``sequential`` / ``thread`` / ``async`` 策略执行图。
|
||||
* :func:`run` ——以 ``sequential`` / ``thread`` / ``async`` / ``dependency``
|
||||
策略执行图(默认 ``dependency``)。
|
||||
* :class:`RunReport` —— 类型化、可查询的运行结果。
|
||||
* :class:`Context` —— 整体上下文注入的标注标记。
|
||||
* 状态后端::class:`StateBackend`、:class:`MemoryBackend`、:class:`JSONBackend`。
|
||||
* :class:`RetryPolicy` —— 重试策略(max_attempts/delay/backoff/jitter/retry_on)。
|
||||
* :class:`TaskHooks` —— 任务生命周期钩子(pre_run/post_run/on_failure)。
|
||||
* :class:`GraphDefaults` —— 图级默认值。
|
||||
* :func:`compose` —— 编程式组合多图。
|
||||
* :func:`switch` / :func:`branch` —— 条件分支 DAG 构造器。
|
||||
* :func:`task_template` —— 批量生成相似 TaskSpec 的工厂。
|
||||
* :func:`sh` —— Shell 命令执行辅助(支持 ``list[str]`` / ``str``,统一中文错误处理)。
|
||||
* 状态后端::class:`StateBackend`、:class:`MemoryBackend`、:class:`JSONBackend`、:class:`SQLiteBackend`。
|
||||
|
||||
快速上手
|
||||
--------
|
||||
import pyflowx as px
|
||||
|
||||
@px.task
|
||||
def extract() -> list[int]: return [1, 2, 3]
|
||||
|
||||
@px.task
|
||||
def double(extract: list[int]) -> list[int]: return [x * 2 for x in extract]
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("extract", extract),
|
||||
px.TaskSpec("double", double, ("extract",)),
|
||||
])
|
||||
report = px.run(graph, strategy="sequential")
|
||||
graph = px.graph(extract, double) # double 自动依赖 extract
|
||||
report = px.run(graph)
|
||||
print(report["double"]) # [2, 4, 6]
|
||||
|
||||
命令行任务示例
|
||||
@@ -28,45 +38,28 @@
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import IS_WINDOWS, BuiltinConditions
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
# 使用命令列表
|
||||
px.TaskSpec("list_files", cmd=["ls", "-la"]),
|
||||
# 使用 shell 命令
|
||||
px.TaskSpec("check_git", cmd="git status"),
|
||||
# 条件执行:仅在 Windows 上运行
|
||||
px.TaskSpec(
|
||||
"win_only",
|
||||
cmd=["dir"],
|
||||
conditions=(IS_WINDOWS,)
|
||||
),
|
||||
# 条件执行:仅在 git 已安装时运行
|
||||
px.TaskSpec(
|
||||
"git_check",
|
||||
cmd=["git", "--version"],
|
||||
conditions=(BuiltinConditions.HAS_INSTALLED("git"),)
|
||||
),
|
||||
# 命令不存在时自动跳过(而非失败)
|
||||
px.TaskSpec(
|
||||
"optional_build",
|
||||
cmd=["maturin", "build"],
|
||||
skip_if_missing=True
|
||||
),
|
||||
])
|
||||
graph = px.graph(
|
||||
px.cmd(["ls", "-la"]),
|
||||
px.cmd("git status", name="check_git"),
|
||||
px.cmd(["dir"], name="win_only", conditions=(IS_WINDOWS,)),
|
||||
px.cmd(["git", "--version"], name="git_check",
|
||||
conditions=(BuiltinConditions.HAS_INSTALLED("git"),)),
|
||||
px.cmd(["maturin", "build"], name="optional_build", skip_if_missing=True),
|
||||
)
|
||||
report = px.run(graph)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .conditions import (
|
||||
IS_LINUX,
|
||||
IS_MACOS,
|
||||
IS_POSIX,
|
||||
IS_WINDOWS,
|
||||
BuiltinConditions,
|
||||
Condition,
|
||||
Constants,
|
||||
)
|
||||
from typing import Any
|
||||
|
||||
from . import fileops
|
||||
from .cancellation import CancelToken
|
||||
from .command import run_command
|
||||
from .compose import GraphComposer, compose
|
||||
from .conditions import IS_LINUX, IS_MACOS, IS_POSIX, IS_WINDOWS, BuiltinConditions, Condition, Constants
|
||||
from .context import Context, build_call_args, describe_injection
|
||||
from .diagnostics import DependencyChain, DiagnosticReport, FailureCluster, diagnose
|
||||
from .errors import (
|
||||
CycleError,
|
||||
DuplicateTaskError,
|
||||
@@ -77,53 +70,157 @@ from .errors import (
|
||||
TaskFailedError,
|
||||
TaskTimeoutError,
|
||||
)
|
||||
from .executors import Strategy, run
|
||||
from .graph import Graph
|
||||
from .executors import Strategy, run, run_iter
|
||||
from .graph import Graph, GraphDefaults
|
||||
from .history import RunHistory
|
||||
from .imaging import image_pipeline
|
||||
from .monitoring import MetricsCollector, health_check, start_metrics_server
|
||||
from .notification import (
|
||||
ALL_LEVELS,
|
||||
CallbackNotifier,
|
||||
DingTalkNotifier,
|
||||
DiscordNotifier,
|
||||
FeishuNotifier,
|
||||
NotificationLevel,
|
||||
Notifier,
|
||||
SlackNotifier,
|
||||
TelegramNotifier,
|
||||
WebhookNotifier,
|
||||
WeChatNotifier,
|
||||
)
|
||||
from .pipelines import branch, command_chain, data_pipeline, fan_out_fan_in, switch
|
||||
from .profiling import ProfileReport, TaskProfile
|
||||
from .progress import ProgressCallback, RichProgressMonitor
|
||||
from .report import RunReport
|
||||
from .runner import CliExitCode, CliRunner
|
||||
from .storage import JSONBackend, MemoryBackend, StateBackend
|
||||
from .task import TaskCmd, TaskEvent, TaskResult, TaskSpec, TaskStatus
|
||||
from .shell import sh
|
||||
from .storage import JSONBackend, MemoryBackend, SQLiteBackend, StateBackend
|
||||
from .task import (
|
||||
CacheKeyFn,
|
||||
LoopSpec,
|
||||
RetryPolicy,
|
||||
TaskCmd,
|
||||
TaskEvent,
|
||||
TaskHooks,
|
||||
TaskResult,
|
||||
TaskSpec,
|
||||
TaskStatus,
|
||||
cmd,
|
||||
task,
|
||||
task_template,
|
||||
)
|
||||
from .tools import ToolSpec, list_subcommands, list_tools, run_tool, tool
|
||||
from .yaml_loader import load_yaml, parse_yaml_string
|
||||
|
||||
__version__ = "0.4.9"
|
||||
|
||||
|
||||
def graph(
|
||||
*specs: TaskSpec[Any] | str,
|
||||
defaults: GraphDefaults | None = None,
|
||||
namespace: str | None = None,
|
||||
) -> Graph:
|
||||
"""快捷构造图:等价于 :meth:`Graph.from_specs`,接受可变参数而非列表。
|
||||
|
||||
对 ``depends_on`` 为空的纯 fn 任务,自动从必需参数名推断依赖
|
||||
(匹配图中任务名的参数被加入 ``depends_on``)。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> import pyflowx as px
|
||||
>>> @px.task
|
||||
... def extract() -> list[int]: return [1, 2, 3]
|
||||
>>> @px.task
|
||||
... def double(extract: list[int]) -> list[int]: return [x * 2 for x in extract]
|
||||
>>> g = px.graph(extract, double) # double 自动依赖 extract
|
||||
"""
|
||||
return Graph.from_specs(specs, defaults=defaults, namespace=namespace)
|
||||
|
||||
__version__ = "0.1.12"
|
||||
|
||||
__all__ = [
|
||||
"ALL_LEVELS",
|
||||
"IS_LINUX",
|
||||
"IS_MACOS",
|
||||
"IS_POSIX",
|
||||
"IS_WINDOWS",
|
||||
"BuiltinConditions",
|
||||
"CacheKeyFn",
|
||||
"CallbackNotifier",
|
||||
"CancelToken",
|
||||
"CliExitCode",
|
||||
# CLI 运行器
|
||||
"CliRunner",
|
||||
# 条件判断
|
||||
"Condition",
|
||||
"Constants",
|
||||
"Context",
|
||||
"CycleError",
|
||||
"DependencyChain",
|
||||
"DiagnosticReport",
|
||||
"DingTalkNotifier",
|
||||
"DiscordNotifier",
|
||||
"DuplicateTaskError",
|
||||
"FailureCluster",
|
||||
"FeishuNotifier",
|
||||
"Graph",
|
||||
"GraphComposer",
|
||||
"GraphDefaults",
|
||||
"InjectionError",
|
||||
"JSONBackend",
|
||||
"LoopSpec",
|
||||
"MemoryBackend",
|
||||
"MetricsCollector",
|
||||
"MissingDependencyError",
|
||||
# 错误
|
||||
"NotificationLevel",
|
||||
"Notifier",
|
||||
"ProfileReport",
|
||||
"ProgressCallback",
|
||||
"PyFlowXError",
|
||||
"RetryPolicy",
|
||||
"RichProgressMonitor",
|
||||
"RunHistory",
|
||||
"RunReport",
|
||||
# 状态后端
|
||||
"SQLiteBackend",
|
||||
"SlackNotifier",
|
||||
"StateBackend",
|
||||
"StorageError",
|
||||
"Strategy",
|
||||
"TaskCmd",
|
||||
"TaskEvent",
|
||||
"TaskFailedError",
|
||||
"TaskHooks",
|
||||
"TaskProfile",
|
||||
"TaskResult",
|
||||
# 核心类型
|
||||
"TaskSpec",
|
||||
"TaskStatus",
|
||||
"TaskTimeoutError",
|
||||
# 辅助(高级)
|
||||
"TelegramNotifier",
|
||||
"ToolSpec",
|
||||
"WeChatNotifier",
|
||||
"WebhookNotifier",
|
||||
"branch",
|
||||
"build_call_args",
|
||||
"cmd",
|
||||
"command_chain",
|
||||
"compose",
|
||||
"data_pipeline",
|
||||
"describe_injection",
|
||||
# 执行
|
||||
"diagnose",
|
||||
"fan_out_fan_in",
|
||||
"fileops",
|
||||
"graph",
|
||||
"health_check",
|
||||
"image_pipeline",
|
||||
"list_subcommands",
|
||||
"list_tools",
|
||||
"load_yaml",
|
||||
"parse_yaml_string",
|
||||
"run",
|
||||
"run_command",
|
||||
"run_iter",
|
||||
"run_tool",
|
||||
"sh",
|
||||
"start_metrics_server",
|
||||
"switch",
|
||||
"task",
|
||||
"task_template",
|
||||
"tool",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""JSON 序列化抽象层。
|
||||
|
||||
优先使用 orjson(性能更高),回退到标准库 json。对外提供与标准库
|
||||
json 兼容的 ``dumps`` / ``loads`` / ``dump`` / ``load`` / ``JSONDecodeError``
|
||||
接口,使调用方无感知切换。
|
||||
|
||||
orjson 与标准库的差异由本模块吸收:
|
||||
|
||||
- ``dumps`` 返回 ``str``(orjson 原生返回 ``bytes``,此处统一 decode)
|
||||
- ``ensure_ascii`` 参数对 orjson 无效(始终输出 UTF-8),此处忽略
|
||||
- ``indent`` 映射为 ``orjson.OPT_INDENT_2``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["JSONDecodeError", "dump", "dumps", "load", "loads"]
|
||||
|
||||
from typing import IO, Any
|
||||
|
||||
try:
|
||||
import orjson # type: ignore[import-not-found]
|
||||
|
||||
_HAS_ORJSON = True
|
||||
except ImportError: # pragma: no cover
|
||||
_HAS_ORJSON = False
|
||||
|
||||
|
||||
if _HAS_ORJSON:
|
||||
JSONDecodeError = orjson.JSONDecodeError # type: ignore[possibly-unbound]
|
||||
_OPT_NON_STR_KEYS = orjson.OPT_NON_STR_KEYS # type: ignore[possibly-unbound]
|
||||
|
||||
def dumps(
|
||||
obj: Any,
|
||||
*,
|
||||
ensure_ascii: bool = False, # noqa: ARG001 — 兼容标准库签名
|
||||
indent: int | None = None,
|
||||
default: Any = None,
|
||||
**_kwargs: Any,
|
||||
) -> str:
|
||||
"""序列化为 JSON 字符串(orjson 后端)。"""
|
||||
opts = _OPT_NON_STR_KEYS
|
||||
if indent is not None:
|
||||
opts |= orjson.OPT_INDENT_2 # type: ignore[possibly-unbound]
|
||||
return orjson.dumps(obj, default=default, option=opts).decode("utf-8") # type: ignore[possibly-unbound]
|
||||
|
||||
def loads(s: str | bytes) -> Any:
|
||||
"""从 JSON 字符串/字节反序列化(orjson 后端)。"""
|
||||
return orjson.loads(s) # type: ignore[possibly-unbound]
|
||||
|
||||
def dump(
|
||||
obj: Any,
|
||||
fh: IO[str],
|
||||
*,
|
||||
ensure_ascii: bool = False,
|
||||
indent: int | None = None,
|
||||
default: Any = None,
|
||||
**_kwargs: Any,
|
||||
) -> None:
|
||||
"""序列化并写入文件句柄(orjson 后端)。"""
|
||||
fh.write(dumps(obj, ensure_ascii=ensure_ascii, indent=indent, default=default))
|
||||
|
||||
def load(fh: IO[str]) -> Any:
|
||||
"""从文件句柄反序列化(orjson 后端)。"""
|
||||
return orjson.loads(fh.read()) # type: ignore[possibly-unbound]
|
||||
|
||||
else: # pragma: no cover
|
||||
import json as _stdlib
|
||||
|
||||
JSONDecodeError = _stdlib.JSONDecodeError
|
||||
|
||||
def dumps(
|
||||
obj: Any,
|
||||
*,
|
||||
ensure_ascii: bool = False,
|
||||
indent: int | None = None,
|
||||
default: Any = None,
|
||||
**_kwargs: Any,
|
||||
) -> str:
|
||||
"""序列化为 JSON 字符串(标准库后端)。"""
|
||||
return _stdlib.dumps(obj, ensure_ascii=ensure_ascii, indent=indent, default=default)
|
||||
|
||||
def loads(s: str | bytes) -> Any:
|
||||
"""从 JSON 字符串/字节反序列化(标准库后端)。"""
|
||||
return _stdlib.loads(s)
|
||||
|
||||
def dump(
|
||||
obj: Any,
|
||||
fh: IO[str],
|
||||
*,
|
||||
ensure_ascii: bool = False,
|
||||
indent: int | None = None,
|
||||
default: Any = None,
|
||||
**_kwargs: Any,
|
||||
) -> None:
|
||||
"""序列化并写入文件句柄(标准库后端)。"""
|
||||
_stdlib.dump(obj, fh, ensure_ascii=ensure_ascii, indent=indent, default=default)
|
||||
|
||||
def load(fh: IO[str]) -> Any:
|
||||
"""从文件句柄反序列化(标准库后端)。"""
|
||||
return _stdlib.load(fh)
|
||||
@@ -0,0 +1,114 @@
|
||||
"""任务取消与优雅停止。
|
||||
|
||||
提供 :class:`CancelToken` 作为线程安全的取消信号,可传入 :func:`pyflowx.run`
|
||||
实现外部取消或响应 ``KeyboardInterrupt``。
|
||||
|
||||
设计要点
|
||||
--------
|
||||
* 基于 :class:`threading.Event`,跨线程/协程安全。
|
||||
* 轻量:不引入额外线程或回调,仅维护一个事件标志。
|
||||
* 可组合:多个 CancelToken 可通过 :meth:`link` 联动。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Any
|
||||
|
||||
__all__ = ["CancelToken"]
|
||||
|
||||
|
||||
class CancelToken:
|
||||
"""线程安全的取消令牌。
|
||||
|
||||
用法::
|
||||
|
||||
token = CancelToken()
|
||||
# 在另一个线程中:
|
||||
token.cancel()
|
||||
|
||||
# 传入 run():
|
||||
report = px.run(graph, cancel_event=token)
|
||||
|
||||
也可直接用 :class:`threading.Event` 代替——本类仅提供语义更清晰的 API。
|
||||
"""
|
||||
|
||||
__slots__ = ("_event", "_reason")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._event = threading.Event()
|
||||
self._reason: str | None = None
|
||||
|
||||
def cancel(self, reason: str | None = None) -> None:
|
||||
"""设置取消信号。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
reason:
|
||||
可选的取消原因描述,供日志或调试使用。
|
||||
"""
|
||||
if reason is not None:
|
||||
self._reason = reason
|
||||
self._event.set()
|
||||
|
||||
@property
|
||||
def is_cancelled(self) -> bool:
|
||||
"""是否已取消。"""
|
||||
return self._event.is_set()
|
||||
|
||||
@property
|
||||
def reason(self) -> str | None:
|
||||
"""取消原因(未取消时为 None)。"""
|
||||
return self._reason
|
||||
|
||||
def wait(self, timeout: float | None = None) -> bool:
|
||||
"""等待取消信号。
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
``True`` 表示已取消,``False`` 表示超时。
|
||||
"""
|
||||
return self._event.wait(timeout)
|
||||
|
||||
def link(self, event: threading.Event) -> None:
|
||||
"""联动一个 ``threading.Event``:当 ``event`` 被设置时,本 token 也被取消。
|
||||
|
||||
适用于将标准 ``threading.Event`` 桥接到 CancelToken。
|
||||
"""
|
||||
|
||||
def _watcher() -> None:
|
||||
event.wait()
|
||||
self.cancel()
|
||||
|
||||
t = threading.Thread(target=_watcher, daemon=True)
|
||||
t.start()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
status = "cancelled" if self._event.is_set() else "active"
|
||||
reason = f", reason={self._reason!r}" if self._reason else ""
|
||||
return f"CancelToken({status}{reason})"
|
||||
|
||||
# 兼容 threading.Event 接口:run() 内部用 .is_set() 检查
|
||||
def is_set(self) -> bool:
|
||||
"""兼容 :meth:`threading.Event.is_set`。"""
|
||||
return self._event.is_set()
|
||||
|
||||
# 允许 ``CancelToken | threading.Event`` 统一类型检查
|
||||
def __bool__(self) -> bool:
|
||||
return self._event.is_set()
|
||||
|
||||
|
||||
def _is_cancelled(token: Any) -> bool:
|
||||
"""统一检查 CancelToken 或 threading.Event 是否已取消。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
token:
|
||||
``None``、``CancelToken`` 或 ``threading.Event``。
|
||||
"""
|
||||
if token is None:
|
||||
return False
|
||||
if isinstance(token, CancelToken):
|
||||
return token.is_cancelled
|
||||
return bool(token.is_set())
|
||||
@@ -1,78 +0,0 @@
|
||||
"""CLI 工具模块.
|
||||
|
||||
提供各种命令行工具的入口点.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# 自动格式化工具
|
||||
from pyflowx.cli.autofmt import main as autofmt_main
|
||||
from pyflowx.cli.bumpversion import main as bumpversion_main
|
||||
from pyflowx.cli.clearscreen import main as clearscreen_main
|
||||
|
||||
# EML 邮件管理工具
|
||||
from pyflowx.cli.emlmanager import main as emlmanager_main
|
||||
|
||||
# EML 邮件管理工具
|
||||
from pyflowx.cli.emlmanager import main as emlmanager_web_main
|
||||
from pyflowx.cli.envpy import main as envpy_main
|
||||
from pyflowx.cli.envqt import main as envqt_main
|
||||
from pyflowx.cli.envrs import main as envrs_main
|
||||
|
||||
# 文件工具
|
||||
from pyflowx.cli.filedate import main as filedate_main
|
||||
from pyflowx.cli.filelevel import main as filelevel_main
|
||||
from pyflowx.cli.folderback import main as folderback_main
|
||||
from pyflowx.cli.folderzip import main as folderzip_main
|
||||
|
||||
# Git 工具
|
||||
from pyflowx.cli.gittool import main as gittool_main
|
||||
|
||||
# 仿真工具
|
||||
from pyflowx.cli.lscalc import main as lscalc_main
|
||||
|
||||
# 打包工具
|
||||
from pyflowx.cli.packtool import main as packtool_main
|
||||
|
||||
# PDF 工具
|
||||
from pyflowx.cli.pdftool import main as pdftool_main
|
||||
|
||||
# 开发工具
|
||||
from pyflowx.cli.piptool import main as piptool_main
|
||||
from pyflowx.cli.pymake import main as pymake_main
|
||||
from pyflowx.cli.screenshot import main as screenshot_main
|
||||
from pyflowx.cli.sshcopyid import main as sshcopyid_main
|
||||
|
||||
__all__ = [
|
||||
# 自动格式化工具
|
||||
"autofmt_main",
|
||||
"bumpversion_main",
|
||||
"clearscreen_main",
|
||||
# EML 邮件管理工具
|
||||
"emlmanager_main",
|
||||
"emlmanager_web_main",
|
||||
"envpy_main",
|
||||
"envqt_main",
|
||||
"envrs_main",
|
||||
# 文件工具
|
||||
"filedate_main",
|
||||
"filelevel_main",
|
||||
"folderback_main",
|
||||
"folderzip_main",
|
||||
# Git 工具
|
||||
"gittool_main",
|
||||
# 仿真工具
|
||||
"lscalc_main",
|
||||
# 打包工具
|
||||
"packtool_main",
|
||||
# PDF 工具
|
||||
"pdftool_main",
|
||||
# 开发工具
|
||||
"piptool_main",
|
||||
"pymake_main",
|
||||
"screenshot_main",
|
||||
"sshcopyid_main",
|
||||
# 系统工具
|
||||
"taskkill_main",
|
||||
"which_main",
|
||||
]
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
"""版本号自动管理工具.
|
||||
|
||||
使用 TaskSpec 模式实现, 支持语义化版本管理和多文件格式的版本号更新.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def bump_version(part: str = "patch", tag: bool = False, commit: bool = False) -> None:
|
||||
"""递增版本号.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
part : str
|
||||
版本部分: patch, minor, major
|
||||
tag : bool
|
||||
是否创建 Git 标签
|
||||
commit : bool
|
||||
是否提交更改
|
||||
"""
|
||||
try:
|
||||
subprocess.run(["bumpversion", part], check=True)
|
||||
if commit:
|
||||
subprocess.run(["git", "add", "."], check=True)
|
||||
subprocess.run(["git", "commit", "-m", f"bump version {part}"], check=True)
|
||||
if tag:
|
||||
# 获取当前版本号
|
||||
result = subprocess.run(
|
||||
["git", "describe", "--tags", "--abbrev=0"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
version = result.stdout.strip() if result.returncode == 0 else f"v{part}"
|
||||
subprocess.run(
|
||||
["git", "tag", "-a", version, "-m", f"version {part}"],
|
||||
check=True,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
print("未找到 bumpversion 工具,请先安装: pip install bumpversion")
|
||||
raise
|
||||
|
||||
|
||||
def bump_version_alpha(part: str = "patch") -> None:
|
||||
"""递增版本号并添加 alpha 预发布标识."""
|
||||
try:
|
||||
subprocess.run(["bumpversion", part, "--new-version", f"{part}-alpha"], check=True)
|
||||
except FileNotFoundError:
|
||||
print("未找到 bumpversion 工具,请先安装: pip install bumpversion")
|
||||
raise
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# TaskSpec 定义
|
||||
# ============================================================================
|
||||
|
||||
bump_patch: px.TaskSpec = px.TaskSpec("bump_patch", fn=lambda: bump_version("patch"))
|
||||
bump_minor: px.TaskSpec = px.TaskSpec("bump_minor", fn=lambda: bump_version("minor"))
|
||||
bump_major: px.TaskSpec = px.TaskSpec("bump_major", fn=lambda: bump_version("major"))
|
||||
bump_patch_tag: px.TaskSpec = px.TaskSpec("bump_patch_tag", fn=lambda: bump_version("patch", tag=True))
|
||||
bump_minor_tag: px.TaskSpec = px.TaskSpec("bump_minor_tag", fn=lambda: bump_version("minor", tag=True))
|
||||
bump_major_tag: px.TaskSpec = px.TaskSpec("bump_major_tag", fn=lambda: bump_version("major", tag=True))
|
||||
bump_patch_alpha: px.TaskSpec = px.TaskSpec("bump_patch_alpha", fn=lambda: bump_version_alpha("patch"))
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""版本号管理工具主函数."""
|
||||
runner = px.CliRunner(
|
||||
strategy="thread",
|
||||
description="BumpVersion - 版本号自动管理工具",
|
||||
graphs={
|
||||
# 递增补丁号 (1.0.0 -> 1.0.1)
|
||||
"p": px.Graph.from_specs([bump_patch]),
|
||||
# 递增次版本号 (1.0.0 -> 1.1.0)
|
||||
"m": px.Graph.from_specs([bump_minor]),
|
||||
# 递增主版本号 (1.0.0 -> 2.0.0)
|
||||
"M": px.Graph.from_specs([bump_major]),
|
||||
# 递增补丁号并创建标签
|
||||
"pt": px.Graph.from_specs([bump_patch_tag]),
|
||||
# 递增次版本号并创建标签
|
||||
"mt": px.Graph.from_specs([bump_minor_tag]),
|
||||
# 递增主版本号并创建标签
|
||||
"Mt": px.Graph.from_specs([bump_major_tag]),
|
||||
# 递增补丁号并添加 alpha 预发布标识
|
||||
"pa": px.Graph.from_specs([bump_patch_alpha]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
@@ -1,27 +0,0 @@
|
||||
"""清屏工具.
|
||||
|
||||
跨平台清屏工具, 支持终端和控制台清屏.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
def clear_screen() -> None:
|
||||
"""使用系统命令清屏."""
|
||||
if Constants.IS_WINDOWS:
|
||||
subprocess.run(["cmd", "/c", "cls"], check=False)
|
||||
else:
|
||||
subprocess.run(["clear"], check=False)
|
||||
|
||||
print("\033[2J\033[H", end="")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""清屏工具主函数."""
|
||||
graph = px.Graph.from_specs([px.TaskSpec("clearscreen", fn=clear_screen)])
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,122 +0,0 @@
|
||||
"""Python 环境配置工具.
|
||||
|
||||
用于设置 pip 镜像源, 支持清华和阿里云等国内镜像源,
|
||||
同时配置 UV 和 Conda 的镜像源.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
PIP_INDEX_URLS: dict[str, str] = {
|
||||
"tsinghua": "https://pypi.tuna.tsinghua.edu.cn/simple",
|
||||
"aliyun": "https://mirrors.aliyun.com/pypi/simple/",
|
||||
}
|
||||
|
||||
PIP_TRUSTED_HOSTS: dict[str, str] = {
|
||||
"tsinghua": "pypi.tuna.tsinghua.edu.cn",
|
||||
"aliyun": "mirrors.aliyun.com",
|
||||
}
|
||||
|
||||
UV_INDEX_URL: str = "https://mirrors.aliyun.com/pypi/simple/"
|
||||
UV_PYTHON_INSTALL_MIRROR: str = "https://registry.npmmirror.com/-/binary/python-build-standalone"
|
||||
|
||||
CONDA_MIRROR_URLS: dict[str, list[str]] = {
|
||||
"tsinghua": [
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/",
|
||||
],
|
||||
"aliyun": [
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/main/",
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/free/",
|
||||
"https://mirrors.aliyun.com/anaconda/cloud/conda-forge/",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def set_pip_mirror(mirror: str = "tsinghua", token: str | None = None) -> None:
|
||||
"""设置 pip 镜像源.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mirror : str
|
||||
镜像源名称: tsinghua, aliyun
|
||||
token : str | None
|
||||
PyPI token for publishing
|
||||
"""
|
||||
index_url = PIP_INDEX_URLS.get(mirror, PIP_INDEX_URLS["tsinghua"])
|
||||
trusted_host = PIP_TRUSTED_HOSTS.get(mirror, "")
|
||||
|
||||
# 设置环境变量
|
||||
os.environ["PIP_INDEX_URL"] = index_url
|
||||
os.environ["UV_INDEX_URL"] = UV_INDEX_URL
|
||||
os.environ["UV_DEFAULT_INDEX"] = UV_INDEX_URL
|
||||
os.environ["UV_PYTHON_INSTALL_MIRROR"] = UV_PYTHON_INSTALL_MIRROR
|
||||
|
||||
# 写入 pip 配置文件
|
||||
pip_dir = Path.home() / "pip"
|
||||
pip_dir.mkdir(exist_ok=True)
|
||||
pip_conf = pip_dir / ("pip.ini" if Constants.IS_WINDOWS else "pip.conf")
|
||||
pip_conf.write_text(f"[global]\nindex-url = {index_url}\n[install]\ntrusted-host = {trusted_host}\n")
|
||||
|
||||
# 写入 conda 配置文件
|
||||
condarc = Path.home() / ".condarc"
|
||||
conda_urls = CONDA_MIRROR_URLS.get(mirror, CONDA_MIRROR_URLS["tsinghua"])
|
||||
condarc.write_text(
|
||||
"show_channel_urls: true\nchannels:\n" + "\n".join(f" - {url}" for url in conda_urls) + "\n - defaults\n"
|
||||
)
|
||||
|
||||
# 写入 pypirc 配置文件 (如果有 token)
|
||||
if token:
|
||||
pypirc = Path.home() / ".pypirc"
|
||||
pypirc.write_text(
|
||||
f"[pypi]\nrepository: https://upload.pypi.org/legacy/\nusername: __token__\npassword: {token}\n"
|
||||
)
|
||||
|
||||
print(f"已设置 pip 镜像源: {mirror} ({index_url})")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Python 环境配置工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="EnvPy - Python 环境配置工具",
|
||||
usage="envpy <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 设置镜像源命令
|
||||
mirror_parser = subparsers.add_parser("mirror", help="设置 pip 镜像源")
|
||||
mirror_parser.add_argument("name", choices=["tsinghua", "aliyun"], help="镜像源名称")
|
||||
mirror_parser.add_argument("--token", type=str, help="PyPI token for publishing")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "mirror":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("set_pip_mirror", fn=set_pip_mirror, args=(args.name,), kwargs={"token": args.token})]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,57 +0,0 @@
|
||||
"""PyQt 环境配置工具.
|
||||
|
||||
用于设置 PyQt 相关环境变量, 安装依赖环境.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
QT_LIBS: list[str] = [
|
||||
"build-essential",
|
||||
"libgl1",
|
||||
"libegl1",
|
||||
"libglib2.0-0",
|
||||
"libfontconfig1",
|
||||
"libfreetype6",
|
||||
"libxkbcommon0",
|
||||
"libdbus-1-3",
|
||||
"libxcb-xinerama0",
|
||||
"libxcb-icccm4",
|
||||
"libxcb-image0",
|
||||
"libxcb-keysyms1",
|
||||
"libxcb-randr0",
|
||||
"libxcb-render-util0",
|
||||
"libxcb-shape0",
|
||||
"libxcb-xfixes0",
|
||||
"libxcb-cursor0",
|
||||
]
|
||||
|
||||
CHINESE_FONTS: list[str] = [
|
||||
"fonts-noto-cjk",
|
||||
"fonts-wqy-microhei",
|
||||
"fonts-wqy-zenhei",
|
||||
"fonts-noto-color-emoji",
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""PyQt 环境配置工具主函数."""
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"envqt_install",
|
||||
cmd=["sudo", "apt", "install", "-y", *QT_LIBS],
|
||||
conditions=(lambda: Constants.IS_LINUX,),
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
"envqt_fonts",
|
||||
cmd=["sudo", "apt", "install", "-y", *CHINESE_FONTS],
|
||||
conditions=(lambda: Constants.IS_LINUX,),
|
||||
verbose=True,
|
||||
),
|
||||
],
|
||||
)
|
||||
px.run(graph, strategy="thread", verbose=True)
|
||||
@@ -1,150 +0,0 @@
|
||||
"""Rust 环境配置工具.
|
||||
|
||||
配置 Rustup 和 Cargo 的国内镜像源,
|
||||
加速 Rust 工具链和依赖包的下载.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
RUSTUP_MIRRORS: dict[str, dict[str, str]] = {
|
||||
"aliyun": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.aliyun.com/rustup",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.aliyun.com/rustup/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.aliyun.com/crates.io-index/",
|
||||
},
|
||||
"ustc": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.ustc.edu.cn/rust-static",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.ustc.edu.cn/rust-static/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.ustc.edu.cn/crates.io-index/",
|
||||
},
|
||||
"tsinghua": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.tuna.tsinghua.edu.cn/rustup",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.tuna.tsinghua.edu.cn/rustup/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.tuna.tsinghua.edu.cn/crates.io-index/",
|
||||
},
|
||||
}
|
||||
|
||||
UsableRustVersion = Literal["stable", "nightly", "beta"]
|
||||
UsableMirror = Literal["aliyun", "ustc", "tsinghua"]
|
||||
|
||||
DEFAULT_RUST_VERSION: str = "stable"
|
||||
DEFAULT_MIRROR: UsableMirror = "tsinghua"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def set_rust_mirror(mirror: UsableMirror = DEFAULT_MIRROR) -> None:
|
||||
"""设置 Rust 镜像源.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mirror : str
|
||||
镜像源名称: aliyun, ustc, tsinghua
|
||||
"""
|
||||
mirror_dict = RUSTUP_MIRRORS.get(mirror, RUSTUP_MIRRORS[DEFAULT_MIRROR])
|
||||
server = mirror_dict["RUSTUP_DIST_SERVER"]
|
||||
update_root = mirror_dict["RUSTUP_UPDATE_ROOT"]
|
||||
toml_registry = mirror_dict["TOML_REGISTRY"]
|
||||
|
||||
# 设置环境变量
|
||||
os.environ["RUSTUP_DIST_SERVER"] = server
|
||||
os.environ["RUSTUP_UPDATE_ROOT"] = update_root
|
||||
|
||||
# 写入 cargo 配置
|
||||
cargo_dir = Path.home() / ".cargo"
|
||||
cargo_dir.mkdir(exist_ok=True)
|
||||
cargo_config = cargo_dir / "config.toml"
|
||||
cargo_config.write_text(
|
||||
f"""[source.crates-io]
|
||||
replace-with = '{mirror}'
|
||||
|
||||
[source.{mirror}]
|
||||
registry = "sparse+{toml_registry}"
|
||||
|
||||
[registries.{mirror}]
|
||||
index = "sparse+{toml_registry}"
|
||||
"""
|
||||
)
|
||||
|
||||
print(f"已设置 Rust 镜像源: {mirror}")
|
||||
|
||||
|
||||
def install_rust(version: UsableRustVersion = DEFAULT_RUST_VERSION) -> None:
|
||||
"""安装 Rust 工具链.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
version : str
|
||||
Rust 版本: stable, nightly, beta
|
||||
"""
|
||||
try:
|
||||
subprocess.run(["rustup", "toolchain", "install", version], check=True)
|
||||
print(f"已安装 Rust {version}")
|
||||
except FileNotFoundError:
|
||||
print("未找到 rustup,请先安装 Rust: https://rustup.rs")
|
||||
raise
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Rust 环境配置工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="EnvRs - Rust 环境配置工具",
|
||||
usage="envrs <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 设置镜像源命令
|
||||
mirror_parser = subparsers.add_parser("mirror", help="设置 Rust 镜像源")
|
||||
mirror_parser.add_argument(
|
||||
"name",
|
||||
nargs="?",
|
||||
default=DEFAULT_MIRROR,
|
||||
choices=get_args(UsableMirror),
|
||||
help=f"镜像源名称 ({get_args(UsableMirror)})",
|
||||
)
|
||||
|
||||
# 安装 Rust 命令
|
||||
install_parser = subparsers.add_parser("install", help="安装 Rust 工具链")
|
||||
install_parser.add_argument(
|
||||
"version",
|
||||
nargs="?",
|
||||
default=DEFAULT_RUST_VERSION,
|
||||
choices=get_args(UsableRustVersion),
|
||||
help=f"Rust 版本 ({get_args(UsableRustVersion)})",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "mirror":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("set_rust_mirror", fn=set_rust_mirror, args=(args.name,), verbose=True)]
|
||||
)
|
||||
elif args.command == "install":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("install_rust", cmd=["rustup", "toolchain", "install", args.version], verbose=True)]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread", verbose=True)
|
||||
@@ -1,109 +0,0 @@
|
||||
"""Git 工具模块.
|
||||
|
||||
提供 Git 仓库管理的常用操作封装,
|
||||
支持初始化、提交、清理、推送等功能.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
EXCLUDE_DIRS = [
|
||||
# 编辑器相关目录
|
||||
".vscode",
|
||||
".idea",
|
||||
".editorconfig",
|
||||
".trae",
|
||||
".qoder",
|
||||
# 项目相关目录
|
||||
".venv",
|
||||
".git",
|
||||
".tox",
|
||||
".pytest_cache",
|
||||
"node_modules",
|
||||
".ruff_cache",
|
||||
]
|
||||
EXCLUDE_CMDS = [arg for d in EXCLUDE_DIRS for arg in ["-e", d]]
|
||||
|
||||
|
||||
def init_sub_dirs() -> None:
|
||||
"""初始化子目录的Git仓库."""
|
||||
sub_dirs = [subdir for subdir in Path.cwd().iterdir() if subdir.is_dir()]
|
||||
for subdir in sub_dirs:
|
||||
px.run(
|
||||
px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"init",
|
||||
cmd=["git", "init"],
|
||||
conditions=[not_has_git_repo],
|
||||
cwd=str(subdir),
|
||||
),
|
||||
px.TaskSpec("add", cmd=["git", "add", "."], depends_on=["init"], cwd=str(subdir)),
|
||||
px.TaskSpec(
|
||||
"commit", cmd=["git", "commit", "-m", "init commit"], depends_on=["add"], cwd=str(subdir)
|
||||
),
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
isub: px.TaskSpec = px.TaskSpec("isub", fn=init_sub_dirs)
|
||||
push: px.TaskSpec = px.TaskSpec("push", cmd=["git", "push"])
|
||||
pull: px.TaskSpec = px.TaskSpec("pull", cmd=["git", "pull"])
|
||||
kill_tgit: px.TaskSpec = px.TaskSpec("task_kill", cmd=["taskkill", "/f", "/t", "/im", "tgitcache.exe"])
|
||||
|
||||
|
||||
def not_has_git_repo() -> bool:
|
||||
"""检查当前目录没有Git仓库."""
|
||||
return not Path.cwd().exists() or not (Path.cwd() / ".git").is_dir()
|
||||
|
||||
|
||||
def has_files() -> bool:
|
||||
"""检查当前目录是否有文件."""
|
||||
return bool(list(Path.cwd().glob("*")))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Git工具主函数."""
|
||||
runner = px.CliRunner(
|
||||
strategy="thread",
|
||||
description="Gittool - Git 执行工具.",
|
||||
graphs={
|
||||
# 添加并提交
|
||||
"a": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("add", cmd=["git", "add", "."], conditions=[has_files]),
|
||||
px.TaskSpec("commit", cmd=["git", "commit", "-m", "chore: update"], depends_on=["add"]),
|
||||
]
|
||||
),
|
||||
# 清理
|
||||
"c": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("clean", cmd=["git", "clean", "-xfd", *EXCLUDE_CMDS]),
|
||||
px.TaskSpec("status", cmd=["git", "status", "--porcelain"], depends_on=["clean"]),
|
||||
]
|
||||
),
|
||||
# 初始化、添加并提交
|
||||
"i": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("init", cmd=["git", "init"], conditions=[not_has_git_repo]),
|
||||
px.TaskSpec("add", cmd=["git", "add", "."], depends_on=["init"], conditions=[has_files]),
|
||||
px.TaskSpec(
|
||||
"commit", cmd=["git", "commit", "-m", "init commit"], depends_on=["add"], conditions=[has_files]
|
||||
),
|
||||
]
|
||||
),
|
||||
# 初始化子目录
|
||||
"isub": px.Graph.from_specs([isub]),
|
||||
# 推送
|
||||
"p": px.Graph.from_specs([push]),
|
||||
# 拉取
|
||||
"pl": px.Graph.from_specs([pull]),
|
||||
# 重启TGit缓存
|
||||
"r": px.Graph.from_specs([kill_tgit]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
@@ -1,86 +0,0 @@
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
HFDownloadType = Literal["model", "dataset", "space"]
|
||||
|
||||
|
||||
def setenvs():
|
||||
"""设置 HuggingFace mirror 环境变量."""
|
||||
os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Download a model from HuggingFace.")
|
||||
parser.add_argument("dataset_name", type=str, help="HuggingFace dataset name.")
|
||||
parser.add_argument(
|
||||
"--type",
|
||||
type=str,
|
||||
nargs="?",
|
||||
default="dataset",
|
||||
choices=get_args(HFDownloadType),
|
||||
help="HuggingFace dataset type.",
|
||||
)
|
||||
parser.add_argument("--use-hfd", action="store_true", help="Use HFD tool to download dataset.")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.dataset_name:
|
||||
parser.error("dataset_name is required")
|
||||
|
||||
dataset_name = args.dataset_name
|
||||
|
||||
# 创建下载目录
|
||||
download_dir = Path.cwd() / dataset_name
|
||||
download_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if args.use_hfd:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(name="setenvs", fn=setenvs, verbose=True),
|
||||
px.TaskSpec(
|
||||
name="download_hfd",
|
||||
cmd=["wget", "https://hf-mirror.com/hfd/hfd.sh"],
|
||||
depends_on=["setenvs"],
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
name="chmod_hfd",
|
||||
cmd=["chmod", "a+x", "hfd.sh"],
|
||||
depends_on=["download_hfd"],
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
name="run_hfd",
|
||||
cmd=["./hfd.sh", dataset_name, args.type],
|
||||
depends_on=["chmod_hfd"],
|
||||
verbose=True,
|
||||
),
|
||||
]
|
||||
)
|
||||
else:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(name="setenvs", fn=setenvs, verbose=True),
|
||||
px.TaskSpec(
|
||||
name="download",
|
||||
cmd=[
|
||||
"uvx",
|
||||
"hf",
|
||||
"download",
|
||||
"--repo-type",
|
||||
args.type,
|
||||
"--force-download",
|
||||
dataset_name,
|
||||
"--local-dir",
|
||||
str(Path.cwd() / dataset_name),
|
||||
],
|
||||
depends_on=["setenvs"],
|
||||
verbose=True,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
px.run(graph, strategy="thread", verbose=True)
|
||||
@@ -0,0 +1,15 @@
|
||||
"""legacy 子包 — 未迁移到 @px.tool 的独立工具.
|
||||
|
||||
每个子模块有自己的 ``main()`` 函数, 由 ``pf`` 入口通过
|
||||
``_LEGACY_TOOLS`` 路由调用. 这些工具因逻辑复杂 (web 应用、runpy 注入等)
|
||||
暂未用 ``@px.tool`` 装饰器重写.
|
||||
|
||||
子模块
|
||||
------
|
||||
- :mod:`emlmanager` —— EML 邮件管理 Web 应用
|
||||
- :mod:`profiler` —— pxp 性能分析器 (monkey-patch + runpy)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__: list[str] = []
|
||||
@@ -88,6 +88,8 @@ class EmailDatabase:
|
||||
|
||||
def insert_email(self, email_data: dict[str, Any]) -> bool:
|
||||
"""插入邮件数据."""
|
||||
assert self.conn, "数据库连接未初始化"
|
||||
|
||||
try:
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
@@ -123,6 +125,8 @@ class EmailDatabase:
|
||||
self, keyword: str = "", field: str = "all", limit: int = 100, offset: int = 0
|
||||
) -> list[dict[str, Any]]:
|
||||
"""搜索邮件."""
|
||||
assert self.conn, "数据库连接未初始化"
|
||||
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
|
||||
@@ -150,16 +154,18 @@ class EmailDatabase:
|
||||
cursor.execute(query, (f"%{keyword}%", f"%{keyword}%", f"%{keyword}%", f"%{keyword}%", limit, offset))
|
||||
|
||||
columns = [description[0] for description in cursor.description]
|
||||
return [dict(zip(columns, row)) for row in cursor.fetchall()]
|
||||
return [dict(zip(columns, row, strict=False)) for row in cursor.fetchall()]
|
||||
|
||||
def get_grouped_emails(self) -> dict[str, list[dict[str, Any]]]:
|
||||
"""获取按主题分组的邮件."""
|
||||
assert self.conn, "数据库连接未初始化"
|
||||
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute(f"SELECT * FROM {TABLE_NAME} ORDER BY subject, date_parsed DESC")
|
||||
|
||||
columns = [description[0] for description in cursor.description]
|
||||
emails = [dict(zip(columns, row)) for row in cursor.fetchall()]
|
||||
emails = [dict(zip(columns, row, strict=False)) for row in cursor.fetchall()]
|
||||
|
||||
# 按主题分组
|
||||
grouped: dict[str, list[dict[str, Any]]] = {}
|
||||
@@ -183,6 +189,8 @@ class EmailDatabase:
|
||||
|
||||
def get_email_count(self) -> int:
|
||||
"""获取邮件总数."""
|
||||
assert self.conn, "数据库连接未初始化"
|
||||
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute(f"SELECT COUNT(*) FROM {TABLE_NAME}")
|
||||
@@ -190,6 +198,8 @@ class EmailDatabase:
|
||||
|
||||
def clear_all(self) -> None:
|
||||
"""清空所有邮件数据."""
|
||||
assert self.conn, "数据库连接未初始化"
|
||||
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute(f"DELETE FROM {TABLE_NAME}")
|
||||
@@ -230,7 +240,7 @@ def _parse_email_date(date_str: str) -> str:
|
||||
try:
|
||||
dt = parsedate_to_datetime(date_str)
|
||||
return dt.isoformat()
|
||||
except Exception:
|
||||
except (ValueError, TypeError, OverflowError):
|
||||
return date_str
|
||||
|
||||
|
||||
@@ -267,11 +277,11 @@ def _extract_email_body_part(part: Any) -> str:
|
||||
decoded_text = payload.decode(charset, errors="replace")
|
||||
except (UnicodeDecodeError, LookupError) as decode_error:
|
||||
# 如果指定编码失败,尝试常见编码
|
||||
logger.warning(f"字符编码 {charset} 解码失败: {decode_error}")
|
||||
logger.warning("字符编码 %s 解码失败: %s", charset, decode_error)
|
||||
for fallback_charset in ["utf-8", "gbk", "gb2312", "latin-1"]:
|
||||
try:
|
||||
decoded_text = payload.decode(fallback_charset, errors="replace")
|
||||
logger.info(f"成功使用备用编码 {fallback_charset} 解码")
|
||||
logger.info("成功使用备用编码 %s 解码", fallback_charset)
|
||||
break
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
continue
|
||||
@@ -283,15 +293,15 @@ def _extract_email_body_part(part: Any) -> str:
|
||||
# 限制长度并返回
|
||||
result = decoded_text[:MAX_BODY_LENGTH]
|
||||
if len(decoded_text) > MAX_BODY_LENGTH:
|
||||
logger.debug(f"正文内容过长,截取前{MAX_BODY_LENGTH}字符")
|
||||
logger.debug("正文内容过长,截取前%d字符", MAX_BODY_LENGTH)
|
||||
|
||||
return result
|
||||
|
||||
except AttributeError as attr_error:
|
||||
logger.error(f"邮件部分对象属性错误: {attr_error}")
|
||||
logger.error("邮件部分对象属性错误: %s", attr_error)
|
||||
return ""
|
||||
except Exception as unexpected_error:
|
||||
logger.error(f"提取邮件正文时发生未知错误: {unexpected_error}")
|
||||
logger.error("提取邮件正文时发生未知错误: %s", unexpected_error)
|
||||
return ""
|
||||
|
||||
|
||||
@@ -578,6 +588,10 @@ class EmlManagerHandler(BaseHTTPRequestHandler):
|
||||
self._send_json_response({"error": "缺少邮件ID"}, 400)
|
||||
return
|
||||
|
||||
if not self.db.conn:
|
||||
self._send_json_response({"error": "数据库连接未初始化"}, 500)
|
||||
return
|
||||
|
||||
with self.db._lock:
|
||||
cursor = self.db.conn.cursor()
|
||||
cursor.execute(f"SELECT * FROM {TABLE_NAME} WHERE id = ?", (int(email_id),))
|
||||
@@ -588,7 +602,7 @@ class EmlManagerHandler(BaseHTTPRequestHandler):
|
||||
self._send_json_response({"error": "邮件不存在"}, 404)
|
||||
return
|
||||
|
||||
email_data = dict(zip(columns, row))
|
||||
email_data = dict(zip(columns, row, strict=False))
|
||||
self._send_json_response({"email": email_data})
|
||||
|
||||
def _api_get_grouped_emails(self) -> None:
|
||||
@@ -630,6 +644,10 @@ class EmlManagerHandler(BaseHTTPRequestHandler):
|
||||
if not eml_files:
|
||||
return
|
||||
|
||||
if not self.db.conn:
|
||||
self._send_json_response({"error": "数据库连接未初始化"}, 500)
|
||||
return
|
||||
|
||||
# 先批量查询所有已存在的文件
|
||||
with self.db._lock:
|
||||
cursor = self.db.conn.cursor()
|
||||
@@ -1268,6 +1286,10 @@ def main() -> None:
|
||||
if eml_files:
|
||||
print(f"发现 {len(eml_files)} 个 EML 文件,开始导入...")
|
||||
|
||||
if not EmlManagerHandler.db.conn:
|
||||
print("数据库连接未初始化,无法导入邮件")
|
||||
return
|
||||
|
||||
# 先批量查询所有已存在的文件
|
||||
with EmlManagerHandler.db._lock:
|
||||
cursor = EmlManagerHandler.db.conn.cursor()
|
||||
@@ -0,0 +1,272 @@
|
||||
"""pxp —— PyFlowX 性能分析器.
|
||||
|
||||
分析包含 ``px`` 调用的 Python 脚本,生成工作流执行性能剖面报告。
|
||||
|
||||
工作原理
|
||||
--------
|
||||
1. 注入 hook:monkey-patch ``pyflowx.run`` / ``pyflowx.executors.run`` /
|
||||
``pyflowx.runner.run``,捕获最后一次执行的 ``Graph`` 与 ``RunReport``。
|
||||
2. 执行目标脚本:用 ``runpy.run_path`` 以 ``__main__`` 身份执行,
|
||||
捕获 ``SystemExit``(脚本可能调 ``sys.exit``)。
|
||||
3. 生成报告:从捕获的 report + graph 构建 :class:`ProfileReport`,
|
||||
默认输出 HTML 并自动打开浏览器。
|
||||
|
||||
使用方式
|
||||
--------
|
||||
# 分析 pymake.py,生成 HTML 报告并打开浏览器
|
||||
pxp pymake.py
|
||||
|
||||
# 传递参数给被分析脚本(用 -- 分隔)
|
||||
pxp pymake.py -- t
|
||||
|
||||
# 指定输出文件
|
||||
pxp pymake.py -o report.html
|
||||
|
||||
# 不打开浏览器
|
||||
pxp pymake.py --no-browser
|
||||
|
||||
# 输出纯文本报告
|
||||
pxp pymake.py -E text
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["main"]
|
||||
|
||||
import argparse
|
||||
import runpy
|
||||
import sys
|
||||
import webbrowser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ... import executors as _executors
|
||||
from ... import runner as _runner
|
||||
from ...profiling import ProfileReport
|
||||
from ...report import RunReport
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
"""构建参数解析器。"""
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="pxp",
|
||||
description="PyFlowX 性能分析器:分析包含 px 调用的脚本,生成性能剖面报告。",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=(
|
||||
"示例:\n"
|
||||
" pxp pymake.py # 分析并打开 HTML 报告\n"
|
||||
" pxp pymake.py -- t # 传递参数 t 给脚本\n"
|
||||
" pxp pymake.py -E text # 输出纯文本报告\n"
|
||||
" pxp pymake.py -o out.html # 指定输出文件\n"
|
||||
),
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"--export",
|
||||
"-E",
|
||||
choices=["html", "text"],
|
||||
default="html",
|
||||
help="导出格式(默认: html)",
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"--no-browser",
|
||||
action="store_true",
|
||||
help="不自动打开浏览器(仅 HTML 格式有效)",
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
help="输出文件路径(默认: <script>_profile.html)",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def _capture_px_run() -> dict[str, Any]:
|
||||
"""注入 hook 捕获 px.run() 调用。
|
||||
|
||||
返回一个字典,``run()`` 执行后填充 ``graph`` 与 ``report``。
|
||||
同时返回还原函数用于 finally 块。
|
||||
|
||||
Note
|
||||
-----
|
||||
需同时 patch 三处引用:
|
||||
* ``pyflowx.executors.run`` —— 实际实现
|
||||
* ``pyflowx.runner.run`` —— ``CliRunner`` 直接 import 的引用
|
||||
* ``pyflowx.run`` —— 顶层包导出的引用(用户脚本常用 ``px.run()``)
|
||||
|
||||
另外 patch ``RunReport.__init__`` 以捕获 ``run()`` 内部创建的 report 实例。
|
||||
这对于 ``run()`` 抛出 ``TaskFailedError`` 的场景至关重要:此时 ``run()``
|
||||
不会正常返回 report,但 report 对象已在内部创建并填充了已执行任务的结果。
|
||||
通过 ``capture_enabled`` 标志确保只在 ``patched_run`` 调用期间捕获。
|
||||
"""
|
||||
captured: dict[str, Any] = {}
|
||||
original_exec_run = _executors.run
|
||||
original_runner_run = _runner.run
|
||||
# 惰性获取顶层 pyflowx.run 引用(避免循环导入)
|
||||
import pyflowx as px_mod
|
||||
|
||||
original_px_run = px_mod.run
|
||||
original_report_init = RunReport.__init__
|
||||
capture_enabled = [False]
|
||||
|
||||
def patched_report_init(self: RunReport, *args: Any, **kwargs: Any) -> None:
|
||||
original_report_init(self, *args, **kwargs)
|
||||
if capture_enabled[0]:
|
||||
captured["report"] = self
|
||||
|
||||
RunReport.__init__ = patched_report_init # type: ignore[assignment]
|
||||
|
||||
def patched_run(graph: Any, *args: Any, **kwargs: Any) -> RunReport:
|
||||
captured["graph"] = graph
|
||||
capture_enabled[0] = True
|
||||
try:
|
||||
report = original_exec_run(graph, *args, **kwargs)
|
||||
# 正常返回时确保 captured["report"] 是返回的 report
|
||||
captured["report"] = report
|
||||
return report
|
||||
finally:
|
||||
capture_enabled[0] = False
|
||||
|
||||
# patch 所有引用 run 的入口
|
||||
_executors.run = patched_run # type: ignore[assignment]
|
||||
_runner.run = patched_run # type: ignore[assignment]
|
||||
px_mod.run = patched_run # type: ignore[assignment]
|
||||
|
||||
def _restore() -> None:
|
||||
_executors.run = original_exec_run # type: ignore[assignment]
|
||||
_runner.run = original_runner_run # type: ignore[assignment]
|
||||
px_mod.run = original_px_run # type: ignore[assignment]
|
||||
RunReport.__init__ = original_report_init # type: ignore[assignment]
|
||||
|
||||
captured["_restore"] = _restore
|
||||
return captured
|
||||
|
||||
|
||||
def _run_target_script(script: Path, script_args: list[str]) -> dict[str, Any]:
|
||||
"""执行目标脚本。
|
||||
|
||||
将脚本所在目录加入 ``sys.path``,设置 ``sys.argv``,然后用
|
||||
``runpy.run_path`` 以 ``__main__`` 身份执行。捕获 ``SystemExit``。
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[str, Any]
|
||||
脚本模块的全局变量字典(含 ``main`` 等定义)。
|
||||
"""
|
||||
sys.argv = [str(script), *script_args]
|
||||
script_dir = str(script.parent.resolve())
|
||||
if script_dir not in sys.path:
|
||||
sys.path.insert(0, script_dir)
|
||||
return runpy.run_path(str(script), run_name="__main__")
|
||||
|
||||
|
||||
def _try_call_main(module_globals: dict[str, Any]) -> None:
|
||||
"""若模块定义了 ``main`` 可调用对象,调用它。
|
||||
|
||||
用于脚本无 ``if __name__ == "__main__"`` 块的场景(如通过 entry points
|
||||
注册的 CLI 工具脚本)。``main`` 通常调用 ``CliRunner.run_cli()``,
|
||||
后者读取 ``sys.argv[1:]`` 执行对应命令。
|
||||
"""
|
||||
main_fn = module_globals.get("main")
|
||||
if callable(main_fn):
|
||||
main_fn()
|
||||
|
||||
|
||||
def _output_report(
|
||||
profile: ProfileReport,
|
||||
export: str,
|
||||
output: str | None,
|
||||
script_stem: str,
|
||||
no_browser: bool,
|
||||
) -> None:
|
||||
"""输出性能报告。"""
|
||||
if export == "text":
|
||||
print(profile.describe())
|
||||
return
|
||||
|
||||
# HTML 格式
|
||||
html = profile.to_html()
|
||||
if output:
|
||||
out_path = Path(output)
|
||||
else:
|
||||
out_path = Path.cwd() / f"{script_stem}_profile.html"
|
||||
out_path.write_text(html, encoding="utf-8")
|
||||
print(f"HTML 报告已生成: {out_path}")
|
||||
|
||||
if not no_browser:
|
||||
try:
|
||||
webbrowser.open(f"file://{out_path.resolve()}")
|
||||
except Exception as e:
|
||||
print(f"警告:无法打开浏览器: {e}", file=sys.stderr)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""pxp CLI 入口。"""
|
||||
parser = _build_parser()
|
||||
pxp_args, remaining = parser.parse_known_args()
|
||||
|
||||
if not remaining:
|
||||
parser.print_help()
|
||||
sys.exit(2)
|
||||
|
||||
script_str = remaining[0]
|
||||
script_args = remaining[1:]
|
||||
script_path = Path(script_str).resolve()
|
||||
|
||||
if not script_path.is_file():
|
||||
print(f"错误:脚本不存在: {script_path}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# 注入 hook
|
||||
captured = _capture_px_run()
|
||||
|
||||
# 执行目标脚本
|
||||
print(f"正在分析: {script_path}")
|
||||
if script_args:
|
||||
print(f"脚本参数: {script_args}")
|
||||
print("-" * 60)
|
||||
|
||||
module_globals: dict[str, Any] = {}
|
||||
try:
|
||||
module_globals = _run_target_script(script_path, script_args)
|
||||
except SystemExit:
|
||||
# 脚本调用了 sys.exit,正常情况
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"警告:脚本执行抛出异常: {e}", file=sys.stderr)
|
||||
|
||||
# 若脚本执行未捕获到 run(),尝试调用模块的 main() 函数
|
||||
# (适用于无 ``if __name__ == "__main__"`` 块的 CLI 脚本)
|
||||
if captured.get("report") is None and module_globals:
|
||||
try:
|
||||
_try_call_main(module_globals)
|
||||
except SystemExit:
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"警告:调用 main() 抛出异常: {e}", file=sys.stderr)
|
||||
|
||||
# 还原 hook
|
||||
restore = captured.pop("_restore", None)
|
||||
if restore is not None:
|
||||
restore()
|
||||
|
||||
# 检查是否捕获到 run() 调用
|
||||
report = captured.get("report")
|
||||
graph = captured.get("graph")
|
||||
if report is None or graph is None:
|
||||
print("错误:未捕获到 px.run() 调用,无法生成性能报告", file=sys.stderr)
|
||||
print("请确保脚本通过 px.run() 或 CliRunner 执行任务流图。", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# 生成报告
|
||||
profile = ProfileReport.from_report(report, graph)
|
||||
_output_report(
|
||||
profile,
|
||||
export=pxp_args.export,
|
||||
output=pxp_args.output,
|
||||
script_stem=script_path.stem,
|
||||
no_browser=pxp_args.no_browser,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,174 +0,0 @@
|
||||
"""LS-DYNA 计算工具.
|
||||
|
||||
用于管理 LS-DYNA 仿真计算任务,
|
||||
支持启动、监控和管理计算进程.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
LS_DYNA_COMMANDS: dict[str, list[str]] = {
|
||||
"windows": ["ls-dyna_mpp", "i=input.k", "ncpu=4"],
|
||||
"linux": ["ls-dyna_mpp", "i=input.k", "ncpu=8"],
|
||||
"macos": ["ls-dyna_mpp", "i=input.k", "ncpu=4"],
|
||||
}
|
||||
|
||||
DEFAULT_INPUT_FILE: str = "input.k"
|
||||
DEFAULT_NCPU: int = 4
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def get_ls_dyna_command(input_file: str, ncpu: int) -> list[str]:
|
||||
"""获取 LS-DYNA 命令.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_file : str
|
||||
输入文件路径
|
||||
ncpu : int
|
||||
CPU 核心数
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[str]
|
||||
LS-DYNA 命令列表
|
||||
"""
|
||||
if Constants.IS_WINDOWS or Constants.IS_MACOS:
|
||||
return ["ls-dyna_mpp", f"i={input_file}", f"ncpu={ncpu}"]
|
||||
else:
|
||||
return ["ls-dyna_mpp", f"i={input_file}", f"ncpu={ncpu}"]
|
||||
|
||||
|
||||
def run_ls_dyna(input_file: str, ncpu: int = DEFAULT_NCPU) -> None:
|
||||
"""运行 LS-DYNA 计算.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_file : str
|
||||
输入文件路径
|
||||
ncpu : int
|
||||
CPU 核心数
|
||||
"""
|
||||
input_path = Path(input_file)
|
||||
if not input_path.exists():
|
||||
print(f"输入文件不存在: {input_path}")
|
||||
return
|
||||
|
||||
cmd = get_ls_dyna_command(input_file, ncpu)
|
||||
try:
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"LS-DYNA 计算完成: {input_file}")
|
||||
except FileNotFoundError:
|
||||
print("未找到 ls-dyna_mpp 命令")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"LS-DYNA 计算失败: {e}")
|
||||
|
||||
|
||||
def run_ls_dyna_mpi(input_file: str, ncpu: int = DEFAULT_NCPU) -> None:
|
||||
"""运行 LS-DYNA MPI 计算.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_file : str
|
||||
输入文件路径
|
||||
ncpu : int
|
||||
CPU 核心数
|
||||
"""
|
||||
input_path = Path(input_file)
|
||||
if not input_path.exists():
|
||||
print(f"输入文件不存在: {input_path}")
|
||||
return
|
||||
|
||||
cmd = ["mpirun", "-np", str(ncpu), "ls-dyna_mpp", f"i={input_file}"]
|
||||
try:
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"LS-DYNA MPI 计算完成: {input_file}")
|
||||
except FileNotFoundError:
|
||||
print("未找到 mpirun 或 ls-dyna_mpp 命令")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"LS-DYNA MPI 计算失败: {e}")
|
||||
|
||||
|
||||
def check_ls_dyna_status() -> None:
|
||||
"""检查 LS-DYNA 进程状态."""
|
||||
try:
|
||||
if Constants.IS_WINDOWS:
|
||||
result = subprocess.run(
|
||||
["tasklist", "/fi", "imagename eq ls-dyna_mpp.exe"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
print(result.stdout)
|
||||
else:
|
||||
result = subprocess.run(
|
||||
["pgrep", "-f", "ls-dyna"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.stdout.strip():
|
||||
print(f"运行中的 LS-DYNA 进程 PID: {result.stdout.strip()}")
|
||||
else:
|
||||
print("没有运行中的 LS-DYNA 进程")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"检查进程状态失败: {e}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""LS-DYNA 计算工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="LSCalc - LS-DYNA 计算工具",
|
||||
usage="lscalc <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 运行计算命令
|
||||
run_parser = subparsers.add_parser("run", help="运行 LS-DYNA 计算")
|
||||
run_parser.add_argument("input_file", help="输入文件路径")
|
||||
run_parser.add_argument("--ncpu", type=int, default=DEFAULT_NCPU, help="CPU 核心数")
|
||||
|
||||
# 运行 MPI 计算命令
|
||||
mpi_parser = subparsers.add_parser("mpi", help="运行 LS-DYNA MPI 计算")
|
||||
mpi_parser.add_argument("input_file", help="输入文件路径")
|
||||
mpi_parser.add_argument("--ncpu", type=int, default=DEFAULT_NCPU, help="CPU 核心数")
|
||||
|
||||
# 检查进程状态命令
|
||||
subparsers.add_parser("status", help="检查 LS-DYNA 进程状态")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "run":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("run_ls_dyna", fn=run_ls_dyna, args=(args.input_file,), kwargs={"ncpu": args.ncpu})]
|
||||
)
|
||||
elif args.command == "mpi":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("run_ls_dyna_mpi", fn=run_ls_dyna_mpi, args=(args.input_file,), kwargs={"ncpu": args.ncpu})]
|
||||
)
|
||||
elif args.command == "status":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("check_ls_dyna_status", fn=check_ls_dyna_status)])
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,349 +0,0 @@
|
||||
"""Python 打包工具模块.
|
||||
|
||||
提供 Python 项目打包的常用功能封装,
|
||||
支持源码打包、依赖打包、嵌入式 Python 安装等功能.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
import subprocess
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
DEFAULT_BUILD_DIR = ".pypack"
|
||||
DEFAULT_DIST_DIR = "dist"
|
||||
DEFAULT_LIB_DIR = "libs"
|
||||
DEFAULT_CACHE_DIR = ".cache/pypack"
|
||||
|
||||
IGNORE_PATTERNS = [
|
||||
"__pycache__",
|
||||
"*.pyc",
|
||||
"*.pyo",
|
||||
".git",
|
||||
".venv",
|
||||
".idea",
|
||||
".vscode",
|
||||
"*.egg-info",
|
||||
"dist",
|
||||
"build",
|
||||
".pytest_cache",
|
||||
".tox",
|
||||
".mypy_cache",
|
||||
]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def pack_source(project_dir: Path, output_dir: Path) -> None:
|
||||
"""打包项目源码.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
project_dir : Path
|
||||
项目目录
|
||||
output_dir : Path
|
||||
输出目录
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 检测项目名称
|
||||
pyproject_file = project_dir / "pyproject.toml"
|
||||
project_name = project_dir.name
|
||||
|
||||
if pyproject_file.exists():
|
||||
try:
|
||||
import tomllib
|
||||
|
||||
content = pyproject_file.read_text(encoding="utf-8")
|
||||
data = tomllib.loads(content)
|
||||
project_name = data.get("project", {}).get("name", project_name)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# 打包源码
|
||||
source_dir = output_dir / "src" / project_name
|
||||
source_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 复制文件
|
||||
src_subdir = project_dir / "src"
|
||||
if src_subdir.exists():
|
||||
shutil.copytree(
|
||||
src_subdir,
|
||||
source_dir / "src",
|
||||
ignore=shutil.ignore_patterns(*IGNORE_PATTERNS),
|
||||
dirs_exist_ok=True,
|
||||
)
|
||||
else:
|
||||
for item in project_dir.iterdir():
|
||||
if item.name in IGNORE_PATTERNS or item.name.startswith("."):
|
||||
continue
|
||||
dst_item = source_dir / item.name
|
||||
if item.is_dir():
|
||||
shutil.copytree(
|
||||
item,
|
||||
dst_item,
|
||||
ignore=shutil.ignore_patterns(*IGNORE_PATTERNS),
|
||||
dirs_exist_ok=True,
|
||||
)
|
||||
else:
|
||||
shutil.copy2(item, dst_item)
|
||||
|
||||
print(f"源码打包完成: {source_dir}")
|
||||
|
||||
|
||||
def pack_dependencies(lib_dir: Path, dependencies: list[str]) -> None:
|
||||
"""打包项目依赖.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lib_dir : Path
|
||||
依赖库目录
|
||||
dependencies : list[str]
|
||||
依赖列表
|
||||
"""
|
||||
lib_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if not dependencies:
|
||||
print("没有依赖需要打包")
|
||||
return
|
||||
|
||||
# 使用 pip 安装依赖到目标目录
|
||||
cmd = [
|
||||
"pip",
|
||||
"install",
|
||||
"--target",
|
||||
str(lib_dir),
|
||||
"--no-compile",
|
||||
"--no-warn-script-location",
|
||||
]
|
||||
cmd.extend(dependencies)
|
||||
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"依赖打包完成: {lib_dir}")
|
||||
|
||||
|
||||
def pack_wheel(project_dir: Path, output_dir: Path) -> None:
|
||||
"""打包项目为 wheel 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
project_dir : Path
|
||||
项目目录
|
||||
output_dir : Path
|
||||
输出目录
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 使用 pip wheel 打包
|
||||
cmd = [
|
||||
"pip",
|
||||
"wheel",
|
||||
"--no-deps",
|
||||
"--wheel-dir",
|
||||
str(output_dir),
|
||||
str(project_dir),
|
||||
]
|
||||
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"Wheel 打包完成: {output_dir}")
|
||||
|
||||
|
||||
def install_embed_python(version: str, output_dir: Path) -> None:
|
||||
"""安装嵌入式 Python.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
version : str
|
||||
Python 版本 (如: 3.10, 3.11)
|
||||
output_dir : Path
|
||||
输出目录
|
||||
"""
|
||||
import platform
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 构建下载 URL
|
||||
arch = platform.machine().lower()
|
||||
if arch in ["x86_64", "amd64"]:
|
||||
arch = "amd64"
|
||||
elif arch in ["arm64", "aarch64"]:
|
||||
arch = "arm64"
|
||||
|
||||
# 解析完整版本号
|
||||
version_map = {
|
||||
"3.8": "3.8.10",
|
||||
"3.9": "3.9.13",
|
||||
"3.10": "3.10.11",
|
||||
"3.11": "3.11.9",
|
||||
"3.12": "3.12.4",
|
||||
}
|
||||
full_version = version_map.get(version, f"{version}.0")
|
||||
|
||||
# Windows 嵌入式 Python 下载 URL
|
||||
url = f"https://www.python.org/ftp/python/{full_version}/python-{full_version}-embed-{arch}.zip"
|
||||
|
||||
# 下载并解压
|
||||
cache_file = Path(DEFAULT_CACHE_DIR) / f"python-{full_version}-embed-{arch}.zip"
|
||||
cache_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if not cache_file.exists():
|
||||
print(f"正在下载嵌入式 Python {full_version}...")
|
||||
import urllib.request
|
||||
|
||||
urllib.request.urlretrieve(url, cache_file)
|
||||
print(f"下载完成: {cache_file}")
|
||||
|
||||
# 解压
|
||||
with zipfile.ZipFile(cache_file, "r") as zf:
|
||||
zf.extractall(output_dir)
|
||||
|
||||
print(f"嵌入式 Python 安装完成: {output_dir}")
|
||||
|
||||
|
||||
def create_zip_package(source_dir: Path, output_file: Path) -> None:
|
||||
"""创建 ZIP 打包文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
source_dir : Path
|
||||
源目录
|
||||
output_file : Path
|
||||
输出文件
|
||||
"""
|
||||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with zipfile.ZipFile(output_file, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
for file in source_dir.rglob("*"):
|
||||
if file.is_file():
|
||||
arcname = file.relative_to(source_dir)
|
||||
zf.write(file, arcname)
|
||||
|
||||
print(f"ZIP 打包完成: {output_file}")
|
||||
|
||||
|
||||
def clean_build_dir(build_dir: Path) -> None:
|
||||
"""清理构建目录.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
build_dir : Path
|
||||
构建目录
|
||||
"""
|
||||
if build_dir.exists():
|
||||
shutil.rmtree(build_dir)
|
||||
print(f"清理完成: {build_dir}")
|
||||
else:
|
||||
print(f"目录不存在: {build_dir}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Python 打包工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="PackTool - Python 打包工具",
|
||||
usage="packtool <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 源码打包命令
|
||||
src_parser = subparsers.add_parser("src", help="打包项目源码")
|
||||
src_parser.add_argument("--project-dir", type=str, default=".", help="项目目录")
|
||||
src_parser.add_argument("--output-dir", type=str, default=DEFAULT_BUILD_DIR, help="输出目录")
|
||||
|
||||
# 依赖打包命令
|
||||
deps_parser = subparsers.add_parser("deps", help="打包项目依赖")
|
||||
deps_parser.add_argument("--lib-dir", type=str, default=DEFAULT_LIB_DIR, help="依赖库目录")
|
||||
deps_parser.add_argument("dependencies", nargs="*", help="依赖列表")
|
||||
|
||||
# Wheel 打包命令
|
||||
wheel_parser = subparsers.add_parser("wheel", help="打包项目为 wheel 文件")
|
||||
wheel_parser.add_argument("--project-dir", type=str, default=".", help="项目目录")
|
||||
wheel_parser.add_argument("--output-dir", type=str, default=DEFAULT_DIST_DIR, help="输出目录")
|
||||
|
||||
# 嵌入式 Python 安装命令
|
||||
embed_parser = subparsers.add_parser("embed", help="安装嵌入式 Python")
|
||||
embed_parser.add_argument("--version", type=str, default="3.10", help="Python 版本")
|
||||
embed_parser.add_argument("--output-dir", type=str, default="python", help="输出目录")
|
||||
|
||||
# ZIP 打包命令
|
||||
zip_parser = subparsers.add_parser("zip", help="创建 ZIP 打包文件")
|
||||
zip_parser.add_argument("--source-dir", type=str, default=".", help="源目录")
|
||||
zip_parser.add_argument("--output-file", type=str, default="package.zip", help="输出文件")
|
||||
|
||||
# 清理命令
|
||||
subparsers.add_parser("clean", help="清理构建目录")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "src":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pack_source",
|
||||
fn=pack_source,
|
||||
args=(Path(args.project_dir), Path(args.output_dir)),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "deps":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pack_deps",
|
||||
fn=pack_dependencies,
|
||||
args=(Path(args.lib_dir), args.dependencies),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "wheel":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pack_wheel",
|
||||
fn=pack_wheel,
|
||||
args=(Path(args.project_dir), Path(args.output_dir)),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "embed":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"install_embed",
|
||||
fn=install_embed_python,
|
||||
args=(args.version, Path(args.output_dir)),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "zip":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"create_zip",
|
||||
fn=create_zip_package,
|
||||
args=(Path(args.source_dir), Path(args.output_file)),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "clean":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("clean_build", fn=clean_build_dir, args=(Path(DEFAULT_BUILD_DIR),))])
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,524 +0,0 @@
|
||||
"""PDF 工具模块.
|
||||
|
||||
提供 PDF 文件操作的常用功能封装,
|
||||
支持合并、拆分、压缩、加密、水印、OCR等功能.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
try:
|
||||
import fitz # PyMuPDF
|
||||
|
||||
HAS_PYMUPDF = True
|
||||
except ImportError:
|
||||
HAS_PYMUPDF = False
|
||||
|
||||
try:
|
||||
import pypdf
|
||||
|
||||
HAS_PYPDF = True
|
||||
except ImportError:
|
||||
HAS_PYPDF = False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
PDF_SUFFIX = ".pdf"
|
||||
DEFAULT_QUALITY = 75
|
||||
DEFAULT_PASSWORD = ""
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def pdf_merge(input_paths: list[Path], output_path: Path) -> None:
|
||||
"""合并多个 PDF 文件."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
writer = pypdf.PdfWriter()
|
||||
for input_path in input_paths:
|
||||
if input_path.exists():
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"合并完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_split(input_path: Path, output_dir: Path) -> None:
|
||||
"""拆分 PDF 文件为单页."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for i, page in enumerate(reader.pages):
|
||||
writer = pypdf.PdfWriter()
|
||||
writer.add_page(page)
|
||||
output_file = output_dir / f"{input_path.stem}_page_{i + 1}.pdf"
|
||||
with open(output_file, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"拆分完成: {output_dir}")
|
||||
|
||||
|
||||
def pdf_compress(input_path: Path, output_path: Path) -> None:
|
||||
"""压缩 PDF 文件."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path), garbage=4, deflate=True, clean=True)
|
||||
doc.close()
|
||||
|
||||
original_size = input_path.stat().st_size
|
||||
new_size = output_path.stat().st_size
|
||||
ratio = (1 - new_size / original_size) * 100
|
||||
print(f"压缩完成: {output_path} (缩小 {ratio:.1f}%)")
|
||||
|
||||
|
||||
def pdf_encrypt(input_path: Path, output_path: Path, password: str) -> None:
|
||||
"""加密 PDF 文件."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
writer = pypdf.PdfWriter()
|
||||
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
writer.encrypt(user_password=password, owner_password=password, use_128bit=True)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"加密完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_decrypt(input_path: Path, output_path: Path, password: str) -> None:
|
||||
"""解密 PDF 文件."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
if reader.is_encrypted:
|
||||
reader.decrypt(password)
|
||||
|
||||
writer = pypdf.PdfWriter()
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"解密完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_extract_text(input_path: Path, output_path: Path) -> None:
|
||||
"""提取 PDF 文本."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
text = ""
|
||||
for page in doc:
|
||||
text += page.get_text() + "\n\n"
|
||||
doc.close()
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(text, encoding="utf-8")
|
||||
print(f"文本提取完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_extract_images(input_path: Path, output_dir: Path) -> None:
|
||||
"""提取 PDF 图片."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
image_count = 0
|
||||
for page_num, page in enumerate(doc):
|
||||
images = page.get_images(full=True)
|
||||
for img_idx, img in enumerate(images):
|
||||
xref = img[0]
|
||||
base_image = doc.extract_image(xref)
|
||||
image_data = base_image["image"]
|
||||
image_ext = base_image["ext"]
|
||||
image_path = output_dir / f"page_{page_num + 1}_img_{img_idx + 1}.{image_ext}"
|
||||
image_path.write_bytes(image_data)
|
||||
image_count += 1
|
||||
|
||||
doc.close()
|
||||
print(f"图片提取完成: {output_dir} (共 {image_count} 张)")
|
||||
|
||||
|
||||
def pdf_add_watermark(input_path: Path, output_path: Path, text: str = "CONFIDENTIAL") -> None:
|
||||
"""添加 PDF 水印."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
for page in doc:
|
||||
rect = page.rect
|
||||
text_width = fitz.get_text_length(text, fontsize=48)
|
||||
x = (rect.width - text_width) / 2
|
||||
y = rect.height / 2
|
||||
page.insert_text((x, y), text, fontsize=48, rotate=45, color=(0, 0, 0))
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"水印添加完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_rotate(input_path: Path, output_path: Path, rotation: int = 90) -> None:
|
||||
"""旋转 PDF 页面."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
for page in doc:
|
||||
page.set_rotation(rotation)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"旋转完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_crop(input_path: Path, output_path: Path, margins: tuple[int, int, int, int]) -> None:
|
||||
"""裁剪 PDF 页面."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
left, top, right, bottom = margins
|
||||
|
||||
for page in doc:
|
||||
rect = page.rect
|
||||
new_rect = fitz.Rect(
|
||||
rect.x0 + left,
|
||||
rect.y0 + top,
|
||||
rect.x1 - right,
|
||||
rect.y1 - bottom,
|
||||
)
|
||||
page.set_cropbox(new_rect)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"裁剪完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_info(input_path: Path) -> None:
|
||||
"""显示 PDF 信息."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
print(f"文件: {input_path}")
|
||||
print(f"页数: {doc.page_count}")
|
||||
print(f"标题: {doc.metadata.get('title', 'N/A')}")
|
||||
print(f"作者: {doc.metadata.get('author', 'N/A')}")
|
||||
print(f"创建日期: {doc.metadata.get('creationDate', 'N/A')}")
|
||||
print(f"修改日期: {doc.metadata.get('modDate', 'N/A')}")
|
||||
print(f"文件大小: {input_path.stat().st_size / 1024:.1f} KB")
|
||||
doc.close()
|
||||
|
||||
|
||||
def pdf_ocr(input_path: Path, output_path: Path, lang: str = "chi_sim+eng") -> None:
|
||||
"""PDF OCR 识别."""
|
||||
try:
|
||||
import pytesseract
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
print("未安装 OCR 相关库,请安装: pip install pytesseract pillow")
|
||||
return
|
||||
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
new_doc = fitz.open()
|
||||
|
||||
for page in doc:
|
||||
pix = page.get_pixmap()
|
||||
img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
|
||||
ocr_text = pytesseract.image_to_string(img, lang=lang)
|
||||
|
||||
new_page = new_doc.new_page(width=page.rect.width, height=page.rect.height)
|
||||
new_page.insert_image(new_page.rect, pixmap=pix)
|
||||
text_rect = fitz.Rect(0, 0, page.rect.width, page.rect.height)
|
||||
new_page.insert_textbox(text_rect, ocr_text)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
new_doc.save(str(output_path))
|
||||
new_doc.close()
|
||||
doc.close()
|
||||
print(f"OCR 识别完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_reorder(input_path: Path, output_path: Path, order: list[int]) -> None:
|
||||
"""重排 PDF 页面顺序."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
writer = pypdf.PdfWriter()
|
||||
|
||||
for page_num in order:
|
||||
if 0 <= page_num < len(reader.pages):
|
||||
writer.add_page(reader.pages[page_num])
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"重排完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_to_images(input_path: Path, output_dir: Path, dpi: int = 300) -> None:
|
||||
"""PDF 转图片."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for page_num, page in enumerate(doc):
|
||||
pix = page.get_pixmap(dpi=dpi)
|
||||
image_path = output_dir / f"{input_path.stem}_page_{page_num + 1}.png"
|
||||
pix.save(str(image_path))
|
||||
|
||||
doc.close()
|
||||
print(f"转换完成: {output_dir}")
|
||||
|
||||
|
||||
def pdf_repair(input_path: Path, output_path: Path) -> None:
|
||||
"""修复 PDF 文件."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path), garbage=4, deflate=True, clean=True)
|
||||
doc.close()
|
||||
print(f"修复完成: {output_path}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None: # noqa: PLR0912
|
||||
"""PDF 工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="PDFTool - PDF 文件工具集",
|
||||
usage="pdftool <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 合并 PDF 命令
|
||||
merge_parser = subparsers.add_parser("m", help="合并 PDF 文件")
|
||||
merge_parser.add_argument("inputs", nargs="+", help="输入 PDF 文件路径")
|
||||
merge_parser.add_argument("--output", type=str, default="merged.pdf", help="输出文件路径")
|
||||
|
||||
# 拆分 PDF 命令
|
||||
split_parser = subparsers.add_parser("s", help="拆分 PDF 文件为单页")
|
||||
split_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
split_parser.add_argument("--output-dir", type=str, default="split", help="输出目录")
|
||||
|
||||
# 压缩 PDF 命令
|
||||
compress_parser = subparsers.add_parser("c", help="压缩 PDF 文件")
|
||||
compress_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
compress_parser.add_argument("--output", type=str, default="compressed.pdf", help="输出文件路径")
|
||||
|
||||
# 加密 PDF 命令
|
||||
encrypt_parser = subparsers.add_parser("e", help="加密 PDF 文件")
|
||||
encrypt_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
encrypt_parser.add_argument("--output", type=str, default="encrypted.pdf", help="输出文件路径")
|
||||
encrypt_parser.add_argument("--password", type=str, required=True, help="密码")
|
||||
|
||||
# 解密 PDF 命令
|
||||
decrypt_parser = subparsers.add_parser("d", help="解密 PDF 文件")
|
||||
decrypt_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
decrypt_parser.add_argument("--output", type=str, default="decrypted.pdf", help="输出文件路径")
|
||||
decrypt_parser.add_argument("--password", type=str, required=True, help="密码")
|
||||
|
||||
# 提取文本命令
|
||||
extract_text_parser = subparsers.add_parser("xt", help="提取 PDF 文本")
|
||||
extract_text_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
extract_text_parser.add_argument("--output", type=str, default="output.txt", help="输出文件路径")
|
||||
|
||||
# 提取图片命令
|
||||
extract_images_parser = subparsers.add_parser("xi", help="提取 PDF 图片")
|
||||
extract_images_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
extract_images_parser.add_argument("--output-dir", type=str, default="images", help="输出目录")
|
||||
|
||||
# 添加水印命令
|
||||
watermark_parser = subparsers.add_parser("w", help="添加 PDF 水印")
|
||||
watermark_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
watermark_parser.add_argument("--output", type=str, default="watermarked.pdf", help="输出文件路径")
|
||||
watermark_parser.add_argument("--text", type=str, default="CONFIDENTIAL", help="水印文本")
|
||||
|
||||
# 旋转 PDF 命令
|
||||
rotate_parser = subparsers.add_parser("r", help="旋转 PDF 页面")
|
||||
rotate_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
rotate_parser.add_argument("--output", type=str, default="rotated.pdf", help="输出文件路径")
|
||||
rotate_parser.add_argument("--rotation", type=int, default=90, help="旋转角度 (90, 180, 270)")
|
||||
|
||||
# 裁剪 PDF 命令
|
||||
crop_parser = subparsers.add_parser("crop", help="裁剪 PDF 页面")
|
||||
crop_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
crop_parser.add_argument("--output", type=str, default="cropped.pdf", help="输出文件路径")
|
||||
crop_parser.add_argument("--left", type=int, default=10, help="左边裁剪")
|
||||
crop_parser.add_argument("--top", type=int, default=10, help="顶部裁剪")
|
||||
crop_parser.add_argument("--right", type=int, default=10, help="右边裁剪")
|
||||
crop_parser.add_argument("--bottom", type=int, default=10, help="底部裁剪")
|
||||
|
||||
# 显示信息命令
|
||||
info_parser = subparsers.add_parser("i", help="显示 PDF 信息")
|
||||
info_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
|
||||
# OCR 识别命令
|
||||
ocr_parser = subparsers.add_parser("ocr", help="PDF OCR 识别")
|
||||
ocr_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
ocr_parser.add_argument("--output", type=str, default="ocr.pdf", help="输出文件路径")
|
||||
ocr_parser.add_argument("--lang", type=str, default="chi_sim+eng", help="OCR 语言")
|
||||
|
||||
# 转换图片命令
|
||||
to_images_parser = subparsers.add_parser("img", help="PDF 转图片")
|
||||
to_images_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
to_images_parser.add_argument("--output-dir", type=str, default="images", help="输出目录")
|
||||
to_images_parser.add_argument("--dpi", type=int, default=300, help="图片 DPI")
|
||||
|
||||
# 修复 PDF 命令
|
||||
repair_parser = subparsers.add_parser("repair", help="修复 PDF 文件")
|
||||
repair_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
repair_parser.add_argument("--output", type=str, default="repaired.pdf", help="输出文件路径")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "m":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_merge", fn=pdf_merge, args=([Path(p) for p in args.inputs], Path(args.output)))]
|
||||
)
|
||||
elif args.command == "s":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_split", fn=pdf_split, args=(Path(args.input), Path(args.output_dir)))]
|
||||
)
|
||||
elif args.command == "c":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_compress", fn=pdf_compress, args=(Path(args.input), Path(args.output)))]
|
||||
)
|
||||
elif args.command == "e":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_encrypt", fn=pdf_encrypt, args=(Path(args.input), Path(args.output), args.password))]
|
||||
)
|
||||
elif args.command == "d":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_decrypt", fn=pdf_decrypt, args=(Path(args.input), Path(args.output), args.password))]
|
||||
)
|
||||
elif args.command == "xt":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_extract_text", fn=pdf_extract_text, args=(Path(args.input), Path(args.output)))]
|
||||
)
|
||||
elif args.command == "xi":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_extract_images", fn=pdf_extract_images, args=(Path(args.input), Path(args.output_dir)))]
|
||||
)
|
||||
elif args.command == "w":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pdf_watermark",
|
||||
fn=pdf_add_watermark,
|
||||
args=(Path(args.input), Path(args.output)),
|
||||
kwargs={"text": args.text},
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "r":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pdf_rotate",
|
||||
fn=pdf_rotate,
|
||||
args=(Path(args.input), Path(args.output)),
|
||||
kwargs={"rotation": args.rotation},
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "crop":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pdf_crop",
|
||||
fn=pdf_crop,
|
||||
args=(Path(args.input), Path(args.output)),
|
||||
kwargs={"margins": (args.left, args.top, args.right, args.bottom)},
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "i":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("pdf_info", fn=pdf_info, args=(Path(args.input),))])
|
||||
elif args.command == "ocr":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_ocr", fn=pdf_ocr, args=(Path(args.input), Path(args.output)), kwargs={"lang": args.lang})]
|
||||
)
|
||||
elif args.command == "img":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pdf_to_images",
|
||||
fn=pdf_to_images,
|
||||
args=(Path(args.input), Path(args.output_dir)),
|
||||
kwargs={"dpi": args.dpi},
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "repair":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_repair", fn=pdf_repair, args=(Path(args.input), Path(args.output)))]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,661 @@
|
||||
"""PyFlowX 统一 CLI 入口.
|
||||
|
||||
通过 ``pf <tool> [command] [options]`` 调用所有工具,
|
||||
工具定义在 ``pyflowx.ops`` 子包中, 每个模块用 ``@px.tool`` 装饰器注册.
|
||||
|
||||
用法
|
||||
----
|
||||
pf # 列出所有可用工具
|
||||
pf filedate # 查看 filedate 工具帮助
|
||||
pf filedate add a.txt # 调用 filedate 的 add 子命令
|
||||
pf pymake b # 调用 pymake 的 b 别名
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import difflib
|
||||
import importlib
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.table import Table
|
||||
from rich.text import Text
|
||||
from rich.tree import Tree
|
||||
|
||||
from pyflowx import __version__
|
||||
from pyflowx.tools import _TOOL_REGISTRY, run_tool
|
||||
|
||||
|
||||
class PfApp:
|
||||
"""pf 统一入口应用.
|
||||
|
||||
路由 ``pf <tool> [command]`` 到 ``@px.tool`` 注册的工具或传统 Python 工具.
|
||||
"""
|
||||
|
||||
# 工具名到 ops 模块名的映射 (支持短别名)
|
||||
_TOOL_ALIASES: dict[str, str] = {
|
||||
"autofmt": "autofmt",
|
||||
"af": "autofmt",
|
||||
"bump": "bumpversion",
|
||||
"bumpversion": "bumpversion",
|
||||
"bv": "bumpversion",
|
||||
"clr": "clr",
|
||||
"clearscreen": "clr",
|
||||
"dockercmd": "dockercmd",
|
||||
"docker": "dockercmd",
|
||||
"envdev": "envdev",
|
||||
"env": "envdev",
|
||||
"filedate": "filedate",
|
||||
"fd": "filedate",
|
||||
"filelevel": "filelevel",
|
||||
"fl": "filelevel",
|
||||
"folderback": "folderback",
|
||||
"foldback": "folderback",
|
||||
"fb": "folderback",
|
||||
"folderzip": "folderzip",
|
||||
"foldzip": "folderzip",
|
||||
"fz": "folderzip",
|
||||
"git": "gittool",
|
||||
"gitt": "gittool",
|
||||
"gittool": "gittool",
|
||||
"gt": "gittool",
|
||||
"image": "imagetool",
|
||||
"imagetool": "imagetool",
|
||||
"img": "imagetool",
|
||||
"ls": "lscalc",
|
||||
"lscalc": "lscalc",
|
||||
"msdown": "msdownload",
|
||||
"msdownload": "msdownload",
|
||||
"msd": "msdownload",
|
||||
"pack": "packtool",
|
||||
"packtool": "packtool",
|
||||
"pk": "packtool",
|
||||
"pdf": "pdftool",
|
||||
"pdftool": "pdftool",
|
||||
"pt": "pdftool",
|
||||
"pip": "piptool",
|
||||
"pymake": "pymake",
|
||||
"piptool": "piptool",
|
||||
"pp": "piptool",
|
||||
"reseticon": "reseticoncache",
|
||||
"reseticoncache": "reseticoncache",
|
||||
"ric": "reseticoncache",
|
||||
"screenshot": "screenshot",
|
||||
"scrcap": "screenshot",
|
||||
"ss": "screenshot",
|
||||
"se": "setenv",
|
||||
"setenv": "setenv",
|
||||
"sglang": "sglang",
|
||||
"sg": "sglang",
|
||||
"ssh": "sshcopyid",
|
||||
"sshcopy": "sshcopyid",
|
||||
"sshcopyid": "sshcopyid",
|
||||
"sc": "sshcopyid",
|
||||
"taskk": "taskkill",
|
||||
"taskkill": "taskkill",
|
||||
"tk": "taskkill",
|
||||
"wch": "which",
|
||||
"which": "which",
|
||||
"wf": "writefile",
|
||||
"writefile": "writefile",
|
||||
}
|
||||
|
||||
# 传统工具: 有自己的 main() 函数 (无法 @px.tool 化的复杂逻辑)
|
||||
_LEGACY_TOOLS: dict[str, str] = {
|
||||
"emlman": "pyflowx.cli.legacy.emlmanager:main",
|
||||
"profiler": "pyflowx.cli.legacy.profiler:main",
|
||||
"pxp": "pyflowx.cli.legacy.profiler:main",
|
||||
}
|
||||
|
||||
# 规范工具名 → 完整模块路径 (ops/ 按功能分组后的动态导入映射)
|
||||
_TOOL_MODULES: dict[str, str] = {
|
||||
"autofmt": "pyflowx.ops.dev.autofmt",
|
||||
"bumpversion": "pyflowx.ops.dev.bumpversion",
|
||||
"clr": "pyflowx.ops.system.clr",
|
||||
"dockercmd": "pyflowx.ops.infra.dockercmd",
|
||||
"envdev": "pyflowx.ops.infra.envdev",
|
||||
"filedate": "pyflowx.ops.files.filedate",
|
||||
"filelevel": "pyflowx.ops.files.filelevel",
|
||||
"folderback": "pyflowx.ops.files.folderback",
|
||||
"folderzip": "pyflowx.ops.files.folderzip",
|
||||
"gittool": "pyflowx.ops.dev.gittool",
|
||||
"imagetool": "pyflowx.ops.files.imagetool",
|
||||
"lscalc": "pyflowx.ops.dev.lscalc",
|
||||
"msdownload": "pyflowx.ops.infra.msdownload",
|
||||
"packtool": "pyflowx.ops.dev.packtool",
|
||||
"pdftool": "pyflowx.ops.files.pdftool",
|
||||
"piptool": "pyflowx.ops.dev.piptool",
|
||||
"pymake": "pyflowx.ops.dev.pymake",
|
||||
"reseticoncache": "pyflowx.ops.system.reseticoncache",
|
||||
"screenshot": "pyflowx.ops.files.screenshot",
|
||||
"setenv": "pyflowx.ops.system.setenv",
|
||||
"sglang": "pyflowx.ops.infra.sglang",
|
||||
"sshcopyid": "pyflowx.ops.infra.sshcopyid",
|
||||
"taskkill": "pyflowx.ops.system.taskkill",
|
||||
"which": "pyflowx.ops.system.which",
|
||||
"writefile": "pyflowx.ops.system.writefile",
|
||||
}
|
||||
|
||||
def __init__(self, argv: Sequence[str] | None = None) -> None:
|
||||
self._argv = list(argv) if argv is not None else sys.argv[1:]
|
||||
self._console = Console()
|
||||
self._err = Console(stderr=True)
|
||||
|
||||
def run(self) -> int:
|
||||
"""主入口, 返回退出码."""
|
||||
if not self._argv or self._argv[0] in ("--help", "-h"):
|
||||
self._list_tools()
|
||||
return 0
|
||||
|
||||
first = self._argv[0]
|
||||
if first in ("--version", "-V"):
|
||||
self._console.print(f"PyFlowX [bold cyan]{__version__}[/bold cyan]")
|
||||
return 0
|
||||
|
||||
# 内建子命令(与 @px.tool 注册的 ops 工具同级)
|
||||
builtin = {
|
||||
"yamlrun": self._run_yaml,
|
||||
"graph": self._run_graph,
|
||||
"info": self._run_info,
|
||||
"completion": self._run_completion,
|
||||
}
|
||||
if first in builtin:
|
||||
return builtin[first](self._argv[1:])
|
||||
|
||||
rest = self._argv[1:]
|
||||
resolved = self._resolve_tool(first)
|
||||
if resolved is None:
|
||||
self._print_unknown_tool(first)
|
||||
return 1
|
||||
|
||||
kind, target = resolved
|
||||
if kind == "legacy":
|
||||
return self._run_legacy(target, rest)
|
||||
return self._run_tool(target, rest)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 工具列表 (rich)
|
||||
# ------------------------------------------------------------------ #
|
||||
def _list_tools(self) -> None:
|
||||
"""rich 表格列出所有可用工具."""
|
||||
self._console.print(
|
||||
Panel(
|
||||
Text(f"PyFlowX v{__version__}", style="bold cyan", justify="center"),
|
||||
subtitle="[dim]pf <tool> [command] [options][/dim]",
|
||||
)
|
||||
)
|
||||
|
||||
table = Table(title="@px.tool 工具", show_header=True, header_style="bold", show_lines=False)
|
||||
table.add_column("命令", style="cyan", no_wrap=True)
|
||||
table.add_column("别名", style="dim", no_wrap=True)
|
||||
table.add_column("说明")
|
||||
|
||||
for tool in sorted(set(self._TOOL_ALIASES.values())):
|
||||
aliases = self._aliases_for(tool)
|
||||
table.add_row(f"pf {tool}", ", ".join(aliases), self._tool_description(tool))
|
||||
|
||||
self._console.print(table)
|
||||
|
||||
if self._LEGACY_TOOLS:
|
||||
legacy = Table(title="传统工具", show_header=True, header_style="bold", show_lines=False)
|
||||
legacy.add_column("命令", style="cyan", no_wrap=True)
|
||||
for tool in sorted(self._LEGACY_TOOLS):
|
||||
legacy.add_row(f"pf {tool}")
|
||||
self._console.print(legacy)
|
||||
|
||||
self._console.print("\n[bold]示例:[/bold]")
|
||||
self._console.print(" [cyan]pf filedate add a.txt[/cyan] # 给文件添加日期前缀")
|
||||
self._console.print(" [cyan]pf pymake b[/cyan] # 构建 Python 包")
|
||||
self._console.print(" [cyan]pf gitt c[/cyan] # 清理并查看 git 状态")
|
||||
self._console.print(" [cyan]pf yamlrun pipeline.yaml[/cyan] # 执行 YAML 任务图")
|
||||
|
||||
def _aliases_for(self, canonical: str) -> list[str]:
|
||||
"""获取工具的别名 (不含规范名本身)."""
|
||||
return sorted(a for a, t in self._TOOL_ALIASES.items() if t == canonical and a != canonical)
|
||||
|
||||
def _tool_description(self, tool_name: str) -> str:
|
||||
"""获取工具描述 (从 _TOOL_REGISTRY 中已注册 ToolSpec 的 description/help)."""
|
||||
with contextlib.suppress(ImportError, KeyError):
|
||||
importlib.import_module(self._TOOL_MODULES[tool_name])
|
||||
|
||||
if tool_name not in _TOOL_REGISTRY:
|
||||
return ""
|
||||
|
||||
subs = _TOOL_REGISTRY[tool_name]
|
||||
for spec in subs.values():
|
||||
if spec.description:
|
||||
return spec.description
|
||||
for spec in subs.values():
|
||||
if not spec.hidden and spec.help:
|
||||
return spec.help
|
||||
return ""
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 路由
|
||||
# ------------------------------------------------------------------ #
|
||||
def _resolve_tool(self, name: str) -> tuple[str, str] | None:
|
||||
"""解析工具名, 返回 (类型, 目标)."""
|
||||
if name in self._TOOL_ALIASES:
|
||||
return ("tool", self._TOOL_ALIASES[name])
|
||||
if name in self._LEGACY_TOOLS:
|
||||
return ("legacy", self._LEGACY_TOOLS[name])
|
||||
return None
|
||||
|
||||
def _print_unknown_tool(self, name: str) -> None:
|
||||
"""打印未知工具错误 + 模糊匹配建议."""
|
||||
self._err.print(f"[red]错误:[/red] 未知工具 [yellow]{name!r}[/yellow]")
|
||||
suggestions = difflib.get_close_matches(name, list(self._TOOL_ALIASES), n=3, cutoff=0.5)
|
||||
if suggestions:
|
||||
self._err.print(f"[dim]是否想用: {', '.join(suggestions)}[/dim]")
|
||||
self._err.print("[dim]运行 'pf' 查看可用工具列表[/dim]")
|
||||
|
||||
def _run_legacy(self, module_path: str, argv: list[str]) -> int:
|
||||
"""运行传统工具的 main() 函数."""
|
||||
module_name, func_name = module_path.split(":", 1)
|
||||
module = importlib.import_module(module_name)
|
||||
func = getattr(module, func_name)
|
||||
|
||||
original_argv = sys.argv
|
||||
sys.argv = [f"pf {module_name.split('.')[-1]}", *argv]
|
||||
try:
|
||||
func()
|
||||
return 0
|
||||
except SystemExit as e:
|
||||
return int(e.code) if e.code is not None else 0
|
||||
finally:
|
||||
sys.argv = original_argv
|
||||
|
||||
def _run_yaml(self, argv: list[str]) -> int:
|
||||
"""执行 yamlrun 命令: 从 YAML 文件加载任务图并执行."""
|
||||
import argparse
|
||||
from typing import get_args
|
||||
|
||||
from pyflowx import Graph, run
|
||||
from pyflowx.errors import PyFlowXError, TaskFailedError
|
||||
from pyflowx.executors import Strategy
|
||||
from pyflowx.runner import CliExitCode
|
||||
|
||||
parser = argparse.ArgumentParser(prog="pf yamlrun", description="执行 YAML 任务图")
|
||||
_ = parser.add_argument("file", help="YAML 文件路径")
|
||||
_ = parser.add_argument(
|
||||
"--strategy",
|
||||
choices=list(get_args(Strategy)),
|
||||
default="dependency",
|
||||
help="执行策略 (默认: %(default)s)",
|
||||
)
|
||||
_ = parser.add_argument("--dry-run", action="store_true", help="仅打印执行计划")
|
||||
_ = parser.add_argument("--list", action="store_true", help="列出所有任务名")
|
||||
_ = parser.add_argument("--list-tag", help="--list 模式下只显示含此标签的任务")
|
||||
_ = parser.add_argument(
|
||||
"--list-name",
|
||||
help="--list 模式下按子串过滤任务名(大小写不敏感)",
|
||||
)
|
||||
_ = parser.add_argument("--quiet", action="store_true", help="静默模式")
|
||||
_ = parser.add_argument("--progress", action="store_true", help="显示 rich 进度条")
|
||||
_ = parser.add_argument("--only", help="只运行指定任务(逗号分隔)及其依赖")
|
||||
_ = parser.add_argument("--tags", help="只运行匹配标签的任务(逗号分隔)及其依赖")
|
||||
|
||||
parsed = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
graph = Graph.from_yaml(parsed.file)
|
||||
except (OSError, ValueError, PyFlowXError) as e:
|
||||
self._err.print(f"[red]错误:[/red] 加载 YAML 失败: {e}")
|
||||
return CliExitCode.FAILURE.value
|
||||
|
||||
if parsed.list:
|
||||
self._print_task_list(
|
||||
graph,
|
||||
tag_filter=parsed.list_tag,
|
||||
name_filter=parsed.list_name,
|
||||
)
|
||||
return CliExitCode.SUCCESS.value
|
||||
|
||||
if parsed.dry_run:
|
||||
self._console.print(graph.describe())
|
||||
return CliExitCode.SUCCESS.value
|
||||
|
||||
only = parsed.only.split(",") if parsed.only else None
|
||||
tags = parsed.tags.split(",") if parsed.tags else None
|
||||
progress = parsed.progress and not parsed.quiet
|
||||
|
||||
try:
|
||||
report = run(
|
||||
graph,
|
||||
strategy=parsed.strategy,
|
||||
verbose=not parsed.quiet,
|
||||
only=only,
|
||||
tags=tags,
|
||||
progress=progress,
|
||||
)
|
||||
if not report.success and not parsed.quiet:
|
||||
self._print_diagnostics(report)
|
||||
return CliExitCode.SUCCESS.value if report.success else CliExitCode.FAILURE.value
|
||||
except KeyboardInterrupt:
|
||||
print("\n操作已取消", file=sys.stderr)
|
||||
return CliExitCode.INTERRUPTED.value
|
||||
except PyFlowXError as e:
|
||||
self._err.print(f"[red]错误:[/red] {e}")
|
||||
# TaskFailedError 携带 report 时打印诊断摘要
|
||||
if isinstance(e, TaskFailedError) and e.report is not None and not parsed.quiet:
|
||||
self._print_diagnostics(e.report)
|
||||
return CliExitCode.FAILURE.value
|
||||
|
||||
def _print_diagnostics(self, report: Any) -> None:
|
||||
"""打印失败诊断摘要到 stderr。"""
|
||||
diag = report.diagnose()
|
||||
if diag is not None:
|
||||
self._err.print()
|
||||
self._err.print(diag.describe(), style="red")
|
||||
|
||||
def _print_task_list(
|
||||
self,
|
||||
graph: Any,
|
||||
tag_filter: str | None,
|
||||
name_filter: str | None,
|
||||
) -> None:
|
||||
"""打印任务列表,可按标签/名称子串过滤。"""
|
||||
name_lower = name_filter.lower() if name_filter else None
|
||||
for name in graph.names:
|
||||
if name_lower is not None and name_filter is not None and name_lower not in name.lower():
|
||||
continue
|
||||
if tag_filter is not None:
|
||||
spec = graph.spec(name)
|
||||
if tag_filter not in spec.tags:
|
||||
continue
|
||||
self._console.print(name)
|
||||
|
||||
def _run_graph(self, argv: list[str]) -> int:
|
||||
"""执行 graph 命令: 终端可视化 YAML 任务图 DAG."""
|
||||
import argparse
|
||||
|
||||
from pyflowx import Graph
|
||||
from pyflowx.errors import PyFlowXError
|
||||
from pyflowx.runner import CliExitCode
|
||||
|
||||
parser = argparse.ArgumentParser(prog="pf graph", description="可视化 YAML 任务图 DAG")
|
||||
_ = parser.add_argument("file", help="YAML 文件路径")
|
||||
_ = parser.add_argument(
|
||||
"--format",
|
||||
choices=["ascii", "mermaid", "list", "deps"],
|
||||
default="ascii",
|
||||
help="输出格式 (默认: %(default)s)",
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"--orientation",
|
||||
default="TD",
|
||||
help="Mermaid 方向: TD/TB/BT/LR/RL (默认: %(default)s)",
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"--color-by",
|
||||
choices=["tag", "none"],
|
||||
default="tag",
|
||||
help="ASCII 格式节点着色策略: tag(按首标签着色) / none(单色) (默认: %(default)s)",
|
||||
)
|
||||
|
||||
parsed = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
graph = Graph.from_yaml(parsed.file)
|
||||
except (OSError, ValueError, PyFlowXError) as e:
|
||||
self._err.print(f"[red]错误:[/red] 加载 YAML 失败: {e}")
|
||||
return CliExitCode.FAILURE.value
|
||||
|
||||
fmt = parsed.format
|
||||
try:
|
||||
if fmt == "mermaid":
|
||||
# Mermaid 输出不加颜色,便于复制粘贴到 mermaid.live
|
||||
self._console.print(graph.to_mermaid(parsed.orientation), markup=False)
|
||||
elif fmt == "list":
|
||||
for name in graph.names:
|
||||
self._console.print(name)
|
||||
elif fmt == "deps":
|
||||
self._render_deps_table(graph)
|
||||
else:
|
||||
self._render_ascii(graph, color_by=parsed.color_by)
|
||||
except ValueError as e:
|
||||
self._err.print(f"[red]错误:[/red] {e}")
|
||||
return CliExitCode.FAILURE.value
|
||||
return CliExitCode.SUCCESS.value
|
||||
|
||||
def _render_ascii(self, graph: Any, color_by: str = "tag") -> None:
|
||||
"""用 rich 渲染分层 DAG 视图与依赖关系表。
|
||||
|
||||
``color_by="tag"`` 时按任务首标签选择 rich 颜色,便于视觉分组;
|
||||
``color_by="none"`` 时统一使用 cyan。
|
||||
"""
|
||||
layers = graph.layers()
|
||||
if not layers:
|
||||
self._console.print("[dim]空图(无任务)[/dim]")
|
||||
return
|
||||
|
||||
palette = ("cyan", "magenta", "yellow", "green", "blue", "red", "bright_cyan")
|
||||
tag_colors: dict[str, str] = {}
|
||||
counter = [0]
|
||||
|
||||
def _color_for(name: str) -> str:
|
||||
if color_by == "none":
|
||||
return "cyan"
|
||||
spec = graph.spec(name)
|
||||
if not spec.tags:
|
||||
return "cyan"
|
||||
tag = spec.tags[0]
|
||||
if tag not in tag_colors:
|
||||
tag_colors[tag] = palette[counter[0] % len(palette)]
|
||||
counter[0] += 1
|
||||
return tag_colors[tag]
|
||||
|
||||
tree = Tree(f"[bold cyan]Graph[/bold cyan] ({len(graph)} 任务, {len(layers)} 层)")
|
||||
for i, layer in enumerate(layers, 1):
|
||||
names = ", ".join(f"[{_color_for(n)}]{n}[/{_color_for(n)}]" for n in layer)
|
||||
tree.add(f"[bold]Layer {i}[/bold] ({len(layer)}): {names}")
|
||||
self._console.print(tree)
|
||||
|
||||
if tag_colors and color_by == "tag":
|
||||
legend = " ".join(f"[{c}]{t}[/{c}]" for t, c in tag_colors.items())
|
||||
self._console.print(f"[dim]标签颜色:[/dim] {legend}")
|
||||
|
||||
self._console.print()
|
||||
self._render_deps_table(graph)
|
||||
|
||||
def _render_deps_table(self, graph: Any) -> None:
|
||||
"""渲染任务依赖关系表."""
|
||||
table = Table(title="任务依赖关系", show_header=True, header_style="bold")
|
||||
table.add_column("任务", style="cyan", no_wrap=True)
|
||||
table.add_column("硬依赖", style="yellow")
|
||||
table.add_column("软依赖", style="dim")
|
||||
table.add_column("标签", style="green")
|
||||
|
||||
for name in graph.names:
|
||||
spec = graph.spec(name)
|
||||
hard = ", ".join(spec.depends_on) if spec.depends_on else "-"
|
||||
soft = ", ".join(spec.soft_depends_on) if spec.soft_depends_on else "-"
|
||||
tags = ", ".join(spec.tags) if spec.tags else "-"
|
||||
table.add_row(name, hard, soft, tags)
|
||||
self._console.print(table)
|
||||
|
||||
def _run_tool(self, target: str, argv: list[str]) -> int:
|
||||
"""运行 @px.tool 工具.
|
||||
|
||||
导入 ``pyflowx.ops.<group>.<target>`` 触发 ``@px.tool`` 注册, 然后调用 ``run_tool``.
|
||||
"""
|
||||
with contextlib.suppress(ImportError, KeyError):
|
||||
importlib.import_module(self._TOOL_MODULES[target])
|
||||
|
||||
if target not in _TOOL_REGISTRY:
|
||||
self._err.print(f"[red]错误:[/red] 未找到工具 [yellow]{target!r}[/yellow]")
|
||||
self._err.print("[dim]运行 'pf' 查看可用工具列表[/dim]")
|
||||
return 1
|
||||
|
||||
return run_tool(target, argv)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# info: 工具详情
|
||||
# ------------------------------------------------------------------ #
|
||||
def _run_info(self, argv: list[str]) -> int:
|
||||
"""显示工具详情: ``pf info [tool]``。
|
||||
|
||||
无参数时列出所有工具的简表(含子命令数、描述);
|
||||
指定工具时显示该工具的所有子命令详情。
|
||||
"""
|
||||
if not argv:
|
||||
self._print_all_tools_detail()
|
||||
return 0
|
||||
|
||||
name = argv[0]
|
||||
resolved = self._resolve_tool(name)
|
||||
if resolved is None or resolved[0] != "tool":
|
||||
self._err.print(f"[red]错误:[/red] 未知工具 [yellow]{name!r}[/yellow]")
|
||||
self._err.print("[dim]运行 'pf' 查看可用工具列表[/dim]")
|
||||
return 1
|
||||
|
||||
canonical = resolved[1]
|
||||
self._print_tool_detail(canonical)
|
||||
return 0
|
||||
|
||||
def _print_all_tools_detail(self) -> None:
|
||||
"""打印所有工具的详情表。"""
|
||||
# 触发工具模块导入以填充 _TOOL_REGISTRY
|
||||
for mod_path in self._TOOL_MODULES.values():
|
||||
with contextlib.suppress(ImportError):
|
||||
importlib.import_module(mod_path)
|
||||
|
||||
table = Table(title="工具详情", show_header=True, header_style="bold")
|
||||
table.add_column("工具", style="cyan", no_wrap=True)
|
||||
table.add_column("别名", style="dim")
|
||||
table.add_column("子命令数", justify="right")
|
||||
table.add_column("描述")
|
||||
|
||||
for canonical in sorted(set(self._TOOL_ALIASES.values())):
|
||||
aliases = ", ".join(self._aliases_for(canonical))
|
||||
subs = _TOOL_REGISTRY.get(canonical, {})
|
||||
desc = self._tool_description(canonical)
|
||||
table.add_row(canonical, aliases or "-", str(len(subs)), desc or "-")
|
||||
|
||||
self._console.print(table)
|
||||
|
||||
def _print_tool_detail(self, canonical: str) -> None:
|
||||
"""打印指定工具的子命令详情。"""
|
||||
with contextlib.suppress(ImportError, KeyError):
|
||||
importlib.import_module(self._TOOL_MODULES[canonical])
|
||||
|
||||
subs = _TOOL_REGISTRY.get(canonical, {})
|
||||
aliases = self._aliases_for(canonical)
|
||||
self._console.print(f"[bold cyan]{canonical}[/bold cyan]", end="")
|
||||
if aliases:
|
||||
self._console.print(f" [dim]别名: {', '.join(aliases)}[/dim]")
|
||||
else:
|
||||
self._console.print()
|
||||
|
||||
desc = self._tool_description(canonical)
|
||||
if desc:
|
||||
self._console.print(f"[dim]描述: {desc}[/dim]")
|
||||
|
||||
if not subs:
|
||||
self._console.print("[dim]无子命令(直接调用)[/dim]")
|
||||
return
|
||||
|
||||
table = Table(title=f"{canonical} 子命令", show_header=True, header_style="bold")
|
||||
table.add_column("子命令", style="cyan", no_wrap=True)
|
||||
table.add_column("描述")
|
||||
table.add_column("隐藏", justify="center")
|
||||
|
||||
for name in sorted(n for n in subs if n is not None):
|
||||
spec = subs[name]
|
||||
table.add_row(
|
||||
name,
|
||||
spec.description or spec.help or "-",
|
||||
"是" if spec.hidden else "-",
|
||||
)
|
||||
self._console.print(table)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# completion: shell 补全脚本生成
|
||||
# ------------------------------------------------------------------ #
|
||||
def _run_completion(self, argv: list[str]) -> int:
|
||||
"""生成 shell 补全脚本: ``pf completion bash|zsh|fish``。"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(prog="pf completion", description="生成 shell 补全脚本")
|
||||
_ = parser.add_argument(
|
||||
"shell",
|
||||
choices=["bash", "zsh", "fish"],
|
||||
help="目标 shell",
|
||||
)
|
||||
parsed = parser.parse_args(argv)
|
||||
|
||||
tools = sorted(set(self._TOOL_ALIASES.values()))
|
||||
builtin_cmds = ["yamlrun", "graph", "info", "completion"]
|
||||
all_cmds = builtin_cmds + tools
|
||||
|
||||
if parsed.shell == "bash":
|
||||
self._console.print(self._bash_completion(all_cmds), markup=False)
|
||||
elif parsed.shell == "zsh":
|
||||
self._console.print(self._zsh_completion(all_cmds), markup=False)
|
||||
else:
|
||||
self._console.print(self._fish_completion(all_cmds), markup=False)
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _bash_completion(cmds: list[str]) -> str:
|
||||
"""生成 bash 补全脚本。"""
|
||||
words = " ".join(cmds)
|
||||
return f"""# pf bash 补全 —— 添加到 ~/.bashrc 或保存到 /etc/bash_completion.d/pf
|
||||
_pf_complete() {{
|
||||
local cur prev
|
||||
cur="${{COMP_WORDS[COMP_CWORD]}}"
|
||||
prev="${{COMP_WORDS[COMP_CWORD-1]}}"
|
||||
|
||||
if [ "$COMP_CWORD" -eq 1 ]; then
|
||||
COMPREPLY=($(compgen -W "{words}" -- "$cur"))
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 二级补全:交给具体子命令(pf <tool> --help 自描述)
|
||||
return 0
|
||||
}}
|
||||
complete -F _pf_complete pf
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _zsh_completion(cmds: list[str]) -> str:
|
||||
"""生成 zsh 补全脚本。"""
|
||||
words = " ".join(cmds)
|
||||
return f"""# pf zsh 补全 —— 添加到 ~/.zshrc
|
||||
#compdef pf
|
||||
|
||||
_pf() {{
|
||||
local -a cmds
|
||||
cmds=({words})
|
||||
|
||||
if [ "$CURRENT" -eq 2 ]; then
|
||||
_describe 'command' cmds
|
||||
return 0
|
||||
fi
|
||||
|
||||
return 0
|
||||
}}
|
||||
|
||||
_pf "$@"
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _fish_completion(cmds: list[str]) -> str:
|
||||
"""生成 fish 补全脚本。"""
|
||||
lines = ["# pf fish 补全 —— 保存到 ~/.config/fish/completions/pf.fish"]
|
||||
for c in cmds:
|
||||
lines.append(f'complete -c pf -n "__fish_use_subcommand" -a "{c}"')
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""pf 统一入口主函数."""
|
||||
sys.exit(PfApp().run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,201 +0,0 @@
|
||||
"""pip 包管理工具模块.
|
||||
|
||||
提供 pip 包管理操作的封装,
|
||||
支持安装、卸载、下载等功能.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import fnmatch
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
PACKAGE_DIR = "packages"
|
||||
REQUIREMENTS_FILE = "requirements.txt"
|
||||
|
||||
# 受保护的包名集合
|
||||
_PROTECTED_PACKAGES: frozenset[str] = frozenset(
|
||||
{
|
||||
"pyflowx",
|
||||
"bitool",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _get_installed_packages() -> list[str]:
|
||||
"""获取当前环境中所有已安装的包名."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pip", "list", "--format=freeze"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
packages: list[str] = []
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
if line and "==" in line:
|
||||
pkg_name = line.split("==")[0].strip()
|
||||
packages.append(pkg_name)
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
return []
|
||||
return packages
|
||||
|
||||
|
||||
def _expand_wildcard_packages(pattern: str) -> list[str]:
|
||||
"""展开通配符模式为实际的包名列表."""
|
||||
if not any(char in pattern for char in ["*", "?", "[", "]"]):
|
||||
return [pattern]
|
||||
|
||||
installed_packages = _get_installed_packages()
|
||||
matched = [pkg for pkg in installed_packages if fnmatch.fnmatchcase(pkg.lower(), pattern.lower())]
|
||||
return matched
|
||||
|
||||
|
||||
def _filter_protected_packages(packages: list[str]) -> list[str]:
|
||||
"""过滤掉受保护的包名."""
|
||||
safe = [p for p in packages if p.lower() not in {p.lower() for p in _PROTECTED_PACKAGES}]
|
||||
filtered = [p for p in packages if p.lower() in {p.lower() for p in _PROTECTED_PACKAGES}]
|
||||
if filtered:
|
||||
print(f"跳过受保护的包: {', '.join(filtered)}")
|
||||
return safe
|
||||
|
||||
|
||||
def pip_uninstall(pkg_names: list[str]) -> None:
|
||||
"""卸载包."""
|
||||
packages_to_uninstall: list[str] = []
|
||||
for pattern in pkg_names:
|
||||
packages_to_uninstall.extend(_expand_wildcard_packages(pattern))
|
||||
|
||||
packages_to_uninstall = _filter_protected_packages(packages_to_uninstall)
|
||||
|
||||
if not packages_to_uninstall:
|
||||
return
|
||||
|
||||
subprocess.run(["pip", "uninstall", "-y", *packages_to_uninstall], check=True)
|
||||
|
||||
|
||||
def pip_reinstall(pkg_names: list[str], offline: bool = False) -> None:
|
||||
"""重新安装包."""
|
||||
safe_pkgs = _filter_protected_packages(pkg_names)
|
||||
if not safe_pkgs:
|
||||
print("所有指定的包均为受保护包, 跳过重装")
|
||||
return
|
||||
|
||||
subprocess.run(["pip", "uninstall", "-y", *safe_pkgs], check=True)
|
||||
|
||||
options = ["--no-index", "--find-links", "."] if offline else []
|
||||
subprocess.run(["pip", "install", *options, *safe_pkgs], check=True)
|
||||
|
||||
|
||||
def pip_download(pkg_names: list[str], offline: bool = False) -> None:
|
||||
"""下载包."""
|
||||
options = ["--no-index", "--find-links", "."] if offline else []
|
||||
subprocess.run(
|
||||
["pip", "download", *pkg_names, *options, "-d", PACKAGE_DIR],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def pip_freeze() -> None:
|
||||
"""冻结依赖."""
|
||||
result = subprocess.run(
|
||||
["pip", "freeze", "--exclude-editable"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
Path(REQUIREMENTS_FILE).write_text(result.stdout)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""pip 工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="PipTool - pip 包管理工具",
|
||||
usage="piptool <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 安装命令
|
||||
install_parser = subparsers.add_parser("i", help="安装包")
|
||||
install_parser.add_argument("packages", nargs="+", help="要安装的包名")
|
||||
|
||||
# 卸载命令
|
||||
uninstall_parser = subparsers.add_parser("u", help="卸载包")
|
||||
uninstall_parser.add_argument("packages", nargs="+", help="要卸载的包名 (支持通配符)")
|
||||
|
||||
# 重装命令
|
||||
reinstall_parser = subparsers.add_parser("r", help="重新安装包")
|
||||
reinstall_parser.add_argument("packages", nargs="+", help="要重装的包名")
|
||||
reinstall_parser.add_argument("--offline", action="store_true", help="使用离线模式")
|
||||
|
||||
# 下载命令
|
||||
download_parser = subparsers.add_parser("d", help="下载包")
|
||||
download_parser.add_argument("packages", nargs="+", help="要下载的包名")
|
||||
download_parser.add_argument("--offline", action="store_true", help="使用离线模式")
|
||||
|
||||
# 升级 pip 命令
|
||||
subparsers.add_parser("up", help="升级 pip")
|
||||
|
||||
# 冻结依赖命令
|
||||
subparsers.add_parser("f", help="冻结依赖到 requirements.txt")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "i":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("pip_install", cmd=["pip", "install", *args.packages], verbose=True)])
|
||||
elif args.command == "u":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pip_uninstall", fn=pip_uninstall, args=(args.packages,), verbose=True)]
|
||||
)
|
||||
elif args.command == "r":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pip_reinstall",
|
||||
fn=pip_reinstall,
|
||||
args=(args.packages,),
|
||||
kwargs={"offline": args.offline},
|
||||
verbose=True,
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "d":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pip_download",
|
||||
fn=pip_download,
|
||||
args=(args.packages,),
|
||||
kwargs={"offline": args.offline},
|
||||
verbose=True,
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "up":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pip_upgrade", cmd=["python", "-m", "pip", "install", "--upgrade", "pip"], verbose=True)]
|
||||
)
|
||||
elif args.command == "f":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("pip_freeze", fn=pip_freeze, verbose=True)])
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,133 +0,0 @@
|
||||
"""Python 构建工具模块.
|
||||
|
||||
完全替代传统的 Makefile,
|
||||
提供更好的跨平台兼容性和 Python 生态集成.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
def maturin_build_cmd() -> list[str]:
|
||||
"""获取 maturin 构建命令(根据平台自动添加参数).
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[str]
|
||||
完整的 maturin 构建命令列表.
|
||||
"""
|
||||
command = ["maturin", "build", "-r"].copy()
|
||||
if Constants.IS_WINDOWS:
|
||||
command.extend(
|
||||
[
|
||||
"--target",
|
||||
"x86_64-win7-windows-msvc",
|
||||
"-Zbuild-std",
|
||||
"-i",
|
||||
"python3.8",
|
||||
]
|
||||
)
|
||||
return command
|
||||
|
||||
|
||||
uv_build: px.TaskSpec = px.TaskSpec("uv_build", cmd=["uv", "build"])
|
||||
maturin_build: px.TaskSpec = px.TaskSpec("maturin_build", cmd=maturin_build_cmd())
|
||||
uv_sync: px.TaskSpec = px.TaskSpec("uv_sync", cmd=["uv", "sync"])
|
||||
git_clean: px.TaskSpec = px.TaskSpec("git_clean", cmd=["gitt", "c"])
|
||||
test: px.TaskSpec = px.TaskSpec(
|
||||
"test", cmd=["pytest", "-m", "not slow", "-n", "8", "--dist", "loadfile", "--color=yes", "--durations=10"]
|
||||
)
|
||||
test_fast: px.TaskSpec = px.TaskSpec(
|
||||
"test_fast", cmd=["pytest", "-m", "not slow", "--dist", "loadfile", "--color=yes", "--durations=10"]
|
||||
)
|
||||
test_coverage: px.TaskSpec = px.TaskSpec(
|
||||
"test_coverage",
|
||||
cmd=["pytest", "--cov", "-n", "8", "--dist", "loadfile", "--tb=short", "-v", "--color=yes", "--durations=10"],
|
||||
)
|
||||
ruff_lint: px.TaskSpec = px.TaskSpec("lint", cmd=["ruff", "check", "--fix", "--unsafe-fixes"])
|
||||
ruff_format: px.TaskSpec = px.TaskSpec("format", cmd=["ruff", "format", "."], depends_on=("lint",))
|
||||
typecheck: px.TaskSpec = px.TaskSpec("pyrefly_check", cmd=["pyrefly", "check", "."])
|
||||
git_add_all: px.TaskSpec = px.TaskSpec("git_add_all", cmd=["git", "add", "-A"])
|
||||
bump: px.TaskSpec = px.TaskSpec("bumpversion", cmd=["bumpversion", "-t"])
|
||||
doc: px.TaskSpec = px.TaskSpec("doc", cmd=["sphinx-build", "-b", "html", "docs", "docs/_build"])
|
||||
git_push: px.TaskSpec = px.TaskSpec("git_push", cmd=["git", "push"])
|
||||
git_push_tags: px.TaskSpec = px.TaskSpec("git_push_tags", cmd=["git", "push", "--tags"])
|
||||
hatch_publish: px.TaskSpec = px.TaskSpec("publish_python", cmd=["hatch", "publish"])
|
||||
twine_publish: px.TaskSpec = px.TaskSpec("twine_publish", cmd=["twine", "upload", "--disable-progress-bar"])
|
||||
tox: px.TaskSpec = px.TaskSpec("tox", cmd=["tox", "-p", "auto"])
|
||||
|
||||
|
||||
def main():
|
||||
"""pymake 构建工具.
|
||||
|
||||
🔨 构建命令:
|
||||
pymake b - 构建 Python 主包 (uv build)
|
||||
pymake bc - 构建 Rust 核心模块 (maturin build)
|
||||
pymake ba - 构建所有包 (先 Python 后 Rust)
|
||||
|
||||
📦 安装命令 (开发模式):
|
||||
pymake sync - 安装依赖包 (uv sync)
|
||||
|
||||
🧹 清理命令:
|
||||
pymake c - 清理所有构建产物 (gitt c)
|
||||
|
||||
🛠️ 开发工具:
|
||||
pymake t - 运行测试 (pytest)
|
||||
pymake tc - 运行测试并生成覆盖率报告
|
||||
pymake tf - 运行快速测试 (pytest -m not slow)
|
||||
pymake lint - 代码格式化与检查 (ruff)
|
||||
pymake type - 类型检查 (mypy, ty)
|
||||
pymake doc - 构建文档 (sphinx)
|
||||
|
||||
🔬 多版本测试:
|
||||
pymake tox - 多版本 Python 测试 (tox -p auto)
|
||||
|
||||
📦 发布命令:
|
||||
pymake pb - 发布到 PyPI (twine + hatch)
|
||||
|
||||
� 版本管理:
|
||||
pymake bump - 自动升级版本号并提交修改 (清理 + 检查 + 格式化 + git add + bumpversion)
|
||||
|
||||
�💡 常用工作流:
|
||||
1. 日常开发: pymake lint && pymake t
|
||||
2. 构建发布包: pymake ba
|
||||
3. 多版本兼容性测试: pymake tox
|
||||
4. 发布到 PyPI: pymake pb
|
||||
|
||||
📝 示例:
|
||||
pymake ba # 构建所有包
|
||||
pymake sync # 安装依赖
|
||||
pymake t # 运行测试
|
||||
pymake tox # 多版本兼容性测试
|
||||
pymake lint # 格式化代码
|
||||
pymake type # 类型检查
|
||||
"""
|
||||
runner = px.CliRunner(
|
||||
strategy="sequential",
|
||||
description="PyMake - Python 构建工具",
|
||||
graphs={
|
||||
# 构建命令
|
||||
"b": px.Graph.from_specs([uv_build]),
|
||||
"bc": px.Graph.from_specs([maturin_build]),
|
||||
"ba": px.Graph.from_specs(["b", "bc"]),
|
||||
# 安装命令
|
||||
"sync": px.Graph.from_specs([uv_sync]),
|
||||
# 清理命令
|
||||
"c": px.Graph.from_specs([git_clean]),
|
||||
# 开发工具
|
||||
"bump": px.Graph.from_specs(["c", "tc", git_add_all, bump]),
|
||||
"cov": px.Graph.from_specs([git_clean, test_coverage]),
|
||||
"doc": px.Graph.from_specs([doc]),
|
||||
"lint": px.Graph.from_specs([ruff_lint, ruff_format]),
|
||||
"pb": px.Graph.from_specs([twine_publish, hatch_publish]),
|
||||
"t": px.Graph.from_specs([test]),
|
||||
"tf": px.Graph.from_specs([test_fast]),
|
||||
"tc": px.Graph.from_specs([typecheck, "lint"]),
|
||||
"tox": px.Graph.from_specs([tox]),
|
||||
# 发布命令
|
||||
"p": px.Graph.from_specs([git_clean, git_push, git_push_tags]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
@@ -1,40 +0,0 @@
|
||||
"""进程终止工具.
|
||||
|
||||
跨平台进程终止工具, 支持按名称终止进程.
|
||||
用法: taskkill proc_name [proc_name ...]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""进程终止工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="TaskKill - 进程终止工具",
|
||||
usage="taskkill <process_name> [process_name ...]",
|
||||
)
|
||||
parser.add_argument(
|
||||
"process_names",
|
||||
type=str,
|
||||
nargs="+",
|
||||
help="进程名称 (如: chrome.exe python node)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if Constants.IS_WINDOWS:
|
||||
cmd = ["taskkill", "/f", "/im"]
|
||||
else:
|
||||
cmd = ["pkill", "-f"]
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(f"kill_{proc_name}", cmd=[*cmd, f"{proc_name}*"], verbose=True)
|
||||
for proc_name in args.process_names
|
||||
]
|
||||
)
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,51 +0,0 @@
|
||||
"""命令查找工具.
|
||||
|
||||
跨平台查找可执行命令路径, 类似 Unix 的 which 命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
|
||||
def which_command(command: str) -> Path | None:
|
||||
"""查找命令路径.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
command : str
|
||||
命令名称
|
||||
|
||||
Returns
|
||||
-------
|
||||
Path | None
|
||||
命令路径, 如果未找到则返回 None
|
||||
"""
|
||||
cmd_path = shutil.which(command)
|
||||
if cmd_path:
|
||||
print(f"匹配路径: - {cmd_path}")
|
||||
return Path(cmd_path)
|
||||
else:
|
||||
print(f"{command}: 未找到")
|
||||
return None
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""命令查找工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Which - 命令查找工具",
|
||||
usage="which <command> [command ...]",
|
||||
)
|
||||
parser.add_argument(
|
||||
"commands",
|
||||
type=str,
|
||||
nargs="+",
|
||||
help="要查找的命令名称 (如: python pip node npm git uv rustc cargo)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
graph = px.Graph.from_specs([px.TaskSpec(f"which_{cmd}", fn=which_command, args=(cmd,)) for cmd in args.commands])
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,105 @@
|
||||
"""命令执行器:把 :class:`~pyflowx.task.TaskSpec` 的 ``cmd`` 字段(list /
|
||||
shell 字符串 / 可调用对象)转换为统一执行入口。
|
||||
|
||||
历史背景:原 ``task.py`` 的模块文档声明其为"纯数据结构",但 ``_run_command``
|
||||
属于命令执行逻辑,违反单一职责。此处将其抽离,``TaskSpec`` 仅持有配置,
|
||||
执行逻辑集中于本模块,便于独立测试与维护。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from typing import Any, cast
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from .task import TaskSpec
|
||||
|
||||
__all__ = ["run_command"]
|
||||
|
||||
_cmd_console = Console()
|
||||
|
||||
|
||||
def run_command(spec: TaskSpec[Any]) -> Any: # noqa: PLR0912
|
||||
"""执行 ``spec.cmd`` 指定的命令(list / shell 字符串 / 可调用对象)。
|
||||
|
||||
与原 ``TaskSpec._run_command`` 行为一致:
|
||||
|
||||
- 可调用对象:直接调用,异常包装为 :class:`RuntimeError`。
|
||||
- list / str:通过 :func:`subprocess.run` 执行,非零返回码抛
|
||||
:class:`RuntimeError`(``verbose=False`` 时附 stderr)。
|
||||
- ``verbose=True`` 时打印执行信息与返回码到 stdout。
|
||||
- ``cwd`` / ``env`` 通过 subprocess 参数隔离(进程级状态仅在 fn 任务路径
|
||||
使用,cmd 路径不依赖 ``os.chdir`` / ``os.environ``)。
|
||||
"""
|
||||
cmd = spec.cmd
|
||||
verbose = spec.verbose
|
||||
cwd = spec.cwd
|
||||
timeout = spec.timeout
|
||||
env_override = spec.env
|
||||
|
||||
# 可调用对象:直接调用,返回其结果。
|
||||
if callable(cmd) and not isinstance(cmd, (list, str)):
|
||||
name = getattr(cmd, "__name__", "callable")
|
||||
if verbose:
|
||||
_cmd_console.print(f"[cyan]▸[/cyan] 执行可调用命令: [bold]{name}[/bold]")
|
||||
if cwd is not None:
|
||||
_cmd_console.print(f" [dim]工作目录: {cwd}[/dim]")
|
||||
try:
|
||||
return cmd()
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"可调用命令执行异常: {name}: {e}") from e
|
||||
|
||||
is_list = isinstance(cmd, list)
|
||||
if is_list:
|
||||
cmd_str = " ".join(arg for arg in cmd) # type: ignore[union-attr]
|
||||
verb = "执行命令"
|
||||
label = "命令"
|
||||
else:
|
||||
cmd_str = cast(str, cmd)
|
||||
verb = "执行 Shell"
|
||||
label = "Shell 命令"
|
||||
|
||||
if verbose:
|
||||
_cmd_console.print(f"[cyan]▸[/cyan] {verb}: [bold]{cmd_str}[/bold]")
|
||||
if cwd is not None:
|
||||
_cmd_console.print(f" [dim]工作目录: {cwd}[/dim]")
|
||||
|
||||
# 合并环境变量
|
||||
run_env: dict[str, str] | None = None
|
||||
if env_override:
|
||||
run_env = dict(os.environ)
|
||||
run_env.update(env_override)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cast(str | list[str], cmd),
|
||||
shell=not is_list,
|
||||
cwd=cwd,
|
||||
env=run_env,
|
||||
timeout=timeout,
|
||||
capture_output=not verbose,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError(f"{label}未找到: {cmd_str}") from None
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError(f"{label}执行超时: {cmd_str} ({timeout}s)") from None
|
||||
except OSError as e:
|
||||
raise RuntimeError(f"{label}执行异常: {cmd_str}: {e}") from e
|
||||
|
||||
if verbose:
|
||||
style = "green" if result.returncode == 0 else "red"
|
||||
_cmd_console.print(f"[{style}]返回码: {result.returncode}[/{style}]")
|
||||
|
||||
if result.returncode == 0:
|
||||
if not verbose and result.stdout:
|
||||
print(result.stdout, end="", flush=True)
|
||||
return None
|
||||
|
||||
err_msg = f"{label}执行失败: `{cmd_str}`, 返回码: {result.returncode}"
|
||||
if not verbose and result.stderr.strip():
|
||||
err_msg += f"\n{result.stderr.strip()}"
|
||||
raise RuntimeError(err_msg)
|
||||
@@ -0,0 +1,115 @@
|
||||
"""图组合:将带字符串引用的多个图展开为纯 :class:`~pyflowx.graph.Graph`。
|
||||
|
||||
历史背景:原 ``graph.py`` 同时承载 DAG 构建/校验/分层与多图组合逻辑,
|
||||
职责过载。组合逻辑(:class:`GraphComposer` / :func:`compose`)与单图 DAG
|
||||
模型正交,此处抽离为独立模块,便于按需导入与独立演进。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from typing import Any
|
||||
|
||||
from .graph import Graph
|
||||
from .task import TaskSpec
|
||||
|
||||
__all__ = ["GraphComposer", "compose"]
|
||||
|
||||
|
||||
class GraphComposer:
|
||||
"""将带字符串引用的图展开为纯 :class:`TaskSpec` 图。
|
||||
|
||||
引用格式:
|
||||
* ``"command_name"`` —— 引用整个命令图。
|
||||
* ``"command_name.task_name"`` —— 引用特定任务。
|
||||
|
||||
引用按顺序展开,后续引用的任务依赖前面引用的最后一个任务;
|
||||
原始 ``TaskSpec`` 之间也按出现顺序串行依赖。
|
||||
"""
|
||||
|
||||
def __init__(self, graphs: dict[str, Graph]) -> None:
|
||||
self.graphs = graphs
|
||||
|
||||
def resolve_all(self) -> dict[str, Graph]:
|
||||
"""解析所有图的字符串引用,返回展开后的新图映射。"""
|
||||
resolved: dict[str, Graph] = {}
|
||||
for cmd_name, graph in self.graphs.items():
|
||||
resolved[cmd_name] = self.expand_refs(graph, cmd_name)
|
||||
return resolved
|
||||
|
||||
def expand_refs(self, graph: Graph, current_cmd: str) -> Graph:
|
||||
"""展开图中的字符串引用。若无 ``_pending_refs``,原样返回。"""
|
||||
pending_refs = graph._pending_refs
|
||||
if not pending_refs:
|
||||
return graph
|
||||
|
||||
all_specs: list[TaskSpec[Any]] = []
|
||||
previous_ref_last_task: str | None = None
|
||||
|
||||
for ref in pending_refs:
|
||||
expanded_specs = self.parse_ref(ref, current_cmd)
|
||||
if previous_ref_last_task and expanded_specs:
|
||||
for i, task in enumerate(expanded_specs):
|
||||
if i == 0 or not task.depends_on:
|
||||
expanded_specs[i] = replace(task, depends_on=tuple({*task.depends_on, previous_ref_last_task}))
|
||||
if expanded_specs:
|
||||
previous_ref_last_task = expanded_specs[-1].name
|
||||
all_specs.extend(expanded_specs)
|
||||
|
||||
original_specs = list(graph.all_specs().values())
|
||||
if original_specs:
|
||||
if previous_ref_last_task:
|
||||
first = original_specs[0]
|
||||
all_specs.append(replace(first, depends_on=tuple({*first.depends_on, previous_ref_last_task})))
|
||||
else:
|
||||
all_specs.append(original_specs[0])
|
||||
for i in range(1, len(original_specs)):
|
||||
current_task = original_specs[i]
|
||||
previous_task_name = original_specs[i - 1].name
|
||||
all_specs.append(
|
||||
replace(current_task, depends_on=tuple({*current_task.depends_on, previous_task_name}))
|
||||
)
|
||||
|
||||
return Graph.from_specs(all_specs, defaults=graph.defaults)
|
||||
|
||||
def parse_ref(self, ref: str, current_cmd: str) -> list[TaskSpec[Any]]:
|
||||
"""解析单个字符串引用,返回对应的 TaskSpec 列表。"""
|
||||
if ref == current_cmd:
|
||||
raise ValueError(f"循环引用: 命令 '{current_cmd}' 引用了自己")
|
||||
|
||||
if "." in ref:
|
||||
cmd_name, task_name = ref.split(".", 1)
|
||||
if cmd_name not in self.graphs:
|
||||
raise ValueError(f"引用的命令 '{cmd_name}' 不存在")
|
||||
ref_graph = self.graphs[cmd_name]
|
||||
if task_name not in ref_graph.all_specs():
|
||||
raise ValueError(f"任务 '{task_name}' 不存在于命令 '{cmd_name}' 中")
|
||||
return [ref_graph.all_specs()[task_name]]
|
||||
else:
|
||||
cmd_name = ref
|
||||
if cmd_name not in self.graphs:
|
||||
raise ValueError(f"引用的命令 '{cmd_name}' 不存在")
|
||||
ref_graph = self.graphs[cmd_name]
|
||||
ref_graph = self.expand_refs(ref_graph, cmd_name)
|
||||
return list(ref_graph.all_specs().values())
|
||||
|
||||
|
||||
def compose(
|
||||
graphs: dict[str, Graph],
|
||||
) -> dict[str, Graph]:
|
||||
"""编程式解析多图的字符串引用,返回展开后的新图映射。
|
||||
|
||||
与 :class:`GraphComposer` 等价,但作为独立函数暴露,供不使用
|
||||
:class:`~pyflowx.runner.CliRunner` 的编程式用户调用。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> graphs = {
|
||||
... "build": px.graph(px.cmd(["echo", "b"], name="b")),
|
||||
... "all": px.graph("build", px.cmd(["echo", "t"], name="t")),
|
||||
... }
|
||||
>>> resolved = px.compose(graphs)
|
||||
>>> "b" in resolved["all"].all_specs()
|
||||
True
|
||||
"""
|
||||
return GraphComposer(graphs).resolve_all()
|
||||
+191
-163
@@ -1,18 +1,30 @@
|
||||
"""条件判断模块.
|
||||
|
||||
提供平台条件、应用安装条件等预定义条件判断函数,
|
||||
用于 TaskSpec 的条件执行功能.
|
||||
所有条件均为 ``Callable[[Context], bool]``,接收依赖上下文映射(可能为空)。
|
||||
这使得条件可基于上游任务的运行时返回值做决策,实现动态分支。
|
||||
|
||||
内置条件分两类:
|
||||
1. *静态条件* —— 不依赖上下文(平台/环境变量/安装检查),通过 ``_static``
|
||||
包装忽略传入的 context,便于作为模块级常量使用。
|
||||
2. *上下文条件* —— 基于上游结果判断,如 :meth:`BuiltinConditions.DEP_EQUALS`。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Callable
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# 条件判断函数类型
|
||||
Condition = Callable[[], bool]
|
||||
from .task import Condition, Context
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = ["BuiltinConditions", "Condition", "Constants"]
|
||||
|
||||
|
||||
class Constants:
|
||||
@@ -24,200 +36,216 @@ class Constants:
|
||||
IS_POSIX: bool = sys.platform != "win32"
|
||||
|
||||
|
||||
def _static(predicate: Callable[[], bool], name: str) -> Condition:
|
||||
"""将无参谓词包装为忽略上下文的 :class:`Condition`。"""
|
||||
|
||||
def _cond(_ctx: Context) -> bool:
|
||||
return predicate()
|
||||
|
||||
_cond.__name__ = name
|
||||
return _cond
|
||||
|
||||
|
||||
def _cond_name(cond: Condition) -> str:
|
||||
"""获取条件的可读名称。"""
|
||||
return getattr(cond, "__name__", repr(cond))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 模块级静态条件常量
|
||||
# ---------------------------------------------------------------------- #
|
||||
IS_WINDOWS: Condition = _static(lambda: Constants.IS_WINDOWS, "IS_WINDOWS")
|
||||
IS_LINUX: Condition = _static(lambda: Constants.IS_LINUX, "IS_LINUX")
|
||||
IS_MACOS: Condition = _static(lambda: Constants.IS_MACOS, "IS_MACOS")
|
||||
IS_POSIX: Condition = _static(lambda: Constants.IS_POSIX, "IS_POSIX")
|
||||
|
||||
|
||||
class BuiltinConditions:
|
||||
"""内置条件判断函数集合."""
|
||||
"""内置条件判断函数集合.
|
||||
|
||||
静态条件工厂返回忽略上下文的 :class:`Condition`;上下文条件工厂返回
|
||||
会读取依赖结果的 :class:`Condition`。
|
||||
"""
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 静态条件
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
def IS_WINDOWS() -> Condition:
|
||||
"""检查是否为 Windows 平台."""
|
||||
return IS_WINDOWS
|
||||
|
||||
@staticmethod
|
||||
def IS_WINDOWS() -> bool:
|
||||
"""是否为 Windows 平台."""
|
||||
return Constants.IS_WINDOWS
|
||||
def IS_LINUX() -> Condition:
|
||||
"""检查是否为 Linux 平台."""
|
||||
return IS_LINUX
|
||||
|
||||
@staticmethod
|
||||
def IS_LINUX() -> bool:
|
||||
bool = Constants.IS_LINUX
|
||||
return bool
|
||||
def IS_MACOS() -> Condition:
|
||||
"""检查是否为 macOS 平台."""
|
||||
return IS_MACOS
|
||||
|
||||
@staticmethod
|
||||
def IS_MACOS() -> bool:
|
||||
"""是否为 macOS 平台."""
|
||||
return Constants.IS_MACOS
|
||||
def IS_POSIX() -> Condition:
|
||||
"""检查是否为 POSIX 平台."""
|
||||
return IS_POSIX
|
||||
|
||||
@staticmethod
|
||||
def IS_POSIX() -> bool:
|
||||
"""是否为 POSIX 系统 (Linux/macOS)."""
|
||||
return Constants.IS_POSIX
|
||||
|
||||
@staticmethod
|
||||
def PYTHON_VERSION(major: int, minor: int | None = None) -> bool:
|
||||
"""检查 Python 版本是否匹配.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
major : int
|
||||
主版本号.
|
||||
minor : int | None
|
||||
次版本号, 若为 None 则仅检查主版本.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
版本是否匹配.
|
||||
"""
|
||||
def PYTHON_VERSION(major: int, minor: int | None = None) -> Condition:
|
||||
"""检查 Python 版本是否匹配."""
|
||||
if minor is None:
|
||||
return sys.version_info.major == major
|
||||
return sys.version_info.major == major and sys.version_info.minor == minor
|
||||
return _static(lambda: sys.version_info.major == major, f"PYTHON_VERSION({major})")
|
||||
return _static(
|
||||
lambda: sys.version_info.major == major and sys.version_info.minor == minor,
|
||||
f"PYTHON_VERSION({major},{minor})",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def PYTHON_VERSION_AT_LEAST(major: int, minor: int = 0) -> bool:
|
||||
"""检查 Python 版本是否 >= 指定版本.
|
||||
def PYTHON_VERSION_AT_LEAST(major: int, minor: int = 0) -> Condition:
|
||||
"""检查 Python 版本是否 >= 指定版本."""
|
||||
return _static(lambda: sys.version_info >= (major, minor), f"PYTHON_VERSION_AT_LEAST({major},{minor})")
|
||||
|
||||
Parameters
|
||||
----------
|
||||
major : int
|
||||
主版本号.
|
||||
minor : int
|
||||
次版本号.
|
||||
@staticmethod
|
||||
def IS_RUNNING(app_name: str) -> Condition:
|
||||
"""检查指定应用是否正在运行."""
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
当前版本是否 >= 指定版本.
|
||||
"""
|
||||
return sys.version_info >= (major, minor)
|
||||
def _check() -> bool:
|
||||
if Constants.IS_WINDOWS:
|
||||
result = subprocess.run(
|
||||
["tasklist", "/nh", "/fi", f"imagename eq {app_name}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return app_name.lower() in result.stdout.lower()
|
||||
else:
|
||||
result = subprocess.run(["pgrep", "-x", app_name], capture_output=True, check=False)
|
||||
return result.returncode == 0
|
||||
|
||||
return _static(_check, f"IS_RUNNING({app_name!r})")
|
||||
|
||||
@staticmethod
|
||||
def HAS_INSTALLED(app_name: str) -> Condition:
|
||||
"""检查指定应用是否已安装.
|
||||
"""检查指定应用是否已安装."""
|
||||
return _static(lambda: shutil.which(app_name) is not None, f"HAS_INSTALLED({app_name!r})")
|
||||
|
||||
Parameters
|
||||
----------
|
||||
app_name : str
|
||||
应用名称 (如 "git", "python", "pytest").
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
条件判断函数.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return shutil.which(app_name) is not None
|
||||
|
||||
_check.__name__ = f"HAS_INSTALLED({app_name!r})"
|
||||
return _check
|
||||
@staticmethod
|
||||
def DIR_EXISTS(path: Path) -> Condition:
|
||||
"""路径是否存在."""
|
||||
return _static(path.exists, f"DIR_EXISTS({path!r})")
|
||||
|
||||
@staticmethod
|
||||
def ENV_VAR_EXISTS(var_name: str) -> Condition:
|
||||
"""检查环境变量是否存在.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
var_name : str
|
||||
环境变量名.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
条件判断函数.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return var_name in os.environ
|
||||
|
||||
_check.__name__ = f"ENV_VAR_EXISTS({var_name!r})"
|
||||
return _check
|
||||
"""检查环境变量是否存在."""
|
||||
return _static(lambda: var_name in os.environ, f"ENV_VAR_EXISTS({var_name!r})")
|
||||
|
||||
@staticmethod
|
||||
def ENV_VAR_EQUALS(var_name: str, value: str) -> Condition:
|
||||
"""检查环境变量是否等于指定值.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
var_name : str
|
||||
环境变量名.
|
||||
value : str
|
||||
期望的值.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
条件判断函数.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return os.environ.get(var_name) == value
|
||||
|
||||
_check.__name__ = f"ENV_VAR_EQUALS({var_name!r}, {value!r})"
|
||||
return _check
|
||||
"""检查环境变量是否等于指定值."""
|
||||
return _static(
|
||||
lambda: os.environ.get(var_name) == value,
|
||||
f"ENV_VAR_EQUALS({var_name!r},{value!r})",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def NOT(condition: Condition) -> Condition:
|
||||
"""对条件取反.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
condition : Condition
|
||||
原始条件.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
取反后的条件.
|
||||
"""
|
||||
def FILE_CONTENT_EXISTS(path: Path | str, content: str) -> Condition:
|
||||
"""检查文件是否包含指定内容."""
|
||||
|
||||
def _check() -> bool:
|
||||
return not condition()
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
return False
|
||||
try:
|
||||
return content in p.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return False
|
||||
|
||||
_check.__name__ = f"NOT({getattr(condition, '__name__', repr(condition))})"
|
||||
return _check
|
||||
return _static(_check, f"FILE_CONTENT_EXISTS({path!r},{content!r})")
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 上下文条件:基于上游依赖结果
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
def DEP_EQUALS(dep_name: str, value: Any) -> Condition:
|
||||
"""上游任务 ``dep_name`` 的返回值等于 ``value`` 时为真。
|
||||
|
||||
若依赖未在上下文中(被跳过或未执行),返回 ``False``。
|
||||
"""
|
||||
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return dep_name in ctx and ctx[dep_name] == value
|
||||
|
||||
_cond.__name__ = f"DEP_EQUALS({dep_name!r},{value!r})"
|
||||
return _cond
|
||||
|
||||
@staticmethod
|
||||
def DEP_MATCHES(dep_name: str, predicate: Callable[[Any], bool]) -> Condition:
|
||||
"""上游任务 ``dep_name`` 的返回值满足 ``predicate`` 时为真。
|
||||
|
||||
依赖不存在时返回 ``False``。
|
||||
"""
|
||||
|
||||
def _cond(ctx: Context) -> bool:
|
||||
if dep_name not in ctx:
|
||||
return False
|
||||
try:
|
||||
return predicate(ctx[dep_name])
|
||||
except Exception as exc:
|
||||
logger.warning("DEP_MATCHES predicate %r raised: %r", dep_name, exc)
|
||||
return False
|
||||
|
||||
_cond.__name__ = f"DEP_MATCHES({dep_name!r},{getattr(predicate, '__name__', 'pred')})"
|
||||
return _cond
|
||||
|
||||
@staticmethod
|
||||
def DEP_PRESENT(dep_name: str) -> Condition:
|
||||
"""上游任务 ``dep_name`` 存在于上下文(即已成功执行)时为真。"""
|
||||
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return dep_name in ctx and ctx[dep_name] is not None
|
||||
|
||||
_cond.__name__ = f"DEP_PRESENT({dep_name!r})"
|
||||
return _cond
|
||||
|
||||
@staticmethod
|
||||
def DEP_TRUTHY(dep_name: str) -> Condition:
|
||||
"""上游任务 ``dep_name`` 的返回值为真值时为真。"""
|
||||
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return bool(ctx.get(dep_name))
|
||||
|
||||
_cond.__name__ = f"DEP_TRUTHY({dep_name!r})"
|
||||
return _cond
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 逻辑组合
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
def NOT(condition: Condition) -> Condition:
|
||||
"""对条件取反."""
|
||||
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return not condition(ctx)
|
||||
|
||||
_cond.__name__ = f"NOT({_cond_name(condition)})"
|
||||
return _cond
|
||||
|
||||
@staticmethod
|
||||
def AND(*conditions: Condition) -> Condition:
|
||||
"""多个条件的逻辑与.
|
||||
"""多个条件的逻辑与."""
|
||||
|
||||
Parameters
|
||||
----------
|
||||
*conditions : Condition
|
||||
条件列表.
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return all(c(ctx) for c in conditions)
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
组合条件.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return all(c() for c in conditions)
|
||||
|
||||
names = [getattr(c, "__name__", repr(c)) for c in conditions]
|
||||
_check.__name__ = f"AND({', '.join(names)})"
|
||||
return _check
|
||||
_cond.__name__ = f"AND({', '.join(_cond_name(c) for c in conditions)})"
|
||||
return _cond
|
||||
|
||||
@staticmethod
|
||||
def OR(*conditions: Condition) -> Condition:
|
||||
"""多个条件的逻辑或.
|
||||
"""多个条件的逻辑或."""
|
||||
|
||||
Parameters
|
||||
----------
|
||||
*conditions : Condition
|
||||
条件列表.
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return any(c(ctx) for c in conditions)
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
组合条件.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return any(c() for c in conditions)
|
||||
|
||||
names = [getattr(c, "__name__", repr(c)) for c in conditions]
|
||||
_check.__name__ = f"OR({', '.join(names)})"
|
||||
return _check
|
||||
|
||||
|
||||
# 导出常用条件
|
||||
IS_WINDOWS: Callable[[], bool] = BuiltinConditions.IS_WINDOWS
|
||||
IS_LINUX: Callable[[], bool] = BuiltinConditions.IS_LINUX
|
||||
IS_MACOS: Callable[[], bool] = BuiltinConditions.IS_MACOS
|
||||
IS_POSIX: Callable[[], bool] = BuiltinConditions.IS_POSIX
|
||||
_cond.__name__ = f"OR({', '.join(_cond_name(c) for c in conditions)})"
|
||||
return _cond
|
||||
|
||||
+88
-64
@@ -1,90 +1,129 @@
|
||||
"""上下文注入:把上游结果转换为函数参数。
|
||||
|
||||
本机制让用户可以编写普通函数,其参数名*就是*依赖声明,从而消除其他
|
||||
DAG 库中泛滥的样板包装器(如 ``def wrapper(): return fn(workflow.get_task_result('x'))``)。
|
||||
DAG 库中泛滥的样板包装器。
|
||||
|
||||
注入规则(按顺序求值)
|
||||
----------------------
|
||||
1. **标注为** :class:`Context` 的参数接收完整结果映射。适用于需要遍历
|
||||
所有输入的任务。
|
||||
2. **名称匹配某个依赖**的参数接收该依赖的结果。
|
||||
1. **标注为** :class:`Context` 的参数接收完整结果映射(含硬依赖与软依赖)。
|
||||
2. **名称匹配某个依赖**(硬或软)的参数接收该依赖的结果。
|
||||
3. ``**kwargs`` 参数以 dict 形式接收*所有*依赖结果。
|
||||
4. ``TaskSpec.args`` / ``TaskSpec.kwargs`` 为*非依赖*参数提供静态值。
|
||||
|
||||
若某参数无法解析且无默认值,则抛出 :class:`~pyflowx.errors.InjectionError`,
|
||||
并附带精确错误信息。
|
||||
若某参数无法解析且无默认值,则抛出 :class:`~pyflowx.errors.InjectionError`。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from typing import Any, Mapping
|
||||
from collections.abc import Mapping
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
from .errors import InjectionError
|
||||
from .task import Context, TaskSpec
|
||||
|
||||
__all__ = ["Context", "_is_context_annotation", "build_call_args", "describe_injection"]
|
||||
__all__ = ["Context", "build_call_args", "describe_injection", "is_context_annotation"]
|
||||
|
||||
|
||||
def _is_context_annotation(annotation: Any) -> bool:
|
||||
"""判断参数标注是否为(或指向)``Context``。
|
||||
@lru_cache(maxsize=1024)
|
||||
def _cached_signature(fn: Any) -> inspect.Signature:
|
||||
"""缓存 ``inspect.signature`` 结果(按 fn 对象键控)。
|
||||
|
||||
处理三种形式:
|
||||
* ``Context`` 别名对象本身;
|
||||
* ``__name__``/``_name`` 为 ``Context`` 或 ``Mapping`` 的 typing 别名;
|
||||
* *字符串*标注(``from __future__ import annotations`` 会在运行时
|
||||
把所有标注变为字符串),如 ``"Context"`` 或 ``"px.Context"``。
|
||||
``fn`` 对象在 :meth:`TaskSpec.effective_fn` 缓存后稳定,签名重复内省
|
||||
属纯开销。对不可哈希的可调用对象,调用方回退到直接内省。
|
||||
"""
|
||||
return inspect.signature(fn)
|
||||
|
||||
|
||||
def _signature(fn: Any) -> inspect.Signature:
|
||||
"""获取签名,优先走缓存;``fn`` 不可哈希时回退到直接内省。"""
|
||||
try:
|
||||
return _cached_signature(fn)
|
||||
except TypeError:
|
||||
return inspect.signature(fn)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1024)
|
||||
def _fn_no_dep_injection(fn: Any) -> tuple[tuple[str, ...] | None, str | None]:
|
||||
"""预计算 fn 在无依赖/无静态参数时的注入计划。
|
||||
|
||||
返回 ``(context_params, error_param)``:
|
||||
- ``context_params``: Context 标注参数名元组;``None`` 表示存在必填参数需走慢路径。
|
||||
- ``error_param``: 第一个无默认值且非 Context 标注的参数名(用于错误信息);``None`` 表示无此类参数。
|
||||
"""
|
||||
sig = inspect.signature(fn)
|
||||
context_params: list[str] = []
|
||||
for pname, param in sig.parameters.items():
|
||||
if is_context_annotation(param.annotation):
|
||||
context_params.append(pname)
|
||||
continue
|
||||
if param.default is inspect.Parameter.empty and param.kind not in (
|
||||
inspect.Parameter.VAR_POSITIONAL,
|
||||
inspect.Parameter.VAR_KEYWORD,
|
||||
):
|
||||
return None, pname
|
||||
return tuple(context_params), None
|
||||
|
||||
|
||||
def is_context_annotation(annotation: Any) -> bool:
|
||||
"""判断参数标注是否为(或指向)``Context``。"""
|
||||
if annotation is Context:
|
||||
return True
|
||||
# `from __future__ import annotations` 产生的字符串标注。
|
||||
if isinstance(annotation, str):
|
||||
# 匹配 "Context"、"px.Context"、"pyflowx.Context" 等。
|
||||
return annotation == "Context" or annotation.endswith(".Context")
|
||||
# 按限定名匹配,支持 ``from pyflowx import Context`` 再导出。
|
||||
name = getattr(annotation, "__name__", None) or getattr(annotation, "_name", None)
|
||||
return name in ("Context", "Mapping")
|
||||
|
||||
|
||||
def _try_fast_path(spec: TaskSpec[Any]) -> tuple[tuple[Any, ...], dict[str, Any]] | None:
|
||||
"""尝试快速路径:cmd 无参任务或 fn 无依赖任务。
|
||||
|
||||
返回 ``None`` 表示快速路径不适用,需走慢路径。
|
||||
"""
|
||||
# 快速路径 1:cmd 任务(无 fn)的 effective_fn 是无参闭包。
|
||||
if spec.fn is None and spec.cmd is not None and not spec.args and not spec.kwargs:
|
||||
return (), {}
|
||||
# 快速路径 2:fn 任务无依赖、无静态 args/kwargs 时,跳过 dep_context/collisions/leftover 构建。
|
||||
if not spec.depends_on and not spec.soft_depends_on and not spec.args and not spec.kwargs:
|
||||
fn = spec.effective_fn
|
||||
context_params, error_param = _fn_no_dep_injection(fn)
|
||||
if error_param is not None:
|
||||
raise InjectionError(
|
||||
spec.name,
|
||||
f"parameter {error_param!r} has no dependency, static value, or default.",
|
||||
)
|
||||
assert context_params is not None # error_param is None ⟺ context_params 非 None
|
||||
return (), {p: {} for p in context_params}
|
||||
return None
|
||||
|
||||
|
||||
def build_call_args(
|
||||
spec: TaskSpec[Any],
|
||||
context: Mapping[str, Any],
|
||||
) -> tuple[tuple[Any, ...], dict[str, Any]]:
|
||||
"""解析用于调用 ``spec.fn`` 的 ``(args, kwargs)``。
|
||||
|
||||
参数
|
||||
----
|
||||
spec:
|
||||
任务 spec,提供 ``fn``、``depends_on``、``args``、``kwargs``。
|
||||
context:
|
||||
依赖名 -> 结果值的映射。仅保证本任务自身的 ``depends_on`` 条目
|
||||
存在;其他任务的结果被排除,以保持注入的确定性。
|
||||
|
||||
返回
|
||||
----
|
||||
(args, kwargs)
|
||||
可直接展开为 ``spec.fn(*args, **kwargs)``。
|
||||
|
||||
抛出
|
||||
----
|
||||
InjectionError
|
||||
若必需参数无法满足,或静态 ``kwargs`` 与注入依赖名冲突。
|
||||
``context`` 必须已包含所有硬依赖与软依赖的结果(软依赖被跳过时由
|
||||
执行器填入 :attr:`TaskSpec.defaults` 中的默认值)。
|
||||
"""
|
||||
# 使用 effective_fn 而不是 fn,以支持 cmd 参数
|
||||
fast = _try_fast_path(spec)
|
||||
if fast is not None:
|
||||
return fast
|
||||
|
||||
fn = spec.effective_fn
|
||||
sig = inspect.signature(fn)
|
||||
sig = _signature(fn)
|
||||
params = sig.parameters
|
||||
|
||||
# 检测特殊参数类型。
|
||||
var_keyword = next(
|
||||
(p for p in params.values() if p.kind == inspect.Parameter.VAR_KEYWORD),
|
||||
None,
|
||||
)
|
||||
|
||||
# 与本任务相关的上下文子集。
|
||||
dep_context: dict[str, Any] = {name: context[name] for name in spec.depends_on if name in context}
|
||||
# 本任务相关的上下文子集:硬依赖 + 软依赖。
|
||||
all_deps = set(spec.depends_on) | set(spec.soft_depends_on)
|
||||
dep_context: dict[str, Any] = {name: context[name] for name in all_deps if name in context}
|
||||
|
||||
# 检测静态 kwargs 与依赖名的冲突。
|
||||
collisions = set(spec.kwargs) & set(dep_context)
|
||||
if collisions:
|
||||
raise InjectionError(
|
||||
@@ -96,8 +135,6 @@ def build_call_args(
|
||||
injected_kwargs: dict[str, Any] = {}
|
||||
leftover_dep_results: dict[str, Any] = dict(dep_context)
|
||||
|
||||
# 被 spec.args 消费的位置参数。记录哪些参数名已被位置填充,
|
||||
# 以便在基于名称的注入(依赖 / Context / 静态 kwargs)时跳过。
|
||||
positional_params: list[str] = []
|
||||
positional_kinds = (
|
||||
inspect.Parameter.POSITIONAL_ONLY,
|
||||
@@ -106,33 +143,25 @@ def build_call_args(
|
||||
for pname, param in params.items():
|
||||
if param.kind in positional_kinds:
|
||||
positional_params.append(pname)
|
||||
# 前 len(spec.args) 个位置参数由 spec.args 填充。
|
||||
args_filled: set[str] = set(positional_params[: len(spec.args)])
|
||||
|
||||
for pname, param in params.items():
|
||||
# 跳过已被位置 spec.args 填充的参数。
|
||||
if pname in args_filled:
|
||||
continue
|
||||
|
||||
# 规则 1:标注为 Context -> 完整映射。
|
||||
if _is_context_annotation(param.annotation):
|
||||
if is_context_annotation(param.annotation):
|
||||
injected_kwargs[pname] = dep_context
|
||||
continue
|
||||
|
||||
# 规则 2:名称匹配某个依赖。
|
||||
if pname in dep_context:
|
||||
injected_kwargs[pname] = dep_context[pname]
|
||||
leftover_dep_results.pop(pname, None)
|
||||
continue
|
||||
|
||||
# 规则 3:在循环后通过 **kwargs 处理。
|
||||
|
||||
# 规则 4:静态 kwargs 填充其余参数。
|
||||
if pname in spec.kwargs:
|
||||
injected_kwargs[pname] = spec.kwargs[pname]
|
||||
continue
|
||||
|
||||
# 该参数无来源:必须有默认值,否则报错。
|
||||
if param.default is inspect.Parameter.empty and param.kind not in (
|
||||
inspect.Parameter.VAR_POSITIONAL,
|
||||
inspect.Parameter.VAR_KEYWORD,
|
||||
@@ -142,9 +171,7 @@ def build_call_args(
|
||||
f"parameter {pname!r} has no dependency, static value, or default.",
|
||||
)
|
||||
|
||||
# 规则 3:**kwargs 吞掉剩余依赖结果。
|
||||
if var_keyword is not None and leftover_dep_results:
|
||||
# 先合并静态 kwargs,再合并依赖结果(冲突已在上方拒绝)。
|
||||
merged = dict(spec.kwargs)
|
||||
merged.update(injected_kwargs)
|
||||
merged.update(leftover_dep_results)
|
||||
@@ -154,14 +181,9 @@ def build_call_args(
|
||||
|
||||
|
||||
def describe_injection(spec: TaskSpec[Any]) -> str:
|
||||
"""生成任务参数注入方式的人类可读描述。
|
||||
|
||||
供 ``dry_run`` 使用,在不执行的情况下展示执行计划。
|
||||
"""
|
||||
# 使用 effective_fn 而不是 fn,以支持 cmd 参数
|
||||
"""生成任务参数注入方式的人类可读描述。供 ``dry_run`` 使用。"""
|
||||
fn = spec.effective_fn
|
||||
sig = inspect.signature(fn)
|
||||
# 确定哪些位置参数由 spec.args 填充。
|
||||
sig = _signature(fn)
|
||||
positional_params = [
|
||||
p
|
||||
for p, param in sig.parameters.items()
|
||||
@@ -172,15 +194,17 @@ def describe_injection(spec: TaskSpec[Any]) -> str:
|
||||
)
|
||||
]
|
||||
args_filled = set(positional_params[: len(spec.args)])
|
||||
all_deps = set(spec.depends_on) | set(spec.soft_depends_on)
|
||||
parts = []
|
||||
for pname, param in sig.parameters.items():
|
||||
if pname in args_filled:
|
||||
idx = positional_params.index(pname)
|
||||
parts.append(f"{pname}={spec.args[idx]!r}")
|
||||
elif _is_context_annotation(param.annotation):
|
||||
elif is_context_annotation(param.annotation):
|
||||
parts.append(f"{pname}=<Context>")
|
||||
elif pname in spec.depends_on:
|
||||
parts.append(f"{pname}=<result:{pname}>")
|
||||
elif pname in all_deps:
|
||||
tag = "soft" if pname in spec.soft_depends_on else "dep"
|
||||
parts.append(f"{pname}=<{tag}:{pname}>")
|
||||
elif pname in spec.kwargs:
|
||||
parts.append(f"{pname}={spec.kwargs[pname]!r}")
|
||||
elif param.default is not inspect.Parameter.empty:
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
"""失败诊断:从 :class:`RunReport` 提取结构化的根因分析。
|
||||
|
||||
本模块不依赖 :class:`Graph`,仅通过 ``TaskResult.spec.depends_on`` 重建
|
||||
依赖关系,因此可对反序列化后的报告进行诊断。
|
||||
|
||||
核心概念
|
||||
--------
|
||||
|
||||
* **根因任务(root cause)**:FAILED 状态且其所有硬依赖都是 SUCCESS
|
||||
(或无依赖)的任务。它是依赖链中"最先出问题"的环节。
|
||||
* **依赖链(dependency chain)**:从根因到某个失败任务的路径。
|
||||
* **相似失败聚类**:按 ``(异常类型, 消息前缀)`` 聚类,发现批量失败模式。
|
||||
* **根因提示**:识别常见异常类型(FileNotFoundError/ImportError 等),
|
||||
给出可操作建议。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .report import RunReport
|
||||
from .task import TaskStatus
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FailureCluster:
|
||||
"""相似失败的聚类。
|
||||
|
||||
属性
|
||||
----
|
||||
pattern:
|
||||
异常类型名 + 消息前缀(如 ``"ValueError: invalid input"``)。
|
||||
tasks:
|
||||
同类失败的任务名列表。
|
||||
sample_error:
|
||||
代表性错误消息(``repr(error)``)。
|
||||
"""
|
||||
|
||||
pattern: str
|
||||
tasks: list[str]
|
||||
sample_error: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DependencyChain:
|
||||
"""失败任务的依赖链分析。
|
||||
|
||||
属性
|
||||
----
|
||||
failed_task:
|
||||
失败的任务名。
|
||||
root_cause:
|
||||
依赖链中最先失败的任务(无失败依赖的失败任务)。
|
||||
``failed_task`` 自身即为根因时与 ``failed_task`` 相同。
|
||||
chain:
|
||||
从 ``root_cause`` 到 ``failed_task`` 的任务名路径(含两端)。
|
||||
broken_at:
|
||||
链中第一个非 SUCCESS 的任务(通常等于 ``root_cause``)。
|
||||
"""
|
||||
|
||||
failed_task: str
|
||||
root_cause: str | None
|
||||
chain: list[str]
|
||||
broken_at: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DiagnosticReport:
|
||||
"""运行失败的诊断报告。
|
||||
|
||||
属性
|
||||
----
|
||||
failed_tasks:
|
||||
所有 FAILED 状态的任务名。
|
||||
skipped_due_to_failure:
|
||||
因上游失败而 SKIPPED 的任务名(reason 含"失败"或"取消")。
|
||||
root_causes:
|
||||
根因任务列表(FAILED 且所有硬依赖 SUCCESS)。
|
||||
dependency_chains:
|
||||
每个失败任务的依赖链分析。
|
||||
failure_clusters:
|
||||
相似失败聚类。
|
||||
hints:
|
||||
针对根因异常的可操作建议。
|
||||
"""
|
||||
|
||||
failed_tasks: list[str]
|
||||
skipped_due_to_failure: list[str]
|
||||
root_causes: list[str]
|
||||
dependency_chains: list[DependencyChain]
|
||||
failure_clusters: list[FailureCluster]
|
||||
hints: list[str] = field(default_factory=list)
|
||||
|
||||
def describe(self) -> str:
|
||||
"""人类可读的多行诊断报告。"""
|
||||
lines: list[str] = ["=== 失败诊断报告 ==="]
|
||||
|
||||
# 根因
|
||||
if self.root_causes:
|
||||
lines.append(f"\n根因任务 ({len(self.root_causes)}):")
|
||||
for rc in self.root_causes:
|
||||
lines.append(f" - {rc}")
|
||||
else:
|
||||
lines.append("\n根因任务: 无法识别(可能所有失败任务的依赖均未成功)")
|
||||
|
||||
# 依赖链
|
||||
if self.dependency_chains:
|
||||
lines.append(f"\n依赖链 ({len(self.dependency_chains)}):")
|
||||
for dc in self.dependency_chains:
|
||||
chain_str = " -> ".join(dc.chain) if dc.chain else dc.failed_task
|
||||
lines.append(f" {dc.failed_task}: {chain_str}")
|
||||
|
||||
# 相似失败聚类
|
||||
if self.failure_clusters:
|
||||
lines.append(f"\n相似失败聚类 ({len(self.failure_clusters)}):")
|
||||
for cluster in self.failure_clusters:
|
||||
tasks_str = ", ".join(cluster.tasks)
|
||||
lines.append(f" [{cluster.pattern}] ({len(cluster.tasks)} 任务): {tasks_str}")
|
||||
|
||||
# 跳过的任务
|
||||
if self.skipped_due_to_failure:
|
||||
lines.append(f"\n因失败而跳过的任务 ({len(self.skipped_due_to_failure)}):")
|
||||
for name in self.skipped_due_to_failure:
|
||||
lines.append(f" - {name}")
|
||||
|
||||
# 提示
|
||||
if self.hints:
|
||||
lines.append("\n建议:")
|
||||
for hint in self.hints:
|
||||
lines.append(f" * {hint}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 内部辅助
|
||||
# ---------------------------------------------------------------------- #
|
||||
_HINT_MAP: dict[str, str] = {
|
||||
"FileNotFoundError": "文件不存在,检查路径配置与工作目录",
|
||||
"ImportError": "模块导入失败,检查依赖是否安装(pip install / uv add)",
|
||||
"ModuleNotFoundError": "模块不存在,检查依赖安装或 PYTHONPATH",
|
||||
"TimeoutError": "任务超时,考虑增加 timeout 配置或优化任务性能",
|
||||
"TaskTimeoutError": "任务超时,考虑增加 timeout 配置或优化任务性能",
|
||||
"ConnectionError": "网络连接失败,检查目标地址可达性与防火墙设置",
|
||||
"PermissionError": "权限不足,检查文件/目录权限或以管理员身份运行",
|
||||
"KeyError": "键缺失,检查上下文注入参数名是否匹配任务签名",
|
||||
"InjectionError": "上下文注入失败,检查任务签名参数与上游任务名",
|
||||
"StorageError": "状态后端错误,检查存储路径权限与磁盘空间",
|
||||
"CycleError": "依赖图存在环,检查 depends_on 是否形成循环",
|
||||
"MemoryError": "内存不足,考虑分批处理或减少并发数",
|
||||
"OSError": "系统错误,检查文件描述符/磁盘空间/权限",
|
||||
}
|
||||
|
||||
|
||||
def _classify_error(error: BaseException | None) -> tuple[str, str]:
|
||||
"""返回 ``(异常类型名, 消息前缀)`` 用于聚类。"""
|
||||
if error is None:
|
||||
return ("Unknown", "")
|
||||
type_name = type(error).__name__
|
||||
msg = str(error)
|
||||
prefix = msg[:50] if len(msg) > 50 else msg
|
||||
return (type_name, prefix)
|
||||
|
||||
|
||||
def _find_root_cause(
|
||||
failed_task: str,
|
||||
report: RunReport,
|
||||
) -> tuple[str | None, list[str]]:
|
||||
"""回溯 ``failed_task`` 的依赖链,返回 (根因任务, 路径)。
|
||||
|
||||
路径从根因到 ``failed_task``(含两端)。根因为 FAILED 且所有硬依赖
|
||||
非 FAILED 的任务;若 ``failed_task`` 自身满足条件,则为其自身。
|
||||
"""
|
||||
# BFS 反向遍历,找到链中"最深处"的根因
|
||||
visited: set[str] = set()
|
||||
|
||||
# 用 DFS 记录路径:path 始终以当前 task 开头,向根因方向延伸
|
||||
def _trace(task: str, path: list[str]) -> tuple[str | None, list[str]]:
|
||||
if task in visited: # 防止循环依赖(diagnose 不依赖 Graph 校验,results 可能有环)
|
||||
return (None, path)
|
||||
visited.add(task)
|
||||
result = report.results[task] # _trace 只接收 failed_tasks 或 failed_deps,均保证在 results 中
|
||||
|
||||
# 检查此任务是否为根因:FAILED 且所有硬依赖非 FAILED
|
||||
deps = result.spec.depends_on
|
||||
failed_deps = [d for d in deps if d in report.results and report.results[d].status == TaskStatus.FAILED]
|
||||
|
||||
if result.status == TaskStatus.FAILED and not failed_deps:
|
||||
return (task, path)
|
||||
|
||||
# 递归向上找
|
||||
for dep in failed_deps:
|
||||
root, new_path = _trace(dep, [dep, *path])
|
||||
if root is not None:
|
||||
return (root, new_path)
|
||||
|
||||
# 递归未找到明确根因时,当前任务作为断点返回
|
||||
return (task, path)
|
||||
|
||||
return _trace(failed_task, [failed_task])
|
||||
|
||||
|
||||
def _generate_hints(report: RunReport, root_causes: list[str]) -> list[str]:
|
||||
"""针对根因异常生成可操作建议。"""
|
||||
hints: list[str] = []
|
||||
seen_types: set[str] = set()
|
||||
for rc_name in root_causes:
|
||||
result = report.results.get(rc_name)
|
||||
if result is None or result.error is None:
|
||||
continue
|
||||
type_name = type(result.error).__name__
|
||||
if type_name in seen_types:
|
||||
continue
|
||||
seen_types.add(type_name)
|
||||
# 精确匹配
|
||||
if type_name in _HINT_MAP:
|
||||
hints.append(f"{rc_name}: {_HINT_MAP[type_name]}")
|
||||
continue
|
||||
# 父类匹配
|
||||
for parent in type(result.error).__mro__:
|
||||
parent_name = parent.__name__
|
||||
if parent_name in _HINT_MAP:
|
||||
hints.append(f"{rc_name}: {_HINT_MAP[parent_name]}")
|
||||
break
|
||||
return hints
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 公共 API
|
||||
# ---------------------------------------------------------------------- #
|
||||
def diagnose(report: RunReport) -> DiagnosticReport | None:
|
||||
"""生成失败诊断报告。
|
||||
|
||||
运行成功(``report.success`` 为 ``True``)时返回 ``None``;
|
||||
失败时返回 :class:`DiagnosticReport`,包含根因分析、依赖链回溯、
|
||||
相似失败聚类与可操作建议。
|
||||
|
||||
本函数仅依赖 ``report.results`` 中的 ``spec.depends_on`` 重建依赖
|
||||
关系,不依赖 :class:`Graph` 对象,因此可对反序列化后的报告进行诊断。
|
||||
"""
|
||||
if report.success:
|
||||
return None
|
||||
|
||||
failed_tasks = report.failed_tasks()
|
||||
if not failed_tasks:
|
||||
# success=False 但无 FAILED 任务(可能是用户中断等情况)
|
||||
return DiagnosticReport(
|
||||
failed_tasks=[],
|
||||
skipped_due_to_failure=report.skipped_tasks(),
|
||||
root_causes=[],
|
||||
dependency_chains=[],
|
||||
failure_clusters=[],
|
||||
hints=["运行未成功但无失败任务,可能是被取消或中断"],
|
||||
)
|
||||
|
||||
# 识别根因:FAILED 且所有硬依赖非 FAILED
|
||||
root_causes: list[str] = []
|
||||
for name in failed_tasks:
|
||||
result = report.results[name]
|
||||
failed_deps = [
|
||||
d for d in result.spec.depends_on if d in report.results and report.results[d].status == TaskStatus.FAILED
|
||||
]
|
||||
if not failed_deps:
|
||||
root_causes.append(name)
|
||||
|
||||
# 依赖链回溯
|
||||
chains: list[DependencyChain] = []
|
||||
for failed in failed_tasks:
|
||||
root, path = _find_root_cause(failed, report)
|
||||
# broken_at 是路径中第一个非 SUCCESS 的任务
|
||||
broken_at: str | None = None
|
||||
for task in path:
|
||||
r = report.results.get(task)
|
||||
if r is not None and r.status != TaskStatus.SUCCESS:
|
||||
broken_at = task
|
||||
break
|
||||
chains.append(
|
||||
DependencyChain(
|
||||
failed_task=failed,
|
||||
root_cause=root,
|
||||
chain=path,
|
||||
broken_at=broken_at,
|
||||
)
|
||||
)
|
||||
|
||||
# 相似失败聚类
|
||||
clusters_map: dict[tuple[str, str], list[str]] = defaultdict(list)
|
||||
sample_errors: dict[tuple[str, str], str] = {}
|
||||
for name in failed_tasks:
|
||||
result = report.results[name]
|
||||
key = _classify_error(result.error)
|
||||
clusters_map[key].append(name)
|
||||
if key not in sample_errors:
|
||||
sample_errors[key] = repr(result.error) if result.error else ""
|
||||
|
||||
clusters: list[FailureCluster] = [
|
||||
FailureCluster(
|
||||
pattern=f"{type_name}: {prefix}" if prefix else type_name,
|
||||
tasks=sorted(tasks),
|
||||
sample_error=sample_errors.get((type_name, prefix), ""),
|
||||
)
|
||||
for (type_name, prefix), tasks in sorted(clusters_map.items(), key=lambda x: -len(x[1]))
|
||||
]
|
||||
|
||||
# 因失败而跳过的任务(reason 包含"失败"或"取消"或"skip")
|
||||
skipped_due_to_failure: list[str] = []
|
||||
for name, result in report.results.items():
|
||||
if result.status == TaskStatus.SKIPPED:
|
||||
reason = (result.reason or "").lower()
|
||||
if any(kw in reason for kw in ("失败", "取消", "fail", "cancel", "abort")):
|
||||
skipped_due_to_failure.append(name)
|
||||
|
||||
# 根因提示
|
||||
hints = _generate_hints(report, root_causes)
|
||||
|
||||
return DiagnosticReport(
|
||||
failed_tasks=failed_tasks,
|
||||
skipped_due_to_failure=skipped_due_to_failure,
|
||||
root_causes=root_causes,
|
||||
dependency_chains=chains,
|
||||
failure_clusters=clusters,
|
||||
hints=hints,
|
||||
)
|
||||
@@ -6,7 +6,11 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Iterable
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .report import RunReport
|
||||
|
||||
|
||||
class PyFlowXError(Exception):
|
||||
@@ -47,7 +51,8 @@ class TaskFailedError(PyFlowXError):
|
||||
"""任务耗尽所有重试后仍失败时抛出。
|
||||
|
||||
原始异常保留在 :attr:`__cause__` 上,同时通过 :attr:`cause` 暴露,
|
||||
便于用户代码访问。
|
||||
便于用户代码访问。``report`` 字段(可选)携带失败时的 :class:`RunReport`,
|
||||
便于调用方生成诊断报告。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -56,6 +61,7 @@ class TaskFailedError(PyFlowXError):
|
||||
cause: BaseException,
|
||||
attempts: int,
|
||||
layer: int | None = None,
|
||||
report: RunReport | None = None,
|
||||
) -> None:
|
||||
location = f" (layer {layer})" if layer is not None else ""
|
||||
super().__init__(f"Task '{task}' failed after {attempts} attempt(s){location}: {cause}")
|
||||
@@ -63,6 +69,7 @@ class TaskFailedError(PyFlowXError):
|
||||
self.cause = cause
|
||||
self.attempts = attempts
|
||||
self.layer = layer
|
||||
self.report = report
|
||||
|
||||
|
||||
class TaskTimeoutError(PyFlowXError):
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
"""Example 3: async aggregation with static args and Context injection.
|
||||
|
||||
Shows:
|
||||
* async task functions executed with strategy="async".
|
||||
* static positional args (TaskSpec.args) for parameterised tasks.
|
||||
* Context annotation to receive the full upstream result mapping.
|
||||
* on_event callback for real-time progress.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
|
||||
async def fetch_user(uid: int) -> dict[str, Any]:
|
||||
await asyncio.sleep(0.2)
|
||||
return {"id": uid, "name": f"User{uid}"}
|
||||
|
||||
|
||||
async def fetch_posts(uid: int) -> list[int]:
|
||||
await asyncio.sleep(0.2)
|
||||
return [uid, uid + 1]
|
||||
|
||||
|
||||
# Context annotation → receives the full mapping of upstream results.
|
||||
def aggregate(ctx: px.Context) -> dict[str, Any]:
|
||||
return dict(ctx)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
# Static positional args parameterise the same function twice.
|
||||
px.TaskSpec("fetch_user", fetch_user, args=(1,)),
|
||||
px.TaskSpec("fetch_posts", fetch_posts, args=(1,)),
|
||||
px.TaskSpec("aggregate", aggregate, depends_on=("fetch_user", "fetch_posts")),
|
||||
]
|
||||
)
|
||||
|
||||
print("=== Dry run ===")
|
||||
_ = px.run(graph, strategy="async", dry_run=True)
|
||||
|
||||
events: list[px.TaskEvent] = []
|
||||
print("\n=== Async execution ===")
|
||||
report = px.run(graph, strategy="async", on_event=events.append)
|
||||
|
||||
for ev in events:
|
||||
print(f" event: {ev.task} -> {ev.status.value}")
|
||||
|
||||
print(f"\naggregate = {report['aggregate']}")
|
||||
print(report.describe())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,75 +0,0 @@
|
||||
"""Example 1: ETL pipeline (sequential strategy).
|
||||
|
||||
Demonstrates the core PyFlowX workflow:
|
||||
* Define tasks as plain functions.
|
||||
* Declare the DAG with a list of TaskSpec.
|
||||
* Parameter names == dependency names → automatic context injection,
|
||||
no wrappers needed (contrast with flowweaver's get_task_result boilerplate).
|
||||
* dry_run to preview, then execute and read typed results from RunReport.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# --- task functions: pure, testable, no framework coupling ------------- #
|
||||
|
||||
|
||||
def extract_customers() -> list[dict]:
|
||||
return [
|
||||
{"id": "C001", "name": "Alice"},
|
||||
{"id": "C002", "name": "Bob"},
|
||||
]
|
||||
|
||||
|
||||
def extract_orders() -> list[dict]:
|
||||
return [
|
||||
{"id": "O001", "customer_id": "C001", "amount": 150.0},
|
||||
{"id": "O002", "customer_id": "C002", "amount": 200.5},
|
||||
]
|
||||
|
||||
|
||||
# Parameter names match dependency names → automatic injection.
|
||||
def transform(
|
||||
extract_customers: list[dict],
|
||||
extract_orders: list[dict],
|
||||
) -> list[dict]:
|
||||
cmap = {c["id"]: c for c in extract_customers}
|
||||
return [{**o, "customer_name": cmap[o["customer_id"]]["name"]} for o in extract_orders if o["customer_id"] in cmap]
|
||||
|
||||
|
||||
def load(transform: list[dict]) -> int:
|
||||
print(f" loaded {len(transform)} records")
|
||||
return len(transform)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("extract_customers", extract_customers, tags=("extract",)),
|
||||
px.TaskSpec("extract_orders", extract_orders, tags=("extract",)),
|
||||
px.TaskSpec(
|
||||
"transform",
|
||||
transform,
|
||||
depends_on=("extract_customers", "extract_orders"),
|
||||
tags=("transform",),
|
||||
),
|
||||
px.TaskSpec("load", load, depends_on=("transform",), retries=1, tags=("load",)),
|
||||
]
|
||||
)
|
||||
|
||||
print("=== Execution plan ===")
|
||||
print(graph.describe())
|
||||
|
||||
print("\n=== Dry run (no execution) ===")
|
||||
_ = px.run(graph, strategy="sequential", dry_run=True)
|
||||
|
||||
print("\n=== Sequential execution ===")
|
||||
report = px.run(graph, strategy="sequential")
|
||||
print(report.describe())
|
||||
print(f"\nload result = {report['load']}")
|
||||
print(f"summary = {report.summary()}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,59 +0,0 @@
|
||||
"""Example 2: parallel execution (thread strategy).
|
||||
|
||||
Same DAG run with sequential vs. thread strategy to show layer-internal
|
||||
parallelism. Tasks within a layer run concurrently; layers are barriers.
|
||||
|
||||
Layer 1: [fetch_a, fetch_b] (parallel)
|
||||
Layer 2: [merge] (waits for both)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
|
||||
def fetch_a() -> str:
|
||||
time.sleep(0.5)
|
||||
return "a"
|
||||
|
||||
|
||||
def fetch_b() -> str:
|
||||
time.sleep(0.5)
|
||||
return "b"
|
||||
|
||||
|
||||
def merge(fetch_a: str, fetch_b: str) -> str:
|
||||
return fetch_a + fetch_b
|
||||
|
||||
|
||||
def main() -> None:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("fetch_a", fetch_a),
|
||||
px.TaskSpec("fetch_b", fetch_b),
|
||||
px.TaskSpec("merge", merge, depends_on=("fetch_a", "fetch_b")),
|
||||
]
|
||||
)
|
||||
|
||||
print("=== Mermaid diagram ===")
|
||||
print(graph.to_mermaid("LR"))
|
||||
|
||||
print("\n=== Sequential (expect ~1.0s) ===")
|
||||
start = time.time()
|
||||
report_seq = px.run(graph, strategy="sequential")
|
||||
t_seq = time.time() - start
|
||||
print(f" result={report_seq['merge']} time={t_seq:.2f}s")
|
||||
|
||||
print("\n=== Threaded (expect ~0.5s) ===")
|
||||
start = time.time()
|
||||
report_thr = px.run(graph, strategy="thread", max_workers=2)
|
||||
t_thr = time.time() - start
|
||||
print(f" result={report_thr['merge']} time={t_thr:.2f}s")
|
||||
|
||||
print(f"\nspeedup = {t_seq / t_thr:.2f}x")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+1228
-351
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,678 @@
|
||||
"""文件操作底层 API —— 纯函数式, 供 DAG 组合调用.
|
||||
|
||||
与 ``ops/files/`` 下的 CLI 工具(pdftool/imagetool/filedate 等)分层:
|
||||
本模块函数**返回数据**(不打印),可被 :func:`px.task` / :class:`px.TaskSpec`
|
||||
包装后在 DAG 中通过 ``outputs`` 声明 + :meth:`RunReport.output_of` 提取,
|
||||
或通过参数名注入传递给下游任务。
|
||||
|
||||
四类能力
|
||||
--------
|
||||
* **查找遍历**::func:`find` / :func:`glob` / :func:`walk_files`
|
||||
* **读写**::func:`read_text` / :func:`read_bytes` / :func:`write_text`
|
||||
/ :func:`write_bytes` / :func:`append_text`
|
||||
* **复制移动删除**::func:`copy` / :func:`move` / :func:`delete`
|
||||
/ :func:`copy_tree`
|
||||
* **元数据校验**::func:`size` / :func:`mtime` / :func:`exists`
|
||||
/ :func:`is_file` / :func:`is_dir` / :func:`sha256` / :func:`stem`
|
||||
/ :func:`suffix`
|
||||
|
||||
设计原则
|
||||
--------
|
||||
- 纯函数, 返回数据, 无副作用优先(写操作返回目标路径或写入字节数)
|
||||
- 仅依赖标准库(pathlib / hashlib / shutil)
|
||||
- 完整类型注解 + 中文 docstring
|
||||
- EAFP 优于 LBYL:文件不存在等错误由标准库异常表达
|
||||
|
||||
DAG 组合示例
|
||||
------------
|
||||
>>> import pyflowx as px
|
||||
>>> from pyflowx.fileops import find, read_text
|
||||
>>> @px.task
|
||||
... def find_logs() -> list[Path]: return find("/var/log", "*.log")
|
||||
>>> @px.task
|
||||
... def read(find_logs: list[Path]) -> list[str]: return [read_text(p) for p in find_logs]
|
||||
>>> graph = px.graph(find_logs, read) # read 自动依赖 find_logs
|
||||
>>> report = px.run(graph)
|
||||
>>> logs = report["find_logs"]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import shutil
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
__all__ = [
|
||||
"append_text",
|
||||
"copy",
|
||||
"copy_tree",
|
||||
"delete",
|
||||
"exists",
|
||||
"find",
|
||||
"glob",
|
||||
"is_dir",
|
||||
"is_file",
|
||||
"move",
|
||||
"mtime",
|
||||
"read_bytes",
|
||||
"read_text",
|
||||
"sha256",
|
||||
"size",
|
||||
"stem",
|
||||
"suffix",
|
||||
"walk_files",
|
||||
"write_bytes",
|
||||
"write_text",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 查找与遍历
|
||||
# ---------------------------------------------------------------------- #
|
||||
def find(
|
||||
root: str | Path,
|
||||
pattern: str = "*",
|
||||
*,
|
||||
recursive: bool = True,
|
||||
max_depth: int | None = None,
|
||||
only_files: bool = False,
|
||||
only_dirs: bool = False,
|
||||
) -> list[Path]:
|
||||
"""查找匹配 ``pattern`` 的路径.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root : str | Path
|
||||
查找根目录
|
||||
pattern : str
|
||||
glob 模式 (默认 ``"*"`` 匹配全部), 如 ``"*.py"`` / ``"test_*.txt"``
|
||||
recursive : bool
|
||||
是否递归子目录 (默认 True)
|
||||
max_depth : int | None
|
||||
递归最大深度 (None 表示无限制; 0 表示仅 root 自身用 pattern 匹配).
|
||||
仅 ``recursive=True`` 时生效. 深度语义: 1 = root 直接子项,
|
||||
2 = root 孙项, 以此类推.
|
||||
only_files : bool
|
||||
仅返回文件 (默认 False)
|
||||
only_dirs : bool
|
||||
仅返回目录 (默认 False); ``only_files`` 与 ``only_dirs`` 同时为 True
|
||||
时返回空列表
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[Path]
|
||||
匹配的路径列表, 按路径名字典序排序
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> find("src", "*.py") # 递归查找所有 .py 文件
|
||||
>>> find("tests", "test_*.py", recursive=False) # 仅 tests 顶层
|
||||
>>> find(".", max_depth=1) # 仅当前目录直接子项
|
||||
"""
|
||||
root_path = Path(root)
|
||||
if not root_path.is_dir():
|
||||
raise FileNotFoundError(f"查找根目录不存在或非目录: {root_path}")
|
||||
if only_files and only_dirs:
|
||||
return []
|
||||
|
||||
if recursive:
|
||||
if max_depth is not None:
|
||||
matches = _glob_with_depth(root_path, pattern, max_depth)
|
||||
else:
|
||||
matches = list(root_path.rglob(pattern))
|
||||
else:
|
||||
matches = list(root_path.glob(pattern))
|
||||
|
||||
result: list[Path] = []
|
||||
seen: set[Path] = set()
|
||||
for p in matches:
|
||||
if p in seen:
|
||||
continue
|
||||
seen.add(p)
|
||||
if only_files and not p.is_file():
|
||||
continue
|
||||
if only_dirs and not p.is_dir():
|
||||
continue
|
||||
result.append(p)
|
||||
result.sort()
|
||||
return result
|
||||
|
||||
|
||||
def _glob_with_depth(root: Path, pattern: str, max_depth: int) -> list[Path]:
|
||||
"""在 ``max_depth`` 深度内递归 glob 匹配."""
|
||||
if max_depth < 0:
|
||||
return []
|
||||
matches = list(root.glob(pattern))
|
||||
if max_depth == 0:
|
||||
return matches
|
||||
for child in root.iterdir():
|
||||
if child.is_dir():
|
||||
matches.extend(_glob_with_depth(child, pattern, max_depth - 1))
|
||||
return matches
|
||||
|
||||
|
||||
def glob(pattern: str, root: str | Path | None = None) -> list[Path]:
|
||||
"""简化 glob 接口.
|
||||
|
||||
与 :func:`find` 区别: ``pattern`` 可包含路径分隔符 (如
|
||||
``"src/**/*.py"``), ``root`` 默认为当前目录.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
pattern : str
|
||||
glob 模式, 支持递归 ``**``
|
||||
root : str | Path | None
|
||||
根目录 (None 时用 ``Path(".")``)
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[Path]
|
||||
匹配的路径列表, 按路径名字典序排序
|
||||
"""
|
||||
base = Path(root) if root is not None else Path()
|
||||
return sorted(base.glob(pattern))
|
||||
|
||||
|
||||
def walk_files(root: str | Path, *, max_depth: int | None = None) -> Iterator[Path]:
|
||||
"""生成器: 深度优先遍历 ``root`` 下所有文件.
|
||||
|
||||
与 :func:`find` 区别: 返回迭代器 (惰性), 且**仅返回文件** (不返回目录),
|
||||
不做 pattern 匹配.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root : str | Path
|
||||
遍历根目录
|
||||
max_depth : int | None
|
||||
最大深度 (None 无限制; 0 表示仅 root 直接子文件)
|
||||
|
||||
Yields
|
||||
------
|
||||
Path
|
||||
文件路径
|
||||
"""
|
||||
root_path = Path(root)
|
||||
if not root_path.is_dir():
|
||||
raise FileNotFoundError(f"遍历根目录不存在或非目录: {root_path}")
|
||||
|
||||
def _walk(path: Path, depth: int) -> Iterator[Path]:
|
||||
if max_depth is not None and depth > max_depth:
|
||||
return
|
||||
for child in sorted(path.iterdir()):
|
||||
if child.is_file():
|
||||
yield child
|
||||
elif child.is_dir():
|
||||
yield from _walk(child, depth + 1)
|
||||
|
||||
yield from _walk(root_path, 0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 读写
|
||||
# ---------------------------------------------------------------------- #
|
||||
def read_text(path: str | Path, encoding: str = "utf-8") -> str:
|
||||
"""读取文本文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
文件路径
|
||||
encoding : str
|
||||
文本编码 (默认 ``"utf-8"``)
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
文件文本内容
|
||||
|
||||
Raises
|
||||
------
|
||||
FileNotFoundError
|
||||
文件不存在
|
||||
"""
|
||||
return Path(path).read_text(encoding=encoding)
|
||||
|
||||
|
||||
def read_bytes(path: str | Path) -> bytes:
|
||||
"""读取二进制文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
文件路径
|
||||
|
||||
Returns
|
||||
-------
|
||||
bytes
|
||||
文件字节内容
|
||||
|
||||
Raises
|
||||
------
|
||||
FileNotFoundError
|
||||
文件不存在
|
||||
"""
|
||||
return Path(path).read_bytes()
|
||||
|
||||
|
||||
def write_text(path: str | Path, content: str, encoding: str = "utf-8") -> int:
|
||||
"""写入文本文件.
|
||||
|
||||
父目录不存在时自动创建.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
文件路径
|
||||
content : str
|
||||
文本内容
|
||||
encoding : str
|
||||
文本编码 (默认 ``"utf-8"``)
|
||||
|
||||
Returns
|
||||
-------
|
||||
int
|
||||
写入字节数
|
||||
"""
|
||||
p = Path(path)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = content.encode(encoding)
|
||||
p.write_bytes(data)
|
||||
return len(data)
|
||||
|
||||
|
||||
def write_bytes(path: str | Path, content: bytes) -> int:
|
||||
"""写入二进制文件.
|
||||
|
||||
父目录不存在时自动创建.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
文件路径
|
||||
content : bytes
|
||||
字节内容
|
||||
|
||||
Returns
|
||||
-------
|
||||
int
|
||||
写入字节数
|
||||
"""
|
||||
p = Path(path)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_bytes(content)
|
||||
return len(content)
|
||||
|
||||
|
||||
def append_text(path: str | Path, content: str, encoding: str = "utf-8") -> int:
|
||||
"""追加文本到文件.
|
||||
|
||||
父目录不存在时自动创建; 文件不存在时创建新文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
文件路径
|
||||
content : str
|
||||
追加文本内容
|
||||
encoding : str
|
||||
文本编码 (默认 ``"utf-8"``)
|
||||
|
||||
Returns
|
||||
-------
|
||||
int
|
||||
追加字节数
|
||||
"""
|
||||
p = Path(path)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = content.encode(encoding)
|
||||
with open(p, "ab") as f:
|
||||
f.write(data)
|
||||
return len(data)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 复制 / 移动 / 删除
|
||||
# ---------------------------------------------------------------------- #
|
||||
def copy(src: str | Path, dst: str | Path, *, overwrite: bool = False) -> Path:
|
||||
"""复制文件或目录树.
|
||||
|
||||
语义:
|
||||
- ``overwrite=False`` 且 ``dst`` 是目录: ``src`` 复制到 ``dst / src.name``
|
||||
- ``overwrite=False`` 且 ``dst`` 是已存在文件: 抛 ``FileExistsError``
|
||||
- ``overwrite=True`` 且 ``dst`` 存在: 先删除 ``dst``, 再把 ``src`` 复制到 ``dst`` 路径
|
||||
|
||||
``src`` 为目录时递归复制 (等价于 :func:`copy_tree`).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
src : str | Path
|
||||
源路径
|
||||
dst : str | Path
|
||||
目标路径
|
||||
overwrite : bool
|
||||
目标已存在时是否覆盖 (默认 False; False 时目标存在抛 ``FileExistsError``)
|
||||
|
||||
Returns
|
||||
-------
|
||||
Path
|
||||
实际目标路径
|
||||
|
||||
Raises
|
||||
------
|
||||
FileNotFoundError
|
||||
``src`` 不存在
|
||||
FileExistsError
|
||||
``overwrite=False`` 且目标已存在 (且非目录)
|
||||
"""
|
||||
src_path = Path(src)
|
||||
dst_path = Path(dst)
|
||||
|
||||
if not src_path.exists():
|
||||
raise FileNotFoundError(f"源路径不存在: {src_path}")
|
||||
|
||||
if overwrite and dst_path.exists():
|
||||
if dst_path.is_dir():
|
||||
shutil.rmtree(dst_path)
|
||||
else:
|
||||
dst_path.unlink()
|
||||
elif not overwrite and dst_path.is_dir():
|
||||
dst_path = dst_path / src_path.name
|
||||
|
||||
if dst_path.exists() and not overwrite:
|
||||
raise FileExistsError(f"目标已存在 (overwrite=False): {dst_path}")
|
||||
|
||||
dst_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if src_path.is_dir():
|
||||
shutil.copytree(src_path, dst_path)
|
||||
else:
|
||||
shutil.copy2(src_path, dst_path)
|
||||
return dst_path
|
||||
|
||||
|
||||
def move(src: str | Path, dst: str | Path, *, overwrite: bool = False) -> Path:
|
||||
"""移动文件或目录.
|
||||
|
||||
跨文件系统时自动 fallback 到 复制+删除.
|
||||
|
||||
语义:
|
||||
- ``overwrite=False`` 且 ``dst`` 是目录: ``src`` 移入 ``dst / src.name``
|
||||
- ``overwrite=False`` 且 ``dst`` 是已存在文件: 抛 ``FileExistsError``
|
||||
- ``overwrite=True`` 且 ``dst`` 存在: 先删除 ``dst``, 再把 ``src`` 移到 ``dst`` 路径
|
||||
|
||||
Parameters
|
||||
----------
|
||||
src : str | Path
|
||||
源路径
|
||||
dst : str | Path
|
||||
目标路径
|
||||
overwrite : bool
|
||||
目标已存在时是否覆盖 (默认 False)
|
||||
|
||||
Returns
|
||||
-------
|
||||
Path
|
||||
实际目标路径
|
||||
|
||||
Raises
|
||||
------
|
||||
FileNotFoundError
|
||||
``src`` 不存在
|
||||
FileExistsError
|
||||
``overwrite=False`` 且目标已存在 (且非目录)
|
||||
"""
|
||||
src_path = Path(src)
|
||||
dst_path = Path(dst)
|
||||
|
||||
if not src_path.exists():
|
||||
raise FileNotFoundError(f"源路径不存在: {src_path}")
|
||||
|
||||
if overwrite and dst_path.exists():
|
||||
if dst_path.is_dir():
|
||||
shutil.rmtree(dst_path)
|
||||
else:
|
||||
dst_path.unlink()
|
||||
elif not overwrite and dst_path.is_dir():
|
||||
dst_path = dst_path / src_path.name
|
||||
|
||||
if dst_path.exists() and not overwrite:
|
||||
raise FileExistsError(f"目标已存在 (overwrite=False): {dst_path}")
|
||||
|
||||
dst_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(src_path), str(dst_path))
|
||||
return dst_path
|
||||
|
||||
|
||||
def delete(path: str | Path, *, missing_ok: bool = False) -> bool:
|
||||
"""删除文件或目录树.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
待删路径
|
||||
missing_ok : bool
|
||||
路径不存在时是否静默返回 False (默认 False; False 时抛 ``FileNotFoundError``)
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
是否实际删除 (``missing_ok=True`` 且不存在时返回 False)
|
||||
|
||||
Raises
|
||||
------
|
||||
FileNotFoundError
|
||||
``missing_ok=False`` 且路径不存在
|
||||
"""
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
if missing_ok:
|
||||
return False
|
||||
raise FileNotFoundError(f"路径不存在: {p}")
|
||||
if p.is_dir():
|
||||
shutil.rmtree(p)
|
||||
else:
|
||||
p.unlink()
|
||||
return True
|
||||
|
||||
|
||||
def copy_tree(src: str | Path, dst: str | Path, *, overwrite: bool = False) -> list[Path]:
|
||||
"""递归复制目录树, 返回所有已复制文件路径.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
src : str | Path
|
||||
源目录
|
||||
dst : str | Path
|
||||
目标目录 (不存在则创建)
|
||||
overwrite : bool
|
||||
目标已存在时是否覆盖整个目录 (默认 False)
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[Path]
|
||||
已复制的文件路径列表 (相对于 ``dst``, 按字典序)
|
||||
|
||||
Raises
|
||||
------
|
||||
FileNotFoundError
|
||||
``src`` 不存在或非目录
|
||||
FileExistsError
|
||||
``overwrite=False`` 且 ``dst`` 已存在
|
||||
"""
|
||||
src_path = Path(src)
|
||||
dst_path = Path(dst)
|
||||
|
||||
if not src_path.is_dir():
|
||||
raise FileNotFoundError(f"源目录不存在或非目录: {src_path}")
|
||||
|
||||
if dst_path.exists() and not overwrite:
|
||||
raise FileExistsError(f"目标已存在 (overwrite=False): {dst_path}")
|
||||
|
||||
if dst_path.exists() and overwrite:
|
||||
shutil.rmtree(dst_path)
|
||||
|
||||
shutil.copytree(src_path, dst_path)
|
||||
return sorted(p for p in dst_path.rglob("*") if p.is_file())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 元数据与校验
|
||||
# ---------------------------------------------------------------------- #
|
||||
def size(path: str | Path) -> int:
|
||||
"""返回文件大小 (字节数).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
文件路径
|
||||
|
||||
Returns
|
||||
-------
|
||||
int
|
||||
字节数
|
||||
|
||||
Raises
|
||||
------
|
||||
FileNotFoundError
|
||||
文件不存在
|
||||
"""
|
||||
return Path(path).stat().st_size
|
||||
|
||||
|
||||
def mtime(path: str | Path) -> float:
|
||||
"""返回文件修改时间 (Unix 时间戳, 秒).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
文件路径
|
||||
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
修改时间戳
|
||||
|
||||
Raises
|
||||
------
|
||||
FileNotFoundError
|
||||
文件不存在
|
||||
"""
|
||||
return Path(path).stat().st_mtime
|
||||
|
||||
|
||||
def exists(path: str | Path) -> bool:
|
||||
"""判断路径是否存在.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
路径
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
是否存在 (跟随符号链接)
|
||||
"""
|
||||
return Path(path).exists()
|
||||
|
||||
|
||||
def is_file(path: str | Path) -> bool:
|
||||
"""判断是否为常规文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
路径
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
是否为文件 (不存在返回 False)
|
||||
"""
|
||||
return Path(path).is_file()
|
||||
|
||||
|
||||
def is_dir(path: str | Path) -> bool:
|
||||
"""判断是否为目录.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
路径
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
是否为目录 (不存在返回 False)
|
||||
"""
|
||||
return Path(path).is_dir()
|
||||
|
||||
|
||||
def sha256(path: str | Path, *, chunk_size: int = 65536) -> str:
|
||||
"""计算文件 SHA256 哈希.
|
||||
|
||||
流式读取, 适用于大文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
文件路径
|
||||
chunk_size : int
|
||||
分块大小 (默认 64KB)
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
十六进制哈希字符串
|
||||
|
||||
Raises
|
||||
------
|
||||
FileNotFoundError
|
||||
文件不存在
|
||||
"""
|
||||
p = Path(path)
|
||||
h = hashlib.sha256()
|
||||
with open(p, "rb") as f:
|
||||
while True:
|
||||
chunk = f.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def stem(path: str | Path) -> str:
|
||||
"""返回文件名 (不含扩展名).
|
||||
|
||||
与 :attr:`pathlib.PurePath.stem` 一致: ``"a.txt"`` → ``"a"``,
|
||||
``"archive.tar.gz"`` → ``"archive.tar"``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
路径
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
文件名主体
|
||||
"""
|
||||
return Path(path).stem
|
||||
|
||||
|
||||
def suffix(path: str | Path) -> str:
|
||||
"""返回扩展名 (含点).
|
||||
|
||||
与 :attr:`pathlib.PurePath.suffix` 一致: ``"a.txt"`` → ``".txt"``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
路径
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
扩展名 (含前导点; 无扩展名返回空串)
|
||||
"""
|
||||
return Path(path).suffix
|
||||
+653
-150
@@ -1,32 +1,177 @@
|
||||
"""DAG 构建、校验、分层与可视化。
|
||||
|
||||
使用标准库的 :mod:`graphlib`(3.9+)或 :mod:`graphlib_backport`(3.8)
|
||||
进行拓扑排序。图以增量方式构建并即时校验,使配置错误在构建时(而非执行时)快速失败。
|
||||
使用自实现的 Kahn 算法进行拓扑排序。图以增量方式构建并即时校验,
|
||||
使配置错误在构建时(而非执行时)快速失败。
|
||||
|
||||
支持:
|
||||
* 图级默认值 :class:`GraphDefaults`,TaskSpec 字段为 ``None`` 时回退。
|
||||
* :meth:`Graph.map` 工厂批量生成 fan-out 任务。
|
||||
* 字符串引用与 :func:`compose` 编程式组合多个图。
|
||||
* 软依赖:仅用于上下文注入,不参与拓扑分层。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterable, Mapping, Sequence
|
||||
__all__ = [
|
||||
"Graph",
|
||||
"GraphDefaults",
|
||||
]
|
||||
|
||||
import inspect
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .context import is_context_annotation
|
||||
from .errors import CycleError, DuplicateTaskError, MissingDependencyError
|
||||
from .task import TaskSpec
|
||||
|
||||
# graphlib 自 3.9 起进入标准库;3.8 回退到 backport。
|
||||
if sys.version_info >= (3, 9): # pragma: no cover
|
||||
import graphlib # pyright: ignore[reportUnreachable]
|
||||
|
||||
_TopologicalSorter = graphlib.TopologicalSorter
|
||||
else: # pragma: no cover
|
||||
import graphlib # type: ignore[import-untyped] # pragma: no cover
|
||||
|
||||
_TopologicalSorter = graphlib.TopologicalSorter # pragma: no cover
|
||||
from .task import Context, RetryPolicy, TaskSpec
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
def _topological_layers(deps: Mapping[str, tuple[str, ...]]) -> tuple[list[list[str]], list[str] | None]:
|
||||
"""Kahn 算法分层拓扑排序。
|
||||
|
||||
返回 ``(layers, cycle_nodes)``:无环时 ``cycle_nodes`` 为 ``None``;
|
||||
有环时为参与环的未处理节点列表(非精确环路径,仅指示存在环)。
|
||||
|
||||
直接用 dict + list 计算入度与反向邻接,在 diamond(1000) 图上约
|
||||
0.45 万 ops/s(含 Graph 构造开销)。
|
||||
"""
|
||||
# 入度(依赖数)与反向邻接表
|
||||
in_degree: dict[str, int] = {}
|
||||
dependents: dict[str, list[str]] = {}
|
||||
for name, d in deps.items():
|
||||
in_degree[name] = len(d)
|
||||
dependents.setdefault(name, [])
|
||||
for name, d in deps.items():
|
||||
for dep in d:
|
||||
if dep in dependents:
|
||||
dependents[dep].append(name)
|
||||
|
||||
# 初始层:入度为 0 的节点
|
||||
current = sorted(name for name, deg in in_degree.items() if deg == 0)
|
||||
layers: list[list[str]] = []
|
||||
processed = 0
|
||||
|
||||
while current:
|
||||
layers.append(current)
|
||||
nxt: list[str] = []
|
||||
for node in current:
|
||||
for dependent in dependents[node]:
|
||||
in_degree[dependent] -= 1
|
||||
if in_degree[dependent] == 0:
|
||||
nxt.append(dependent)
|
||||
processed += 1
|
||||
nxt.sort()
|
||||
current = nxt
|
||||
|
||||
if processed < len(deps):
|
||||
cycle_nodes = [name for name, deg in in_degree.items() if deg > 0]
|
||||
return layers, cycle_nodes
|
||||
|
||||
return layers, None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class GraphDefaults:
|
||||
"""图级默认值。TaskSpec 对应字段为 ``None`` 时回退到此处。
|
||||
|
||||
仅对可空字段生效(retry/timeout/strategy/env/cwd/tags/priority/
|
||||
continue_on_error/concurrency_key)。非空字段(name/fn/cmd)不回退。
|
||||
"""
|
||||
|
||||
retry: RetryPolicy | None = None
|
||||
timeout: float | None = None
|
||||
strategy: str | None = None
|
||||
tags: tuple[str, ...] = ()
|
||||
env: Mapping[str, str] | None = None
|
||||
cwd: Any = None # Path | None
|
||||
priority: int = 0
|
||||
continue_on_error: bool = False
|
||||
concurrency_key: str | None = None
|
||||
verbose: bool = False
|
||||
|
||||
|
||||
def _prune_deps(spec: TaskSpec[Any], keep: Callable[[str], bool]) -> TaskSpec[Any]:
|
||||
"""返回新 spec,其 ``depends_on`` / ``soft_depends_on`` 仅保留 ``keep(dep)`` 为真的依赖。"""
|
||||
return replace(
|
||||
spec,
|
||||
depends_on=tuple(d for d in spec.depends_on if keep(d)),
|
||||
soft_depends_on=tuple(d for d in spec.soft_depends_on if keep(d)),
|
||||
)
|
||||
|
||||
|
||||
def _make_namespaced_fn(orig_fn: Any, ns: str, dep_names: set[str]) -> Any:
|
||||
"""包装 fn,使其能接收带 ``ns:`` 前缀的依赖名,调用时映射回原参数名。
|
||||
|
||||
命名空间合并后,依赖名带前缀(如 ``build:extract``),但 Python 参数名
|
||||
不能含 ``:``。wrapper 用 ``**kwargs`` 接收所有依赖,内部把带前缀的依赖名
|
||||
映射回原参数名后调用原 fn。
|
||||
|
||||
无依赖参数时直接返回原 fn。
|
||||
"""
|
||||
if not dep_names or orig_fn is None:
|
||||
return orig_fn
|
||||
try:
|
||||
orig_sig = inspect.signature(orig_fn)
|
||||
except (TypeError, ValueError):
|
||||
return orig_fn
|
||||
|
||||
# 带前缀依赖名 -> 原参数名
|
||||
name_map: dict[str, str] = {f"{ns}:{orig}": orig for orig in dep_names}
|
||||
prefix = f"{ns}:"
|
||||
|
||||
# 检查原 fn 是否有 Context 标注参数
|
||||
context_param_name: str | None = None
|
||||
for p in orig_sig.parameters.values():
|
||||
ann = p.annotation
|
||||
if ann is not Context and not (isinstance(ann, str) and ann.endswith("Context")):
|
||||
continue
|
||||
context_param_name = p.name
|
||||
break
|
||||
|
||||
if context_param_name is not None:
|
||||
|
||||
def wrapper(ctx: Any = None, **kwargs: Any) -> Any:
|
||||
# ctx 是 dep_context,键为带前缀的依赖名;映射回原始键
|
||||
orig_ctx: dict[str, Any] = {}
|
||||
for k, v in (ctx or {}).items():
|
||||
orig_ctx[name_map.get(k, k)] = v
|
||||
# kwargs 中带前缀的依赖也映射回原参数名
|
||||
for k, v in kwargs.items():
|
||||
if k in name_map:
|
||||
orig_ctx[name_map[k]] = v
|
||||
return orig_fn(**{context_param_name: orig_ctx})
|
||||
|
||||
ctx_param = inspect.Parameter("ctx", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=Context)
|
||||
kw_param = inspect.Parameter("kwargs", inspect.Parameter.VAR_KEYWORD)
|
||||
wrapper.__signature__ = inspect.Signature( # type: ignore[attr-defined]
|
||||
parameters=[ctx_param, kw_param],
|
||||
return_annotation=orig_sig.return_annotation,
|
||||
)
|
||||
else:
|
||||
|
||||
def wrapper(**kwargs: Any) -> Any: # type: ignore[no-redef]
|
||||
orig_kwargs: dict[str, Any] = {}
|
||||
for k, v in kwargs.items():
|
||||
if k.startswith(prefix):
|
||||
orig_kwargs[k[len(prefix) :]] = v
|
||||
return orig_fn(**orig_kwargs)
|
||||
|
||||
kw_param = inspect.Parameter("kwargs", inspect.Parameter.VAR_KEYWORD)
|
||||
wrapper.__signature__ = inspect.Signature( # type: ignore[attr-defined]
|
||||
parameters=[kw_param],
|
||||
return_annotation=orig_sig.return_annotation,
|
||||
)
|
||||
|
||||
wrapper.__name__ = f"{ns}_{getattr(orig_fn, '__name__', 'fn')}"
|
||||
wrapper.__doc__ = getattr(orig_fn, "__doc__", None)
|
||||
return wrapper
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Graph:
|
||||
"""校验后不可变的有向无环任务图。
|
||||
"""校验后的有向无环任务图。
|
||||
|
||||
通过添加 :class:`~pyflowx.task.TaskSpec` 实例构建。每次 ``add`` 都
|
||||
执行即时校验(重名、缺失依赖),:meth:`validate` / :meth:`layers`
|
||||
@@ -38,108 +183,370 @@ class Graph:
|
||||
|
||||
specs: dict[str, TaskSpec[Any]] = field(default_factory=dict)
|
||||
deps: dict[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
defaults: GraphDefaults = field(default_factory=GraphDefaults)
|
||||
namespace: str | None = None
|
||||
|
||||
# 待解析的字符串引用列表(由 GraphComposer 消费);为空表示无引用。
|
||||
_pending_refs: list[str] = field(default_factory=list)
|
||||
|
||||
# resolved_spec 缓存:避免执行期每个任务多次重复 dataclasses.replace 判断。
|
||||
# 在 specs / defaults 变更时失效。
|
||||
_resolved_cache: dict[str, TaskSpec[Any]] = field(default_factory=dict)
|
||||
|
||||
# layers() 缓存:避免重复 run() 调用时重算拓扑排序。
|
||||
# 在 specs 变更时失效。
|
||||
_layers_cache: list[list[str]] | None = field(default=None)
|
||||
|
||||
# loop 组映射:原 spec 名 → 展开后的实例名列表。
|
||||
# 用于依赖重写:其他任务引用 loop 原名时,自动展开为依赖所有实例。
|
||||
_loop_groups: dict[str, list[str]] = field(default_factory=dict)
|
||||
|
||||
# 任务分组映射:组名 → 组内任务名元组。
|
||||
# 用于依赖重写:引用组名等价于依赖组内所有任务。
|
||||
_groups: dict[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 构建
|
||||
# ------------------------------------------------------------------ #
|
||||
def add(self, spec: TaskSpec[Any]) -> Graph:
|
||||
"""注册一个任务 spec,并即时校验。
|
||||
def _auto_infer_deps_single(self, spec: TaskSpec[Any], all_names: set[str]) -> TaskSpec[Any]:
|
||||
"""对 ``depends_on`` 为空的纯 fn 任务,从必需参数名自动推断依赖。
|
||||
|
||||
返回 ``self`` 以支持链式调用,但推荐入口是 :meth:`from_specs`,
|
||||
它会整批校验(允许单次调用中的前向引用)。
|
||||
"""
|
||||
if spec.name in self.specs:
|
||||
raise DuplicateTaskError(spec.name)
|
||||
self.specs[spec.name] = spec
|
||||
self.deps[spec.name] = spec.depends_on
|
||||
# 为增量 API 即时检查重名与缺失依赖。
|
||||
self._validate_references()
|
||||
return self
|
||||
|
||||
@classmethod
|
||||
def from_specs(cls, specs: Iterable[TaskSpec[Any] | str]) -> Graph:
|
||||
"""从可迭代的 task spec 构建图.
|
||||
|
||||
先收集所有 spec,再统一校验。这意味着任务可以引用*后出现*的
|
||||
依赖——顺序无关,就像声明式配置文件的读取方式。
|
||||
|
||||
支持字符串引用,允许引用其他命令图中的任务。
|
||||
字符串引用将在CliRunner中解析展开。
|
||||
仅匹配 ``all_names`` 中任务名的必需参数(无默认值、非 Context 标注、
|
||||
非 ``*args``/``**kwargs``)被加入 ``depends_on``。显式声明
|
||||
``depends_on`` 的任务、cmd 任务、无 fn 的任务不受影响。
|
||||
已在 ``soft_depends_on`` 中声明的参数名不会被重复加入 ``depends_on``。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
specs : Iterable[TaskSpec[Any] | str]
|
||||
TaskSpec对象或字符串引用的列表
|
||||
spec:
|
||||
待推断的 TaskSpec。
|
||||
all_names:
|
||||
当前可见的任务名集合(``from_specs`` 时为全部任务名,
|
||||
``add`` 时为图中已有任务名)。
|
||||
"""
|
||||
if spec.depends_on or spec.cmd is not None or spec.fn is None:
|
||||
return spec
|
||||
try:
|
||||
sig = inspect.signature(spec.fn)
|
||||
except (TypeError, ValueError):
|
||||
return spec
|
||||
soft = set(spec.soft_depends_on)
|
||||
inferred: list[str] = []
|
||||
for pname, param in sig.parameters.items():
|
||||
if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD):
|
||||
continue
|
||||
if param.default is not inspect.Parameter.empty:
|
||||
continue
|
||||
if is_context_annotation(param.annotation):
|
||||
continue
|
||||
if pname in soft:
|
||||
continue
|
||||
if pname in all_names and pname != spec.name:
|
||||
inferred.append(pname)
|
||||
if inferred:
|
||||
return replace(spec, depends_on=tuple(inferred))
|
||||
return spec
|
||||
|
||||
Returns
|
||||
-------
|
||||
Graph
|
||||
构建完成的图
|
||||
def add(self, spec: TaskSpec[Any]) -> Graph:
|
||||
"""注册一个任务 spec,并即时校验。返回 ``self`` 支持链式调用。
|
||||
|
||||
Note
|
||||
-----
|
||||
字符串引用格式:
|
||||
- "command_name" - 引用整个命令图
|
||||
- "command_name.task_name" - 引用特定任务
|
||||
对 ``depends_on`` 为空的纯 fn 任务,自动从必需参数名推断依赖
|
||||
(匹配图中已有任务名的参数被加入 ``depends_on``)。
|
||||
"""
|
||||
spec = self._auto_infer_deps_single(spec, set(self.specs))
|
||||
self._register(spec)
|
||||
self._validate_references()
|
||||
return self
|
||||
|
||||
def chain(self, *specs: TaskSpec[Any]) -> Graph:
|
||||
"""链式注册任务:每个 spec 自动依赖前一个。
|
||||
|
||||
``chain(a, b, c)`` 等价于 ``b`` 依赖 ``a``,``c`` 依赖 ``b``。
|
||||
若 spec 已带 ``depends_on``,则前驱名追加到现有依赖前。
|
||||
返回 ``self`` 支持链式调用。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> graph = Graph.from_specs([
|
||||
... TaskSpec("build", cmd=["uv", "build"]),
|
||||
... "test", # 引用test命令图
|
||||
... ])
|
||||
>>> graph = px.Graph().chain(extract, transform, load)
|
||||
"""
|
||||
graph = cls()
|
||||
pending_refs: list[str] = []
|
||||
prev_name: str | None = None
|
||||
for s in specs:
|
||||
current = s
|
||||
if prev_name is not None:
|
||||
# 将前驱追加到 depends_on 最前(保持显式依赖优先)
|
||||
new_deps = (prev_name, *s.depends_on) if prev_name not in s.depends_on else s.depends_on
|
||||
current = replace(s, depends_on=new_deps)
|
||||
self.add(current)
|
||||
prev_name = current.name
|
||||
return self
|
||||
|
||||
def pipeline(self, *specs: TaskSpec[Any]) -> Graph:
|
||||
"""创建数据流水线:每个任务的结果通过上下文注入传给下一个。
|
||||
|
||||
与 :meth:`chain` 行为一致(依赖关系相同),但语义上强调数据流:
|
||||
前驱任务的返回值会自动注入后继任务的同名参数。
|
||||
|
||||
后继任务的 ``fn`` 参数名需匹配前驱任务名才能接收数据。
|
||||
``outputs`` 字段声明的命名输出可通过
|
||||
:meth:`RunReport.output_of` 查询。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> graph = px.Graph().pipeline(extract_spec, transform_spec, load_spec)
|
||||
>>> report = px.run(graph)
|
||||
>>> report.output_of("extract", "items")
|
||||
"""
|
||||
return self.chain(*specs)
|
||||
|
||||
def group(self, name: str, tasks: Sequence[str]) -> Graph:
|
||||
"""声明任务分组,引用组名等价于依赖组内所有任务。
|
||||
|
||||
组名必须不与任何已注册任务名冲突。组内任务必须已注册(或在本批
|
||||
``from_specs`` 收集阶段先于引用方注册)。声明后,后续 ``add`` 或
|
||||
``from_specs`` 中 ``depends_on``/``soft_depends_on`` 引用组名时,
|
||||
会被重写为依赖组内全部任务。
|
||||
|
||||
参数
|
||||
----
|
||||
name:
|
||||
组名。不能与 ``self.specs`` 中任务名重复,也不能与已有组名重复。
|
||||
tasks:
|
||||
组内任务名序列。所有任务必须已在图中注册。
|
||||
|
||||
返回 ``self`` 支持链式调用。
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
组名与任务名冲突、组名已存在、或组内任务未注册时。
|
||||
"""
|
||||
if not name:
|
||||
raise ValueError("group.name 必须为非空字符串。")
|
||||
if name in self.specs:
|
||||
raise ValueError(f"组名 {name!r} 与已注册任务名冲突。")
|
||||
if name in self._groups:
|
||||
raise ValueError(f"组名 {name!r} 已存在。")
|
||||
missing = [t for t in tasks if t not in self.specs]
|
||||
if missing:
|
||||
raise ValueError(f"组 {name!r} 引用了未注册的任务: {missing}")
|
||||
if not tasks:
|
||||
raise ValueError(f"组 {name!r} 不能为空。")
|
||||
self._groups[name] = tuple(tasks)
|
||||
return self
|
||||
|
||||
def _register(self, spec: TaskSpec[Any]) -> None:
|
||||
"""注册单个 spec。若 ``spec.loop`` 非 ``None``,展开为多实例。"""
|
||||
if spec.loop is not None:
|
||||
for expanded in self._expand_loop_to_specs(spec):
|
||||
self._register_single(expanded)
|
||||
return
|
||||
self._register_single(self._rewrite_deps(spec))
|
||||
|
||||
def _register_single(self, spec: TaskSpec[Any]) -> None:
|
||||
"""注册单个已展开的 spec(无 loop)。"""
|
||||
if spec.name in self.specs:
|
||||
raise DuplicateTaskError(spec.name)
|
||||
self.specs[spec.name] = spec
|
||||
# 拓扑依赖仅含硬依赖;软依赖仅用于注入,不影响分层。
|
||||
self.deps[spec.name] = spec.depends_on
|
||||
self._resolved_cache.clear()
|
||||
self._layers_cache = None
|
||||
|
||||
def _expand_loop_to_specs(self, spec: TaskSpec[Any]) -> list[TaskSpec[Any]]:
|
||||
"""展开 ``spec.loop`` 为多实例列表,并记录到 ``_loop_groups``。
|
||||
|
||||
返回展开后的 spec 列表(``loop`` 字段已清空,``args`` 已追加 item)。
|
||||
不写入 ``specs``/``deps``,由调用方注册。
|
||||
"""
|
||||
loop = spec.loop
|
||||
assert loop is not None # 由 _register 保证
|
||||
expanded: list[TaskSpec[Any]] = []
|
||||
expanded_names: list[str] = []
|
||||
for i, item in enumerate(loop.items):
|
||||
key = loop.key_fn(i, item) if loop.key_fn is not None else f"{spec.name}_{i}"
|
||||
new_spec = replace(spec, name=key, args=(*spec.args, item), loop=None)
|
||||
new_spec = self._rewrite_deps(new_spec)
|
||||
expanded.append(new_spec)
|
||||
expanded_names.append(key)
|
||||
self._loop_groups[spec.name] = expanded_names
|
||||
return expanded
|
||||
|
||||
def _rewrite_deps(self, spec: TaskSpec[Any]) -> TaskSpec[Any]:
|
||||
"""重写 spec 的依赖:引用 loop 组名或 group 组名时展开为所有实例。"""
|
||||
if not self._loop_groups and not self._groups:
|
||||
return spec
|
||||
# 合并 loop 组与普通组:组名 → 展开后的任务名列表
|
||||
combined: dict[str, Sequence[str]] = {}
|
||||
combined.update(self._loop_groups)
|
||||
combined.update(self._groups)
|
||||
new_hard: list[str] = []
|
||||
new_soft: list[str] = []
|
||||
changed = False
|
||||
for dep in spec.depends_on:
|
||||
if dep in combined:
|
||||
new_hard.extend(combined[dep])
|
||||
changed = True
|
||||
else:
|
||||
new_hard.append(dep)
|
||||
for dep in spec.soft_depends_on:
|
||||
if dep in combined:
|
||||
new_soft.extend(combined[dep])
|
||||
changed = True
|
||||
else:
|
||||
new_soft.append(dep)
|
||||
if not changed:
|
||||
return spec
|
||||
return replace(spec, depends_on=tuple(new_hard), soft_depends_on=tuple(new_soft))
|
||||
|
||||
@classmethod
|
||||
def from_specs(
|
||||
cls,
|
||||
specs: Iterable[TaskSpec[Any] | str],
|
||||
defaults: GraphDefaults | None = None,
|
||||
*,
|
||||
namespace: str | None = None,
|
||||
) -> Graph:
|
||||
"""从可迭代的 task spec 构建图。
|
||||
|
||||
先收集所有 spec,再统一校验。允许前向引用。支持字符串引用,
|
||||
由 :func:`compose` 或 :class:`GraphComposer` 解析展开。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
specs:
|
||||
TaskSpec 对象或字符串引用的列表。
|
||||
defaults:
|
||||
图级默认值。``None`` 使用空 :class:`GraphDefaults`。
|
||||
namespace:
|
||||
可选命名空间,用于 :meth:`add_subgraph` 合并时加前缀。
|
||||
"""
|
||||
graph = cls(defaults=defaults or GraphDefaults(), namespace=namespace)
|
||||
pending_refs: list[str] = []
|
||||
collected: list[TaskSpec[Any]] = []
|
||||
for spec in specs:
|
||||
if isinstance(spec, str):
|
||||
# 字符串引用,稍后解析
|
||||
pending_refs.append(spec)
|
||||
elif isinstance(spec, TaskSpec):
|
||||
if spec.name in graph.specs:
|
||||
raise DuplicateTaskError(spec.name)
|
||||
graph.specs[spec.name] = spec
|
||||
graph.deps[spec.name] = spec.depends_on
|
||||
collected.append(spec)
|
||||
else:
|
||||
raise TypeError(f"from_specs只接受TaskSpec或str,收到: {type(spec)}")
|
||||
raise TypeError(f"from_specs 只接受 TaskSpec 或 str,收到: {type(spec)}")
|
||||
|
||||
# 两阶段:先展开所有 loop(记录 _loop_groups),再统一重写依赖并批量注册。
|
||||
# 批量注册跳过每次 _register 的 cache 清空(N 次清空降为 0 次);
|
||||
# cache 失效由后续 validate()/layers() 首次调用自然触发。
|
||||
expanded: list[TaskSpec[Any]] = []
|
||||
for spec in collected:
|
||||
if spec.loop is not None:
|
||||
expanded.extend(graph._expand_loop_to_specs(spec))
|
||||
else:
|
||||
expanded.append(graph._rewrite_deps(spec))
|
||||
|
||||
# 自动推断 depends_on:对 depends_on 为空的纯 fn 任务,
|
||||
# 从必需参数名匹配图中任务名,消除显式声明 depends_on 的样板代码。
|
||||
all_names = {spec.name for spec in expanded}
|
||||
expanded = [graph._auto_infer_deps_single(spec, all_names) for spec in expanded]
|
||||
|
||||
for spec in expanded:
|
||||
if spec.name in graph.specs:
|
||||
raise DuplicateTaskError(spec.name)
|
||||
graph.specs[spec.name] = spec
|
||||
graph.deps[spec.name] = spec.depends_on
|
||||
|
||||
# 存储待解析的引用
|
||||
if pending_refs:
|
||||
# 使用特殊属性存储引用,稍后在CliRunner中解析
|
||||
# 由于Graph是frozen dataclass,我们需要特殊处理
|
||||
object.__setattr__(graph, "_pending_refs", pending_refs)
|
||||
graph._pending_refs = pending_refs
|
||||
|
||||
graph._validate_references()
|
||||
graph.validate()
|
||||
return graph
|
||||
|
||||
@classmethod
|
||||
def from_yaml(cls, path: str | Path) -> Graph:
|
||||
"""从 YAML 文件加载任务图(GitHub Actions 风格)。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path:
|
||||
YAML 文件路径。
|
||||
|
||||
Returns
|
||||
-------
|
||||
Graph
|
||||
解析后的任务图,支持 ``jobs``/``needs``/``strategy.matrix``/
|
||||
``if``/``continue-on-error``/``env``/``defaults`` 等字段。
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
YAML 结构不符合 schema 时。
|
||||
"""
|
||||
from .yaml_loader import load_yaml
|
||||
|
||||
return load_yaml(path)
|
||||
|
||||
def add_subgraph(self, sub: Graph, *, namespace: str | None = None) -> Graph:
|
||||
"""将子图合并到当前图,任务名加命名空间前缀避免冲突。
|
||||
|
||||
参数
|
||||
----
|
||||
sub:
|
||||
待合并的子图。
|
||||
namespace:
|
||||
命名空间前缀。``None`` 时使用 ``sub.namespace``,若子图也无命名空间
|
||||
则抛出 ``ValueError``。最终任务名为 ``f"{ns}:{original_name}"``。
|
||||
|
||||
合并后,子图内任务的依赖名也会被加前缀;与子图外部任务的依赖保持原样。
|
||||
|
||||
返回 ``self`` 支持链式调用。
|
||||
"""
|
||||
ns = namespace or sub.namespace
|
||||
if not ns:
|
||||
raise ValueError("add_subgraph 需要 namespace 或子图自带 namespace")
|
||||
|
||||
def _rename(name: str) -> str:
|
||||
# 仅对子图内部任务名加前缀;外部依赖保持原样
|
||||
return f"{ns}:{name}" if name in sub.specs else name
|
||||
|
||||
sub_names = set(sub.specs.keys())
|
||||
for spec in sub.specs.values():
|
||||
# 子图内部依赖名需加前缀,对应的 fn 参数也需包装
|
||||
internal_deps = (set(spec.depends_on) | set(spec.soft_depends_on)) & sub_names
|
||||
new_fn = _make_namespaced_fn(spec.fn, ns, internal_deps) if spec.fn else spec.fn
|
||||
new_spec = replace(
|
||||
spec,
|
||||
name=_rename(spec.name),
|
||||
fn=new_fn,
|
||||
depends_on=tuple(_rename(d) for d in spec.depends_on),
|
||||
soft_depends_on=tuple(_rename(d) for d in spec.soft_depends_on),
|
||||
)
|
||||
self._register(new_spec)
|
||||
self._validate_references()
|
||||
self.validate()
|
||||
return self
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 校验
|
||||
# ------------------------------------------------------------------ #
|
||||
def _validate_references(self) -> None:
|
||||
"""确保每个依赖名都存在于图中。"""
|
||||
for name, deps in self.deps.items():
|
||||
for dep in deps:
|
||||
"""确保每个依赖名都存在于图中。硬依赖与软依赖都校验。"""
|
||||
for name, spec in self.specs.items():
|
||||
for dep in spec.depends_on:
|
||||
if dep not in self.specs:
|
||||
raise MissingDependencyError(name, dep)
|
||||
for dep in spec.soft_depends_on:
|
||||
if dep not in self.specs:
|
||||
raise MissingDependencyError(name, dep)
|
||||
|
||||
def validate(self) -> None:
|
||||
"""执行完整 DAG 校验。
|
||||
"""执行完整 DAG 校验。存在环时抛出 :class:`CycleError`。
|
||||
|
||||
存在环时抛出 :class:`~pyflowx.errors.CycleError`。
|
||||
依赖存在性由 :meth:`_validate_references` 检查。
|
||||
顺带填充 :attr:`_layers_cache`(无环时),使后续 :meth:`layers`
|
||||
直接命中缓存,避免 :func:`_topological_layers` 二次计算。
|
||||
"""
|
||||
self._validate_references()
|
||||
sorter = _TopologicalSorter(self.deps)
|
||||
try:
|
||||
# prepare() 在有环时抛出 CycleError;此处不需要
|
||||
# static_order() 的结果,仅利用其校验副作用。
|
||||
sorter.prepare()
|
||||
except graphlib.CycleError as exc:
|
||||
# exc.args[1] 是构成环的节点列表。
|
||||
cycle: Sequence[str] = exc.args[1] if len(exc.args) > 1 else []
|
||||
raise CycleError(list(cycle)) from exc
|
||||
layers, cycle_nodes = _topological_layers(self.deps)
|
||||
if cycle_nodes is not None:
|
||||
raise CycleError(cycle_nodes)
|
||||
self._layers_cache = layers
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 内省
|
||||
@@ -153,10 +560,54 @@ class Graph:
|
||||
"""返回 ``name`` 的 spec;不存在则 ``KeyError``。"""
|
||||
return self.specs[name]
|
||||
|
||||
def resolved_spec(self, name: str) -> TaskSpec[Any]:
|
||||
"""返回应用图级默认值后的 spec(不修改原图)。
|
||||
|
||||
对于 ``retry``/``timeout``/``strategy``/``env``/``cwd`` 等可空
|
||||
字段,若 spec 字段为默认空值且图级默认值非空,则用
|
||||
:func:`dataclasses.replace` 生成带默认值的副本。
|
||||
|
||||
结果按 ``name`` 缓存;specs / defaults 变更时缓存失效。
|
||||
"""
|
||||
cached = self._resolved_cache.get(name)
|
||||
if cached is not None:
|
||||
return cached
|
||||
spec = self.specs[name]
|
||||
d = self.defaults
|
||||
overrides: dict[str, Any] = {}
|
||||
if spec.retry == RetryPolicy() and d.retry is not None:
|
||||
overrides["retry"] = d.retry
|
||||
if spec.timeout is None and d.timeout is not None:
|
||||
overrides["timeout"] = d.timeout
|
||||
if spec.strategy is None and d.strategy is not None:
|
||||
overrides["strategy"] = d.strategy
|
||||
if spec.env is None and d.env is not None:
|
||||
overrides["env"] = d.env
|
||||
if spec.cwd is None and d.cwd is not None:
|
||||
overrides["cwd"] = d.cwd
|
||||
if spec.priority == 0 and d.priority != 0:
|
||||
overrides["priority"] = d.priority
|
||||
if not spec.continue_on_error and d.continue_on_error:
|
||||
overrides["continue_on_error"] = True
|
||||
if spec.concurrency_key is None and d.concurrency_key is not None:
|
||||
overrides["concurrency_key"] = d.concurrency_key
|
||||
if not spec.verbose and d.verbose:
|
||||
overrides["verbose"] = True
|
||||
if not spec.tags and d.tags:
|
||||
overrides["tags"] = d.tags
|
||||
resolved = spec if not overrides else replace(spec, **overrides)
|
||||
self._resolved_cache[name] = resolved
|
||||
return resolved
|
||||
|
||||
def dependencies(self, name: str) -> tuple[str, ...]:
|
||||
"""``name`` 的直接前驱。"""
|
||||
"""``name`` 的直接硬依赖前驱。"""
|
||||
return self.deps[name]
|
||||
|
||||
def all_deps(self, name: str) -> tuple[str, ...]:
|
||||
"""``name`` 的硬依赖 + 软依赖。"""
|
||||
spec = self.specs[name]
|
||||
return tuple(spec.depends_on) + tuple(spec.soft_depends_on)
|
||||
|
||||
def all_specs(self) -> Mapping[str, TaskSpec[Any]]:
|
||||
"""name -> spec 的只读视图。"""
|
||||
return self.specs
|
||||
@@ -164,57 +615,37 @@ class Graph:
|
||||
def layers(self) -> list[list[str]]:
|
||||
"""将任务分组为可并行执行的层(Kahn 算法)。
|
||||
|
||||
同层任务无相互依赖,可并发执行。层按执行顺序返回。
|
||||
同层任务无相互硬依赖,可并发执行。软依赖不参与分层。
|
||||
层按执行顺序返回。图有环时抛出 :class:`CycleError`。
|
||||
|
||||
图有环时抛出 :class:`~pyflowx.errors.CycleError`。
|
||||
结果按实例缓存;specs 变更时失效(:meth:`add` / :meth:`_register`)。
|
||||
|
||||
.. note::
|
||||
本方法假定图已通过 :meth:`validate` 校验(由 :func:`pyflowx.run`
|
||||
在入口统一执行一次)。若直接调用本方法,需自行先校验。
|
||||
"""
|
||||
self.validate()
|
||||
sorter = _TopologicalSorter(self.deps)
|
||||
result: list[list[str]] = []
|
||||
# ``get_ready`` + ``done`` 每次给出一层,正好是并行执行所需的分组。
|
||||
sorter.prepare()
|
||||
while sorter.is_active():
|
||||
ready = list(sorter.get_ready())
|
||||
# 排序以保证确定性、可复现的执行计划。
|
||||
ready.sort()
|
||||
result.append(ready)
|
||||
for node in ready:
|
||||
sorter.done(node)
|
||||
if self._layers_cache is not None:
|
||||
return self._layers_cache
|
||||
result, cycle_nodes = _topological_layers(self.deps)
|
||||
if cycle_nodes is not None:
|
||||
raise CycleError(cycle_nodes)
|
||||
self._layers_cache = result
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 子图 / 标签过滤
|
||||
# ------------------------------------------------------------------ #
|
||||
def subgraph(self, tags: Iterable[str]) -> Graph:
|
||||
"""返回仅包含匹配任意标签的任务的新图。
|
||||
|
||||
依赖会被修剪,仅保留被保留任务之间的边;指向被丢弃任务的边
|
||||
会被移除(被保留的任务不再等待它们)。用于调试时运行大型
|
||||
DAG 的切片。
|
||||
"""
|
||||
"""返回仅包含匹配任意标签的任务的新图。依赖边被修剪。"""
|
||||
wanted: set[str] = set(tags)
|
||||
kept: list[TaskSpec[Any]] = []
|
||||
for spec in self.specs.values():
|
||||
if wanted & set(spec.tags):
|
||||
pruned_deps = tuple(
|
||||
d for d in spec.depends_on if d in self.specs and (wanted & set(self.specs[d].tags))
|
||||
)
|
||||
kept.append(
|
||||
TaskSpec[Any](
|
||||
name=spec.name,
|
||||
fn=spec.fn,
|
||||
cmd=spec.cmd,
|
||||
depends_on=pruned_deps,
|
||||
args=spec.args,
|
||||
kwargs=spec.kwargs,
|
||||
retries=spec.retries,
|
||||
timeout=spec.timeout,
|
||||
tags=spec.tags,
|
||||
conditions=spec.conditions,
|
||||
cwd=spec.cwd,
|
||||
)
|
||||
)
|
||||
return Graph.from_specs(kept)
|
||||
|
||||
def _dep_kept(dep: str) -> bool:
|
||||
return dep in self.specs and bool(wanted & set(self.specs[dep].tags))
|
||||
|
||||
kept: list[TaskSpec[Any]] = [
|
||||
_prune_deps(spec, _dep_kept) for spec in self.specs.values() if wanted & set(spec.tags)
|
||||
]
|
||||
return Graph.from_specs(kept, defaults=self.defaults)
|
||||
|
||||
def subgraph_by_names(self, names: Iterable[str]) -> Graph:
|
||||
"""返回限定于 ``names`` 的新图(边已修剪)。"""
|
||||
@@ -222,36 +653,104 @@ class Graph:
|
||||
for n in wanted:
|
||||
if n not in self.specs:
|
||||
raise KeyError(f"Unknown task name: {n!r}")
|
||||
kept: list[TaskSpec[Any]] = []
|
||||
for spec in self.specs.values():
|
||||
if spec.name in wanted:
|
||||
pruned_deps = tuple(d for d in spec.depends_on if d in wanted)
|
||||
kept.append(
|
||||
TaskSpec[Any](
|
||||
name=spec.name,
|
||||
fn=spec.fn,
|
||||
cmd=spec.cmd,
|
||||
depends_on=pruned_deps,
|
||||
args=spec.args,
|
||||
kwargs=spec.kwargs,
|
||||
retries=spec.retries,
|
||||
timeout=spec.timeout,
|
||||
tags=spec.tags,
|
||||
conditions=spec.conditions,
|
||||
cwd=spec.cwd,
|
||||
)
|
||||
)
|
||||
return Graph.from_specs(kept)
|
||||
kept: list[TaskSpec[Any]] = [
|
||||
_prune_deps(spec, lambda d: d in wanted) for spec in self.specs.values() if spec.name in wanted
|
||||
]
|
||||
return Graph.from_specs(kept, defaults=self.defaults)
|
||||
|
||||
def subgraph_with_deps(self, names: Iterable[str]) -> Graph:
|
||||
"""返回包含 ``names`` 及其所有传递依赖的新图。
|
||||
|
||||
与 :meth:`subgraph_by_names` 不同,本方法会沿 ``depends_on`` 和
|
||||
``soft_depends_on`` 向上遍历,确保被选中的任务所需的上游全部包含在内,
|
||||
使子图可独立执行。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
names:
|
||||
需要执行的任务名。每个名称必须存在于当前图中。
|
||||
"""
|
||||
seeds: set[str] = set(names)
|
||||
for n in seeds:
|
||||
if n not in self.specs:
|
||||
raise KeyError(f"Unknown task name: {n!r}")
|
||||
# BFS 收集传递依赖(硬依赖 + 软依赖)
|
||||
closure: set[str] = set()
|
||||
queue: list[str] = list(seeds)
|
||||
while queue:
|
||||
name = queue.pop()
|
||||
if name in closure:
|
||||
continue
|
||||
closure.add(name)
|
||||
for dep in self.all_deps(name):
|
||||
if dep not in closure:
|
||||
queue.append(dep)
|
||||
kept: list[TaskSpec[Any]] = [
|
||||
_prune_deps(spec, lambda d: d in closure) for spec in self.specs.values() if spec.name in closure
|
||||
]
|
||||
return Graph.from_specs(kept, defaults=self.defaults)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Fan-out / map-reduce
|
||||
# ------------------------------------------------------------------ #
|
||||
def map(
|
||||
self,
|
||||
name_fn: Callable[[int], str],
|
||||
spec: TaskSpec[Any],
|
||||
items: Sequence[Any],
|
||||
arg_factory: Callable[[Any], tuple[Any, ...]] | None = None,
|
||||
depends_on_per: Callable[[int], tuple[str, ...]] | None = None,
|
||||
) -> list[TaskSpec[Any]]:
|
||||
"""为 ``items`` 中每个元素生成一个 TaskSpec 并加入图。
|
||||
|
||||
用于 fan-out / map-reduce 模式。返回生成的 spec 列表,便于
|
||||
后续 reduce 任务依赖。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name_fn:
|
||||
接受索引 ``i``,返回任务名。需保证唯一。
|
||||
spec:
|
||||
模板 spec。其 ``name`` 与 ``args`` 会被覆盖。
|
||||
items:
|
||||
待分发的数据序列。
|
||||
arg_factory:
|
||||
接受一个 item,返回位置参数元组,覆盖 spec.args。
|
||||
``None`` 则将单个 item 作为唯一位置参数。
|
||||
depends_on_per:
|
||||
接受索引 ``i``,返回该任务的额外硬依赖。``None`` 则继承 spec.depends_on。
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[TaskSpec]
|
||||
生成的 spec 列表(已加入图)。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> fetch_tmpl = px.TaskSpec("", fn=fetch_user)
|
||||
>>> specs = graph.map(lambda i: f"fetch_{i}", fetch_tmpl, [1, 2, 3])
|
||||
>>> reduce_spec = px.TaskSpec("reduce", fn=reduce_fn, depends_on=tuple(s.name for s in specs))
|
||||
"""
|
||||
generated: list[TaskSpec[Any]] = []
|
||||
for i, item in enumerate(items):
|
||||
name = name_fn(i)
|
||||
args = arg_factory(item) if arg_factory is not None else (item,)
|
||||
extra_deps = depends_on_per(i) if depends_on_per is not None else ()
|
||||
new_spec = replace(
|
||||
spec,
|
||||
name=name,
|
||||
args=tuple(args),
|
||||
depends_on=tuple(spec.depends_on) + tuple(extra_deps),
|
||||
)
|
||||
self.add(new_spec)
|
||||
generated.append(new_spec)
|
||||
return generated
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 可视化
|
||||
# ------------------------------------------------------------------ #
|
||||
def to_mermaid(self, orientation: str = "TD") -> str:
|
||||
"""将 DAG 渲染为 Mermaid ``graph`` 定义字符串。
|
||||
|
||||
无外部依赖;输出可粘贴到 Markdown、由 VS Code 的 Mermaid 预览
|
||||
渲染,或保存为文件。
|
||||
"""
|
||||
"""将 DAG 渲染为 Mermaid ``graph`` 定义字符串。"""
|
||||
valid = {"TD", "TB", "BT", "LR", "RL"}
|
||||
orientation = orientation.upper()
|
||||
if orientation not in valid:
|
||||
@@ -262,6 +761,10 @@ class Graph:
|
||||
for name, deps in self.deps.items():
|
||||
for dep in deps:
|
||||
lines.append(f" {dep} --> {name}")
|
||||
# 软依赖用虚线
|
||||
for name, spec in self.specs.items():
|
||||
for dep in spec.soft_depends_on:
|
||||
lines.append(f" {dep} -.-> {name}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
"""运行历史管理:存储、查询与比较多次运行的 :class:`RunReport`。
|
||||
|
||||
适用于:
|
||||
* 断点续跑:从最近一次失败运行恢复,跳过已成功任务。
|
||||
* 趋势分析:对比多次运行的任务耗时、状态变化。
|
||||
* 审计追溯:按 run_id 查询历史报告。
|
||||
|
||||
用法
|
||||
----
|
||||
history = RunHistory(".pyflowx/runs")
|
||||
history.save(report)
|
||||
latest = history.latest()
|
||||
if latest is not None and not latest.success:
|
||||
# 从最近失败处恢复
|
||||
px.run(graph, resume_from=latest)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["RunHistory"]
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from .report import RunReport
|
||||
|
||||
|
||||
class RunHistory:
|
||||
"""运行历史记录管理器。
|
||||
|
||||
将 :class:`RunReport` 以 JSON 文件存储到指定目录,文件名为
|
||||
``{run_id}.json``。支持按 run_id 加载、列出全部、查询最近一次、删除。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
dir:
|
||||
存储目录。不存在时自动创建。
|
||||
"""
|
||||
|
||||
def __init__(self, dir: str | Path) -> None:
|
||||
self.dir = Path(dir)
|
||||
self.dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def save(self, report: RunReport) -> Path:
|
||||
"""保存运行报告,返回文件路径。"""
|
||||
path = self.dir / f"{report.run_id}.json"
|
||||
path.write_text(report.to_json(), encoding="utf-8")
|
||||
return path
|
||||
|
||||
def load(self, run_id: str) -> RunReport:
|
||||
"""按 run_id 加载报告。
|
||||
|
||||
Raises
|
||||
------
|
||||
FileNotFoundError
|
||||
指定 run_id 的文件不存在。
|
||||
"""
|
||||
path = self.dir / f"{run_id}.json"
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"运行报告 {run_id!r} 不存在: {path}")
|
||||
return RunReport.from_json(path.read_text(encoding="utf-8"))
|
||||
|
||||
def list_runs(self) -> list[str]:
|
||||
"""列出全部 run_id(按文件名排序)。"""
|
||||
return sorted(p.stem for p in self.dir.glob("*.json"))
|
||||
|
||||
def latest(self) -> RunReport | None:
|
||||
"""返回最近修改的报告;目录为空时返回 ``None``。"""
|
||||
runs = sorted(self.dir.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
if not runs:
|
||||
return None
|
||||
return RunReport.from_json(runs[0].read_text(encoding="utf-8"))
|
||||
|
||||
def delete(self, run_id: str) -> bool:
|
||||
"""删除指定 run_id 的报告。返回是否实际删除。"""
|
||||
path = self.dir / f"{run_id}.json"
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
return True
|
||||
return False
|
||||
|
||||
def __len__(self) -> int:
|
||||
return sum(1 for _ in self.dir.glob("*.json"))
|
||||
|
||||
def __contains__(self, run_id: object) -> bool:
|
||||
if not isinstance(run_id, str):
|
||||
return False
|
||||
return (self.dir / f"{run_id}.json").exists()
|
||||
@@ -0,0 +1,130 @@
|
||||
"""图片处理流水线 DAG 构造器.
|
||||
|
||||
将 :mod:`pyflowx.ops.files.imagetool` 的操作封装为链式 :class:`Graph`,
|
||||
便于在 DAG 中组合多步图片处理 (如 resize → watermark → convert).
|
||||
|
||||
设计参照 :mod:`pyflowx.compose` 的函数式构造器模式.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["image_pipeline"]
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .graph import Graph
|
||||
from .task import TaskSpec
|
||||
|
||||
|
||||
def image_pipeline(
|
||||
source: str | Path,
|
||||
steps: list[tuple[str, dict[str, Any]]],
|
||||
*,
|
||||
output_dir: str | Path | None = None,
|
||||
naming: str = "{stem}_{step}{ext}",
|
||||
) -> Graph:
|
||||
"""图片处理流水线 DAG 构造器.
|
||||
|
||||
每个 step 是 ``(操作名, 参数字典)``, 操作名对应 imagetool 子命令
|
||||
(resize/crop/rotate/flip/convert/watermark/compress).
|
||||
前一步输出作为后一步输入, 形成链式 DAG.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
source : str | Path
|
||||
源图片路径
|
||||
steps : list[tuple[str, dict[str, Any]]]
|
||||
操作步骤列表, 如 ``[("resize", {"width": 800}), ("watermark", {"text": "X"})]``
|
||||
output_dir : str | Path | None
|
||||
输出目录 (None 时用源文件所在目录)
|
||||
naming : str
|
||||
输出文件名模板, 可用占位符 ``{stem}`` (累积 stem) / ``{step}`` (操作名) /
|
||||
``{ext}`` (当前扩展名, convert 步骤可能改变). 默认 ``"{stem}_{step}{ext}"``
|
||||
|
||||
Returns
|
||||
-------
|
||||
Graph
|
||||
链式 DAG, 每步为一个 :class:`TaskSpec`, 通过 ``depends_on`` 链接
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
操作名不在支持列表中时
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> graph = px.image_pipeline(
|
||||
... "input.jpg",
|
||||
... steps=[
|
||||
... ("resize", {"width": 800}),
|
||||
... ("watermark", {"text": "© 2026"}),
|
||||
... ("convert", {"format": "webp", "quality": 85}),
|
||||
... ],
|
||||
... )
|
||||
>>> report = px.run(graph, strategy="sequential")
|
||||
"""
|
||||
from .ops.files import imagetool
|
||||
|
||||
operations = {
|
||||
"resize": imagetool.image_resize,
|
||||
"crop": imagetool.image_crop,
|
||||
"rotate": imagetool.image_rotate,
|
||||
"flip": imagetool.image_flip,
|
||||
"convert": imagetool.image_convert,
|
||||
"watermark": imagetool.image_watermark,
|
||||
"compress": imagetool.image_compress,
|
||||
}
|
||||
|
||||
source_path = Path(source)
|
||||
out_dir = Path(output_dir) if output_dir else source_path.parent
|
||||
|
||||
specs: list[TaskSpec[Any]] = []
|
||||
prev_name: str | None = None
|
||||
prev_output: Path | None = None
|
||||
current_stem = source_path.stem
|
||||
current_ext = source_path.suffix
|
||||
|
||||
for i, (op_name, params) in enumerate(steps):
|
||||
if op_name not in operations:
|
||||
raise ValueError(f"未知图片操作: {op_name!r}, 支持: {sorted(operations)}")
|
||||
|
||||
if op_name == "convert" and "format" in params:
|
||||
current_ext = f".{params['format'].lower()}"
|
||||
|
||||
output_name = naming.format(stem=current_stem, step=op_name, ext=current_ext)
|
||||
output_path = out_dir / output_name
|
||||
input_path = source_path if prev_output is None else prev_output
|
||||
|
||||
task_name = f"step_{i:02d}_{op_name}"
|
||||
base_fn = operations[op_name]
|
||||
fn = _make_step_fn(base_fn, input_path, output_path, params)
|
||||
fn.__name__ = task_name
|
||||
|
||||
depends_on = (prev_name,) if prev_name else ()
|
||||
spec = TaskSpec(task_name, fn=fn, depends_on=depends_on)
|
||||
specs.append(spec)
|
||||
|
||||
prev_name = task_name
|
||||
prev_output = output_path
|
||||
current_stem = output_path.stem
|
||||
|
||||
return Graph.from_specs(specs)
|
||||
|
||||
|
||||
def _make_step_fn(
|
||||
base_fn: Any,
|
||||
input_path: Path,
|
||||
output_path: Path,
|
||||
params: dict[str, Any],
|
||||
) -> Any:
|
||||
"""创建步骤执行闭包, 绑定输入/输出路径与参数.
|
||||
|
||||
闭包捕获的是 :func:`_make_step_fn` 的参数 (每次调用独立作用域),
|
||||
不会受外层循环变量重绑定影响.
|
||||
"""
|
||||
|
||||
def _run() -> None:
|
||||
base_fn(input_path, output_path, **params)
|
||||
|
||||
return _run
|
||||
@@ -0,0 +1,217 @@
|
||||
"""监控导出:Prometheus 指标收集与健康检查。
|
||||
|
||||
提供零依赖的轻量监控方案:
|
||||
|
||||
* :class:`MetricsCollector` —— 通过 ``on_event`` 回调收集任务指标,
|
||||
导出 Prometheus 文本格式。可配合 ``http.server`` 或 ``prometheus_client``
|
||||
暴露指标端点。
|
||||
* :func:`health_check` —— 分析 :class:`RunReport` 返回结构化健康状态。
|
||||
|
||||
用法
|
||||
----
|
||||
collector = px.MetricsCollector()
|
||||
report = px.run(graph, on_event=collector.on_event)
|
||||
print(collector.metrics_text())
|
||||
print(px.health_check(report))
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["MetricsCollector", "health_check", "start_metrics_server"]
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from .report import RunReport
|
||||
from .task import TaskEvent, TaskStatus
|
||||
|
||||
|
||||
class MetricsCollector:
|
||||
"""收集任务执行指标,导出 Prometheus 文本格式。
|
||||
|
||||
作为 :func:`run` 的 ``on_event`` 回调使用,收集每个任务的生命周期
|
||||
事件并聚合为 Prometheus 指标。线程安全(回调可能从多个线程调用)。
|
||||
|
||||
指标列表
|
||||
--------
|
||||
- ``pyflowx_task_total{task, status}`` — 任务计数器
|
||||
- ``pyflowx_task_duration_seconds{task}`` — 任务耗时(gauge,最近值)
|
||||
- ``pyflowx_task_duration_seconds_sum{task}`` — 任务累计耗时
|
||||
- ``pyflowx_task_retries_total{task}`` — 重试次数
|
||||
- ``pyflowx_run_total{status}`` — 运行计数器
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._task_count: dict[str, dict[str, int]] = {}
|
||||
self._task_duration_last: dict[str, float] = {}
|
||||
self._task_duration_sum: dict[str, float] = {}
|
||||
self._task_retries: dict[str, int] = {}
|
||||
self._lock_counts: dict[str, int] = {}
|
||||
|
||||
def on_event(self, event: TaskEvent) -> None:
|
||||
"""``on_event`` 回调:记录任务生命周期事件。"""
|
||||
task = event.task
|
||||
status = event.status.value
|
||||
|
||||
# 任务计数
|
||||
self._task_count.setdefault(task, {})
|
||||
self._task_count[task][status] = self._task_count[task].get(status, 0) + 1
|
||||
|
||||
# 耗时(仅在终态时记录)
|
||||
if event.duration is not None and event.status in (TaskStatus.SUCCESS, TaskStatus.FAILED):
|
||||
self._task_duration_last[task] = event.duration
|
||||
self._task_duration_sum[task] = self._task_duration_sum.get(task, 0.0) + event.duration
|
||||
|
||||
# 重试次数
|
||||
if event.attempts > 1 and event.status in (TaskStatus.SUCCESS, TaskStatus.FAILED):
|
||||
self._task_retries[task] = self._task_retries.get(task, 0) + event.attempts - 1
|
||||
|
||||
def record_run(self, report: RunReport) -> None:
|
||||
"""记录一次完整运行的结果。"""
|
||||
status = "success" if report.success else "failed"
|
||||
self._lock_counts[status] = self._lock_counts.get(status, 0) + 1
|
||||
|
||||
def metrics_text(self) -> str:
|
||||
"""导出 Prometheus 文本格式指标字符串。"""
|
||||
lines: list[str] = []
|
||||
|
||||
# pyflowx_task_total
|
||||
if self._task_count:
|
||||
lines.append("# HELP pyflowx_task_total Total tasks by status.")
|
||||
lines.append("# TYPE pyflowx_task_total counter")
|
||||
for task in sorted(self._task_count):
|
||||
for status in sorted(self._task_count[task]):
|
||||
count = self._task_count[task][status]
|
||||
lines.append(f'pyflowx_task_total{{task="{task}",status="{status}"}} {count}')
|
||||
|
||||
# pyflowx_task_duration_seconds (last)
|
||||
if self._task_duration_last:
|
||||
lines.append("# HELP pyflowx_task_duration_seconds Last task duration in seconds.")
|
||||
lines.append("# TYPE pyflowx_task_duration_seconds gauge")
|
||||
for task in sorted(self._task_duration_last):
|
||||
lines.append(f'pyflowx_task_duration_seconds{{task="{task}"}} {self._task_duration_last[task]}')
|
||||
|
||||
# pyflowx_task_duration_seconds_sum
|
||||
if self._task_duration_sum:
|
||||
lines.append("# HELP pyflowx_task_duration_seconds_sum Total task duration in seconds.")
|
||||
lines.append("# TYPE pyflowx_task_duration_seconds_sum counter")
|
||||
for task in sorted(self._task_duration_sum):
|
||||
lines.append(f'pyflowx_task_duration_seconds_sum{{task="{task}"}} {self._task_duration_sum[task]}')
|
||||
|
||||
# pyflowx_task_retries_total
|
||||
if self._task_retries:
|
||||
lines.append("# HELP pyflowx_task_retries_total Total task retries.")
|
||||
lines.append("# TYPE pyflowx_task_retries_total counter")
|
||||
for task in sorted(self._task_retries):
|
||||
lines.append(f'pyflowx_task_retries_total{{task="{task}"}} {self._task_retries[task]}')
|
||||
|
||||
# pyflowx_run_total
|
||||
if self._lock_counts:
|
||||
lines.append("# HELP pyflowx_run_total Total runs by status.")
|
||||
lines.append("# TYPE pyflowx_run_total counter")
|
||||
for status in sorted(self._lock_counts):
|
||||
lines.append(f'pyflowx_run_total{{status="{status}"}} {self._lock_counts[status]}')
|
||||
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
def reset(self) -> None:
|
||||
"""清空所有收集的指标。"""
|
||||
self._task_count.clear()
|
||||
self._task_duration_last.clear()
|
||||
self._task_duration_sum.clear()
|
||||
self._task_retries.clear()
|
||||
self._lock_counts.clear()
|
||||
|
||||
|
||||
def health_check(report: RunReport) -> dict[str, Any]:
|
||||
"""分析运行报告,返回结构化健康状态。
|
||||
|
||||
状态判定:
|
||||
- ``healthy`` — 全部任务 SUCCESS
|
||||
- ``degraded`` — 部分(非全部)任务 FAILED
|
||||
- ``unhealthy`` — 全部任务 FAILED
|
||||
- ``unknown`` — 无任务
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict
|
||||
包含 ``status``/``total``/``success``/``failed``/``skipped``
|
||||
/``duration`` 字段。
|
||||
"""
|
||||
total = len(report.results)
|
||||
if total == 0:
|
||||
return {"status": "unknown", "total": 0, "message": "无任务结果"}
|
||||
|
||||
counts: dict[str, int] = {}
|
||||
total_duration = 0.0
|
||||
for r in report.results.values():
|
||||
status = r.status.value
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
if r.duration is not None:
|
||||
total_duration += r.duration
|
||||
|
||||
success = counts.get("success", 0)
|
||||
failed = counts.get("failed", 0)
|
||||
skipped = counts.get("skipped", 0)
|
||||
|
||||
if failed == 0:
|
||||
status = "healthy"
|
||||
elif failed < total:
|
||||
status = "degraded"
|
||||
else:
|
||||
status = "unhealthy"
|
||||
|
||||
return {
|
||||
"status": status,
|
||||
"total": total,
|
||||
"success": success,
|
||||
"failed": failed,
|
||||
"skipped": skipped,
|
||||
"duration": round(total_duration, 3),
|
||||
}
|
||||
|
||||
|
||||
def start_metrics_server(collector: MetricsCollector, port: int = 9100, host: str = "0.0.0.0") -> Callable[[], None]:
|
||||
"""启动简单 HTTP 服务器暴露 Prometheus 指标。
|
||||
|
||||
返回停止函数,调用后关闭服务器。阻塞调用方线程,建议在后台线程运行。
|
||||
|
||||
.. note::
|
||||
使用标准库 ``http.server``,零依赖。生产环境建议配合
|
||||
``prometheus_client`` 或反向代理使用。
|
||||
"""
|
||||
import sys
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override
|
||||
else:
|
||||
from typing_extensions import override # pragma: no cover
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
if self.path == "/metrics":
|
||||
body = collector.metrics_text().encode("utf-8")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
else:
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
|
||||
@override
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
pass # 静默访问日志
|
||||
|
||||
server = HTTPServer((host, port), _Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
def stop() -> None:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
return stop
|
||||
@@ -0,0 +1,320 @@
|
||||
"""任务通知系统。
|
||||
|
||||
在任务生命周期事件(RUNNING/SUCCESS/FAILED/SKIPPED)发生时发送通知到
|
||||
外部系统。支持两类 sink:
|
||||
|
||||
* :class:`CallbackNotifier` —— 包装 Python 回调,零外部依赖
|
||||
* :class:`WebhookNotifier` 及其子类 —— 通过 HTTP POST 发送到飞书/钉钉/企业微信
|
||||
|
||||
通知系统与 :class:`pyflowx.progress.ProgressCallback` 独立且可共存。
|
||||
``notifiers`` 参数在 :func:`pyflowx.run` 中传入,通知器的 ``notify`` 方法
|
||||
被加入回调链,``close`` 在执行结束时调用。
|
||||
|
||||
设计要点
|
||||
--------
|
||||
* 事件过滤在 Notifier 内部,通过 ``levels`` 参数控制哪些事件触发通知。
|
||||
* webhook 使用 stdlib :mod:`urllib.request`,零新依赖。
|
||||
* 通知发送失败仅记录日志,不影响任务执行。
|
||||
|
||||
快速上手
|
||||
--------
|
||||
import pyflowx as px
|
||||
|
||||
# 1. 回调通知
|
||||
notifier = px.CallbackNotifier(
|
||||
lambda e: print(f"[通知] {e.task}: {e.status.value}")
|
||||
)
|
||||
px.run(graph, notifiers=notifier)
|
||||
|
||||
# 2. 飞书 webhook(仅失败时通知)
|
||||
feishu = px.FeishuNotifier(
|
||||
url="https://open.feishu.cn/open-apis/bot/v2/hook/xxx",
|
||||
levels=(px.NotificationLevel.FAILED,),
|
||||
)
|
||||
px.run(graph, notifiers=[feishu])
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = [
|
||||
"ALL_LEVELS",
|
||||
"CallbackNotifier",
|
||||
"DingTalkNotifier",
|
||||
"DiscordNotifier",
|
||||
"FeishuNotifier",
|
||||
"NotificationLevel",
|
||||
"Notifier",
|
||||
"SlackNotifier",
|
||||
"TelegramNotifier",
|
||||
"WeChatNotifier",
|
||||
"WebhookNotifier",
|
||||
]
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections.abc import Callable, Iterable
|
||||
from enum import Enum
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override
|
||||
else:
|
||||
from typing_extensions import override # pragma: no cover
|
||||
|
||||
from ._json import dumps
|
||||
from .task import TaskEvent, TaskStatus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NotificationLevel(Enum):
|
||||
"""通知触发的事件级别。"""
|
||||
|
||||
RUNNING = "running"
|
||||
SUCCESS = "success"
|
||||
FAILED = "failed"
|
||||
SKIPPED = "skipped"
|
||||
|
||||
|
||||
_LEVEL_STATUS_MAP: dict[NotificationLevel, TaskStatus] = {
|
||||
NotificationLevel.RUNNING: TaskStatus.RUNNING,
|
||||
NotificationLevel.SUCCESS: TaskStatus.SUCCESS,
|
||||
NotificationLevel.FAILED: TaskStatus.FAILED,
|
||||
NotificationLevel.SKIPPED: TaskStatus.SKIPPED,
|
||||
}
|
||||
|
||||
#: 所有事件级别的冻结集合。
|
||||
ALL_LEVELS: frozenset[NotificationLevel] = frozenset(_LEVEL_STATUS_MAP.keys())
|
||||
|
||||
#: 各平台消息前缀图标。
|
||||
_STATUS_ICONS: dict[TaskStatus, str] = {
|
||||
TaskStatus.RUNNING: "▶",
|
||||
TaskStatus.SUCCESS: "✓",
|
||||
TaskStatus.FAILED: "✗",
|
||||
TaskStatus.SKIPPED: "○",
|
||||
}
|
||||
|
||||
|
||||
def _status_to_level(status: TaskStatus) -> NotificationLevel | None:
|
||||
"""将 TaskStatus 映射为 NotificationLevel,无映射时返回 None。"""
|
||||
for level, st in _LEVEL_STATUS_MAP.items():
|
||||
if st == status:
|
||||
return level
|
||||
return None
|
||||
|
||||
|
||||
def _format_event_message(event: TaskEvent) -> str:
|
||||
"""格式化事件为人类可读消息文本(各平台共享)。"""
|
||||
icon = _STATUS_ICONS.get(event.status, "•")
|
||||
dur = f" ({event.duration:.3f}s)" if event.duration is not None else ""
|
||||
parts = [f"{icon} 任务 {event.task} {event.status.value}{dur}"]
|
||||
if event.error:
|
||||
parts.append(f" 错误: {event.error}")
|
||||
if event.reason:
|
||||
parts.append(f" 原因: {event.reason}")
|
||||
if event.attempts > 1:
|
||||
parts.append(f" 尝试次数: {event.attempts}")
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class Notifier(Protocol):
|
||||
"""通知器协议。
|
||||
|
||||
生命周期:执行期间多次 ``notify`` → 结束时 ``close``。
|
||||
实现者应确保 ``notify`` 不抛异常(或仅在不可恢复时抛),
|
||||
通知失败不应影响任务执行。
|
||||
"""
|
||||
|
||||
def notify(self, event: TaskEvent) -> None:
|
||||
"""处理单个任务事件。"""
|
||||
...
|
||||
|
||||
def close(self) -> None:
|
||||
"""释放资源(如关闭连接)。"""
|
||||
...
|
||||
|
||||
|
||||
class CallbackNotifier:
|
||||
"""包装 Python 回调为 :class:`Notifier`。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
callback:
|
||||
接收 :class:`TaskEvent` 的回调函数。
|
||||
levels:
|
||||
触发通知的事件级别集合。默认 :data:`ALL_LEVELS`(全生命周期)。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
callback: Callable[[TaskEvent], None],
|
||||
levels: Iterable[NotificationLevel] = ALL_LEVELS,
|
||||
) -> None:
|
||||
self._callback = callback
|
||||
self._levels = frozenset(levels)
|
||||
|
||||
def notify(self, event: TaskEvent) -> None:
|
||||
level = _status_to_level(event.status)
|
||||
if level is not None and level in self._levels:
|
||||
self._callback(event)
|
||||
|
||||
def close(self) -> None:
|
||||
"""无资源需释放。"""
|
||||
|
||||
|
||||
class WebhookNotifier:
|
||||
"""通过 HTTP POST 发送通知的基类。
|
||||
|
||||
使用 stdlib :mod:`urllib.request`,零外部依赖。子类覆盖
|
||||
:meth:`_build_payload` 以适配不同平台的消息格式。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
url:
|
||||
webhook 接收地址。
|
||||
levels:
|
||||
触发通知的事件级别集合。默认 :data:`ALL_LEVELS`。
|
||||
timeout:
|
||||
HTTP 请求超时秒数。默认 ``10``。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str,
|
||||
levels: Iterable[NotificationLevel] = ALL_LEVELS,
|
||||
timeout: float = 10.0,
|
||||
) -> None:
|
||||
self._url = url
|
||||
self._levels = frozenset(levels)
|
||||
self._timeout = timeout
|
||||
|
||||
def notify(self, event: TaskEvent) -> None:
|
||||
level = _status_to_level(event.status)
|
||||
if level is None or level not in self._levels:
|
||||
return
|
||||
payload = self._build_payload(event)
|
||||
self._send(payload)
|
||||
|
||||
def _build_payload(self, event: TaskEvent) -> dict[str, Any]:
|
||||
"""构建请求体。子类覆盖以适配平台格式。"""
|
||||
return {"text": _format_event_message(event)}
|
||||
|
||||
def _send(self, payload: dict[str, Any]) -> None:
|
||||
"""发送 JSON POST 请求,失败仅记录日志。"""
|
||||
data = dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
self._url,
|
||||
data=data,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=self._timeout) as resp:
|
||||
_ = resp.read()
|
||||
except (urllib.error.URLError, OSError) as exc:
|
||||
logger.warning("webhook 通知发送失败 (%s): %s", self._url, exc)
|
||||
|
||||
def close(self) -> None:
|
||||
"""无持久连接需关闭。"""
|
||||
|
||||
|
||||
class FeishuNotifier(WebhookNotifier):
|
||||
"""飞书(Lark)自定义机器人通知器。
|
||||
|
||||
消息格式:``{"msg_type": "text", "content": {"text": ...}}``。
|
||||
"""
|
||||
|
||||
@override
|
||||
def _build_payload(self, event: TaskEvent) -> dict[str, Any]:
|
||||
return {
|
||||
"msg_type": "text",
|
||||
"content": {"text": _format_event_message(event)},
|
||||
}
|
||||
|
||||
|
||||
class DingTalkNotifier(WebhookNotifier):
|
||||
"""钉钉自定义机器人通知器。
|
||||
|
||||
消息格式:``{"msgtype": "text", "text": {"content": ...}}``。
|
||||
"""
|
||||
|
||||
@override
|
||||
def _build_payload(self, event: TaskEvent) -> dict[str, Any]:
|
||||
return {
|
||||
"msgtype": "text",
|
||||
"text": {"content": _format_event_message(event)},
|
||||
}
|
||||
|
||||
|
||||
class WeChatNotifier(WebhookNotifier):
|
||||
"""企业微信群机器人通知器。
|
||||
|
||||
消息格式:``{"msgtype": "text", "text": {"content": ...}}``。
|
||||
"""
|
||||
|
||||
@override
|
||||
def _build_payload(self, event: TaskEvent) -> dict[str, Any]:
|
||||
return {
|
||||
"msgtype": "text",
|
||||
"text": {"content": _format_event_message(event)},
|
||||
}
|
||||
|
||||
|
||||
class SlackNotifier(WebhookNotifier):
|
||||
"""Slack Incoming Webhook 通知器。
|
||||
|
||||
消息格式:``{"text": ...}``。Slack webhook 仅需 ``text`` 字段,
|
||||
与基类格式一致,独立成类便于类型区分与后续扩展(如 channel/username)。
|
||||
"""
|
||||
|
||||
@override
|
||||
def _build_payload(self, event: TaskEvent) -> dict[str, Any]:
|
||||
return {"text": _format_event_message(event)}
|
||||
|
||||
|
||||
class DiscordNotifier(WebhookNotifier):
|
||||
"""Discord Webhook 通知器。
|
||||
|
||||
消息格式:``{"content": ...}``。Discord webhook 使用 ``content`` 字段
|
||||
承载消息文本。
|
||||
"""
|
||||
|
||||
@override
|
||||
def _build_payload(self, event: TaskEvent) -> dict[str, Any]:
|
||||
return {"content": _format_event_message(event)}
|
||||
|
||||
|
||||
class TelegramNotifier(WebhookNotifier):
|
||||
"""Telegram Bot 通知器。
|
||||
|
||||
通过 ``sendMessage`` API 发送消息。需要在 ``__init__`` 中传入
|
||||
``chat_id``(聊天/群组 ID),并使用 bot token 拼接 webhook URL:
|
||||
``https://api.telegram.org/bot<TOKEN>/sendMessage``。
|
||||
|
||||
消息格式:``{"chat_id": ..., "text": ...}``。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
chat_id:
|
||||
接收消息的聊天或群组 ID(数字字符串或 @channelusername)。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str,
|
||||
chat_id: str,
|
||||
levels: Iterable[NotificationLevel] = ALL_LEVELS,
|
||||
timeout: float = 10.0,
|
||||
) -> None:
|
||||
super().__init__(url=url, levels=levels, timeout=timeout)
|
||||
self._chat_id = chat_id
|
||||
|
||||
@override
|
||||
def _build_payload(self, event: TaskEvent) -> dict[str, Any]:
|
||||
return {
|
||||
"chat_id": self._chat_id,
|
||||
"text": _format_event_message(event),
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
"""ops 子包 — CLI 工具模块集合 (按功能分组).
|
||||
|
||||
每个子模块使用 ``@px.tool`` 装饰器注册 CLI 工具, 由 ``pf`` 入口按需
|
||||
``importlib.import_module`` 触发注册. 子模块按功能分组到子目录.
|
||||
|
||||
子目录
|
||||
------
|
||||
- :mod:`files` —— 文件与文档操作 (filedate/filelevel/folderback/folderzip/pdftool/screenshot)
|
||||
- :mod:`dev` —— 开发与构建工具 (autofmt/bumpversion/gittool/packtool/piptool/pymake/lscalc)
|
||||
- :mod:`system` —— 系统工具 (clr/reseticoncache/taskkill/which)
|
||||
- :mod:`infra` —— 基础设施与服务部署 (dockercmd/envdev/sshcopyid/msdownload/sglang)
|
||||
|
||||
:mod:`_common` 保留在根目录, 提供跨平台命令选择、平台守卫等共享辅助.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__: list[str] = []
|
||||
@@ -0,0 +1,66 @@
|
||||
"""ops 共享辅助: 跨平台命令选择、平台守卫、共享忽略模式."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
__all__ = ["IGNORE_PATTERNS", "ensure_platform", "platform_command"]
|
||||
|
||||
IGNORE_PATTERNS: list[str] = [
|
||||
"__pycache__",
|
||||
"*.pyc",
|
||||
"*.pyo",
|
||||
".git",
|
||||
".venv",
|
||||
".idea",
|
||||
".vscode",
|
||||
"*.egg-info",
|
||||
"dist",
|
||||
"build",
|
||||
".pytest_cache",
|
||||
".tox",
|
||||
".mypy_cache",
|
||||
]
|
||||
|
||||
|
||||
def platform_command(windows: Sequence[str], posix: Sequence[str]) -> list[str]:
|
||||
"""按当前平台返回对应命令.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
windows : Sequence[str]
|
||||
Windows 平台命令 (含参数).
|
||||
posix : Sequence[str]
|
||||
Linux/macOS 平台命令 (含参数).
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[str]
|
||||
当前平台对应的命令列表 (副本, 可安全修改).
|
||||
"""
|
||||
return list(windows) if Constants.IS_WINDOWS else list(posix)
|
||||
|
||||
|
||||
def ensure_platform(predicate: bool, tool_name: str, platform_name: str) -> bool:
|
||||
"""平台不满足时打印提示, 返回是否满足.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
predicate : bool
|
||||
平台匹配谓词 (如 ``Constants.IS_WINDOWS``).
|
||||
tool_name : str
|
||||
工具名称, 用于提示信息前缀.
|
||||
platform_name : str
|
||||
目标平台名称, 用于提示信息.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
平台满足时返回 True, 否则打印 ``{tool_name}: 仅在 {platform_name} 上支持`` 并返回 False.
|
||||
"""
|
||||
if not predicate:
|
||||
print(f"{tool_name}: 仅在 {platform_name} 上支持")
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1,5 @@
|
||||
"""dev 子包 — 开发与构建工具."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__: list[str] = []
|
||||
@@ -1,52 +1,60 @@
|
||||
"""自动格式化工具模块.
|
||||
"""autofmt - 自动格式化工具.
|
||||
|
||||
提供 Python 代码自动格式化的常用功能封装,
|
||||
支持 docstring 自动生成、pyproject.toml 配置同步等功能.
|
||||
提供格式化代码/代码检查/自动添加文档/同步配置 子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.ops._common import IGNORE_PATTERNS
|
||||
|
||||
try:
|
||||
import tomllib # noqa: F401
|
||||
|
||||
HAS_TOMLLIB = True
|
||||
except ImportError:
|
||||
HAS_TOMLLIB = False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
IGNORE_PATTERNS = [
|
||||
"__pycache__",
|
||||
"*.pyc",
|
||||
"*.pyo",
|
||||
".git",
|
||||
".venv",
|
||||
".idea",
|
||||
".vscode",
|
||||
"*.egg-info",
|
||||
"dist",
|
||||
"build",
|
||||
".pytest_cache",
|
||||
".tox",
|
||||
".mypy_cache",
|
||||
__all__ = [
|
||||
"add_docstring",
|
||||
"auto_add_docstrings",
|
||||
"fmt",
|
||||
"format_all",
|
||||
"format_with_ruff",
|
||||
"generate_module_docstring",
|
||||
"lint",
|
||||
"lint_with_ruff",
|
||||
"sync_pyproject_config",
|
||||
]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool("autofmt", subcommand="fmt", help="格式化代码")
|
||||
def fmt(target: str = ".") -> None:
|
||||
"""格式化代码 (ruff format).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
target : str
|
||||
目标路径 (默认: .)
|
||||
"""
|
||||
format_with_ruff(Path(target), fix=False)
|
||||
|
||||
|
||||
@px.tool("autofmt", subcommand="lint", help="代码检查")
|
||||
def lint(target: str = ".", fix: bool = False) -> None:
|
||||
"""代码检查 (ruff check).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
target : str
|
||||
目标路径 (默认: .)
|
||||
fix : bool
|
||||
自动修复问题
|
||||
"""
|
||||
lint_with_ruff(Path(target), fix=fix)
|
||||
|
||||
|
||||
def format_with_ruff(target: Path, fix: bool = True) -> None:
|
||||
"""使用 ruff 格式化代码.
|
||||
|
||||
@@ -102,12 +110,10 @@ def add_docstring(file_path: Path, docstring: str) -> bool:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
tree = ast.parse(content)
|
||||
|
||||
# 检查是否已有 docstring
|
||||
first_node = tree.body[0] if tree.body else None
|
||||
if first_node and isinstance(first_node, ast.Expr) and isinstance(first_node.value, ast.Constant):
|
||||
return False
|
||||
|
||||
# 添加 docstring
|
||||
lines = content.splitlines()
|
||||
doc_lines = docstring.splitlines()
|
||||
doc_lines.append("")
|
||||
@@ -138,7 +144,6 @@ def generate_module_docstring(file_path: Path) -> str:
|
||||
stem = file_path.stem
|
||||
parent = file_path.parent.name
|
||||
|
||||
# 关键词匹配
|
||||
keywords = {
|
||||
"cli": f"Command-line interface for {parent}",
|
||||
"gui": f"Graphical user interface for {parent}",
|
||||
@@ -155,13 +160,14 @@ def generate_module_docstring(file_path: Path) -> str:
|
||||
return f'"""{stem.replace("_", " ").title()} module."""'
|
||||
|
||||
|
||||
def auto_add_docstrings(root_dir: Path) -> int:
|
||||
@px.tool("autofmt", subcommand="doc", help="自动添加文档字符串")
|
||||
def auto_add_docstrings(root_dir: Path = Path()) -> int:
|
||||
"""自动为所有 Python 文件添加 docstring.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root_dir : Path
|
||||
根目录
|
||||
根目录 (默认: 当前目录)
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -170,7 +176,6 @@ def auto_add_docstrings(root_dir: Path) -> int:
|
||||
"""
|
||||
count = 0
|
||||
for py_file in root_dir.rglob("*.py"):
|
||||
# 跳过忽略的文件
|
||||
if any(pattern in str(py_file) for pattern in IGNORE_PATTERNS):
|
||||
continue
|
||||
|
||||
@@ -182,20 +187,20 @@ def auto_add_docstrings(root_dir: Path) -> int:
|
||||
return count
|
||||
|
||||
|
||||
def sync_pyproject_config(root_dir: Path) -> None:
|
||||
@px.tool("autofmt", subcommand="sync", help="同步 pyproject 配置")
|
||||
def sync_pyproject_config(root_dir: Path = Path()) -> None:
|
||||
"""同步 pyproject.toml 配置到子项目.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root_dir : Path
|
||||
根目录
|
||||
根目录 (默认: 当前目录)
|
||||
"""
|
||||
main_toml = root_dir / "pyproject.toml"
|
||||
if not main_toml.exists():
|
||||
print(f"主项目配置文件不存在: {main_toml}")
|
||||
return
|
||||
|
||||
# 查找所有子项目的 pyproject.toml
|
||||
sub_tomls = [p for p in root_dir.rglob("pyproject.toml") if p != main_toml and ".venv" not in str(p)]
|
||||
|
||||
if not sub_tomls:
|
||||
@@ -204,7 +209,6 @@ def sync_pyproject_config(root_dir: Path) -> None:
|
||||
|
||||
print(f"找到 {len(sub_tomls)} 个子项目配置文件")
|
||||
|
||||
# 对每个子项目调用 ruff format
|
||||
for sub_toml in sub_tomls:
|
||||
subprocess.run(["ruff", "format", str(sub_toml)], check=False)
|
||||
|
||||
@@ -219,64 +223,6 @@ def format_all(root_dir: Path) -> None:
|
||||
root_dir : Path
|
||||
根目录
|
||||
"""
|
||||
# 使用 ruff format
|
||||
subprocess.run(["ruff", "format", str(root_dir)], check=True)
|
||||
|
||||
# 使用 ruff check
|
||||
subprocess.run(["ruff", "check", "--fix", "--unsafe-fixes", str(root_dir)], check=True)
|
||||
|
||||
print(f"格式化完成: {root_dir}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""自动格式化工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="AutoFmt - 自动格式化工具",
|
||||
usage="autofmt <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# ruff format 命令
|
||||
format_parser = subparsers.add_parser("fmt", help="使用 ruff 格式化代码")
|
||||
format_parser.add_argument("--target", type=str, default=".", help="目标路径")
|
||||
|
||||
# ruff check 命令
|
||||
lint_parser = subparsers.add_parser("lint", help="使用 ruff 检查代码")
|
||||
lint_parser.add_argument("--target", type=str, default=".", help="目标路径")
|
||||
lint_parser.add_argument("--fix", action="store_true", help="自动修复")
|
||||
|
||||
# 自动添加 docstring 命令
|
||||
doc_parser = subparsers.add_parser("doc", help="自动添加 docstring")
|
||||
doc_parser.add_argument("--root-dir", type=str, default=".", help="根目录")
|
||||
|
||||
# 同步配置命令
|
||||
sync_parser = subparsers.add_parser("sync", help="同步 pyproject.toml 配置")
|
||||
sync_parser.add_argument("--root-dir", type=str, default=".", help="根目录")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "fmt":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("ruff_format", cmd=["ruff", "format", args.target], verbose=True)])
|
||||
elif args.command == "lint":
|
||||
cmd = ["ruff", "check", args.target]
|
||||
if args.fix:
|
||||
cmd.extend(["--fix", "--unsafe-fixes"])
|
||||
graph = px.Graph.from_specs([px.TaskSpec("ruff_check", cmd=cmd, verbose=True)])
|
||||
elif args.command == "doc":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("auto_docstring", fn=auto_add_docstrings, args=(Path(args.root_dir),), verbose=True)]
|
||||
)
|
||||
elif args.command == "sync":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("sync_config", fn=sync_pyproject_config, args=(Path(args.root_dir),), verbose=True)]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,232 @@
|
||||
"""版本号管理模块.
|
||||
|
||||
提供单文件版本号更新 (``bump_file_version``) 与项目级批量版本号同步
|
||||
(``bump_project_version``) 能力. ``bump_project_version`` 通过 ``@px.tool``
|
||||
注册为 CLI 工具, 可通过 ``pf bumpversion`` 调用.
|
||||
|
||||
设计要点
|
||||
--------
|
||||
``bump_project_version`` 采用 "先读取基准、再统一写入" 的两阶段策略:
|
||||
先扫描所有 ``__init__.py`` / ``pyproject.toml`` 文件, 读取各自的版本号,
|
||||
取最大值作为基准版本计算新版本号, 然后把新版本号统一写入所有文件,
|
||||
避免文件间版本号不同步导致的跳号问题.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"BumpVersionType",
|
||||
"bump_file_version",
|
||||
"bump_project_version",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
BumpVersionType = Literal["patch", "minor", "major"]
|
||||
|
||||
_PYPROJECT_VERSION_PATTERN = re.compile(
|
||||
r'(?:^|\n)\s*version\s*=\s*["\']'
|
||||
r"(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
|
||||
r"(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
|
||||
r"(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?"
|
||||
r'["\']',
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
_INIT_VERSION_PATTERN = re.compile(
|
||||
r'(?:^|\n)\s*__version__\s*=\s*["\']'
|
||||
r"(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
|
||||
r"(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
|
||||
r"(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?"
|
||||
r'["\']',
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
_IGNORE_DIRS = frozenset({".venv", "venv", ".git", "__pycache__", ".tox", "node_modules", "build", "dist", ".eggs"})
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 私有辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _get_pattern_for_file(file_name: str) -> re.Pattern[str] | None:
|
||||
"""根据文件类型获取对应的正则表达式."""
|
||||
if file_name == "pyproject.toml":
|
||||
return _PYPROJECT_VERSION_PATTERN
|
||||
if file_name == "__init__.py":
|
||||
return _INIT_VERSION_PATTERN
|
||||
return None
|
||||
|
||||
|
||||
def _calculate_new_version(major: int, minor: int, patch: int, part: BumpVersionType) -> str:
|
||||
"""计算新版本号."""
|
||||
if part == "major":
|
||||
return f"{major + 1}.0.0"
|
||||
if part == "minor":
|
||||
return f"{major}.{minor + 1}.0"
|
||||
return f"{major}.{minor}.{patch + 1}"
|
||||
|
||||
|
||||
def _build_replacement_string(original_match: str, new_version: str, file_name: str) -> str:
|
||||
"""构建替换字符串, 保留原始格式."""
|
||||
quote_char = '"' if '"' in original_match else "'"
|
||||
key = "__version__" if file_name == "__init__.py" else "version"
|
||||
prefix_match = re.match(rf"(\s*{key}\s*=\s*)[\"']", original_match)
|
||||
prefix = prefix_match.group(1) if prefix_match else f"{key} = "
|
||||
return f"{prefix}{quote_char}{new_version}{quote_char}"
|
||||
|
||||
|
||||
def _read_version_tuple(file_path: Path) -> tuple[int, int, int] | None:
|
||||
"""从文件中读取版本号, 返回 (major, minor, patch) 元组; 未找到返回 None.
|
||||
|
||||
读取失败时抛出 ``OSError`` / ``UnicodeDecodeError`` 由调用方处理.
|
||||
"""
|
||||
pattern = _get_pattern_for_file(file_path.name)
|
||||
if pattern is None:
|
||||
return None
|
||||
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
match = pattern.search(content)
|
||||
if not match:
|
||||
return None
|
||||
|
||||
return int(match.group("major")), int(match.group("minor")), int(match.group("patch"))
|
||||
|
||||
|
||||
def _write_version_to_file(file_path: Path, new_version: str) -> bool:
|
||||
"""把新版本号写入指定文件; 成功返回 True, 未匹配到版本号返回 False."""
|
||||
pattern = _get_pattern_for_file(file_path.name)
|
||||
if pattern is None: # pragma: no cover - 调用方已保证 pattern 不为 None
|
||||
return False
|
||||
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
match = pattern.search(content)
|
||||
if not match: # pragma: no cover - 调用方已通过 _read_version_tuple 验证
|
||||
return False
|
||||
|
||||
replacement = _build_replacement_string(match.group(0), new_version, file_path.name)
|
||||
content = content.replace(match.group(0), replacement)
|
||||
|
||||
try:
|
||||
file_path.write_text(content, encoding="utf-8")
|
||||
except OSError as e:
|
||||
print(f"更新文件 {file_path} 版本号时出错: {e}")
|
||||
raise
|
||||
|
||||
return True
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 公共函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def bump_file_version(file_path: Path, part: BumpVersionType = "patch") -> str | None:
|
||||
"""更新单个文件中的版本号.
|
||||
|
||||
读取文件当前版本号, 按 ``part`` 指定的部分递增, 写回文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_path : Path
|
||||
要更新的文件路径 (``pyproject.toml`` 或 ``__init__.py``)
|
||||
part : BumpVersionType
|
||||
版本部分: patch, minor, major
|
||||
|
||||
Returns
|
||||
-------
|
||||
str | None
|
||||
更新后的新版本号; 文件中未找到版本号或读取失败时返回 None
|
||||
"""
|
||||
version_tuple = _read_version_tuple(file_path)
|
||||
if version_tuple is None:
|
||||
print(f"文件 {file_path} 中未找到版本号模式")
|
||||
return None
|
||||
|
||||
major, minor, patch = version_tuple
|
||||
new_version = _calculate_new_version(major, minor, patch, part)
|
||||
|
||||
if not _write_version_to_file(file_path, new_version): # pragma: no cover - _read_version_tuple 已验证
|
||||
return None
|
||||
|
||||
return new_version
|
||||
|
||||
|
||||
@px.tool("bumpversion", help="版本号自动管理")
|
||||
def bump_project_version(part: BumpVersionType = "patch", no_tag: bool = False) -> str | None:
|
||||
"""批量同步项目所有版本号文件并提交.
|
||||
|
||||
扫描当前目录下所有 ``__init__.py`` 和 ``pyproject.toml`` 文件
|
||||
(排除虚拟环境和缓存目录), 先读取每个文件的当前版本号取最大值作为基准,
|
||||
计算新版本号后统一写入所有文件, 最后执行 git add (按文件名) + commit + tag.
|
||||
|
||||
采用 "先读取基准、再统一写入" 的两阶段策略, 即使某些文件版本号不同步,
|
||||
也能在一次 bump 后重新对齐, 避免跳号.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
part : BumpVersionType
|
||||
版本部分: patch, minor, major
|
||||
no_tag : bool
|
||||
提交后不创建 git tag
|
||||
|
||||
Returns
|
||||
-------
|
||||
str | None
|
||||
更新后的新版本号; 未找到版本号文件时返回 None
|
||||
"""
|
||||
all_files: set[Path] = set()
|
||||
for pattern in ("__init__.py", "pyproject.toml"):
|
||||
for file in Path.cwd().rglob(pattern):
|
||||
if not any(ignore_dir in file.parts for ignore_dir in _IGNORE_DIRS):
|
||||
all_files.add(file)
|
||||
|
||||
if not all_files:
|
||||
print("未找到包含版本号的文件")
|
||||
return None
|
||||
|
||||
print(f"找到 {len(all_files)} 个文件需要更新版本号")
|
||||
cwd = Path.cwd()
|
||||
for file in sorted(all_files):
|
||||
print(f" - {file.relative_to(cwd)}")
|
||||
|
||||
# 阶段 1: 读取所有文件版本号, 取最大值作为基准
|
||||
versions: list[tuple[int, int, int]] = []
|
||||
for file in sorted(all_files):
|
||||
v = _read_version_tuple(file)
|
||||
if v is not None:
|
||||
versions.append(v)
|
||||
|
||||
if not versions:
|
||||
print("未能从任何文件读取版本号")
|
||||
return None
|
||||
|
||||
major, minor, patch = max(versions)
|
||||
new_version = _calculate_new_version(major, minor, patch, part)
|
||||
print(f"基准版本: {major}.{minor}.{patch} -> 新版本: {new_version}")
|
||||
|
||||
# 阶段 2: 统一写入新版本号到所有文件
|
||||
for file in sorted(all_files):
|
||||
_write_version_to_file(file, new_version)
|
||||
|
||||
# 阶段 3: git add (按文件名) + commit + tag
|
||||
relative_files = [str(file.relative_to(cwd)) for file in sorted(all_files)]
|
||||
subprocess.run(["git", "add", *relative_files], check=True)
|
||||
subprocess.run(["git", "commit", "-m", f"bump version to {new_version}"], check=True)
|
||||
|
||||
if not no_tag:
|
||||
tag_name = f"v{new_version}"
|
||||
subprocess.run(["git", "tag", "-a", tag_name, "-m", f"Release {tag_name}"], check=True)
|
||||
print(f"已创建标签: {tag_name}")
|
||||
|
||||
return new_version
|
||||
@@ -0,0 +1,142 @@
|
||||
"""gittool - Git 执行工具.
|
||||
|
||||
提供添加提交/清理/初始化/推送/拉取 子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
EXCLUDE_DIRS = [
|
||||
# 编辑器相关目录
|
||||
".vscode",
|
||||
".idea",
|
||||
".editorconfig",
|
||||
".trae",
|
||||
".qoder",
|
||||
# 项目相关目录
|
||||
".venv",
|
||||
".git",
|
||||
".ruff_cache",
|
||||
".tox",
|
||||
"node_modules",
|
||||
]
|
||||
|
||||
EXCLUDE_CMDS = [arg for d in EXCLUDE_DIRS for arg in ["-e", d]]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 私有辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def not_has_git_repo() -> bool:
|
||||
"""检查当前目录没有 Git 仓库."""
|
||||
return not Path.cwd().exists() or not (Path.cwd() / ".git").is_dir()
|
||||
|
||||
|
||||
def has_files() -> bool:
|
||||
"""检查当前 Git 仓库是否有未提交的更改."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "status", "--porcelain"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return bool(result.stdout.strip())
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# fn 子命令
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool("gittool", subcommand="a", help="添加并提交")
|
||||
def git_add_commit(message: str = "chore: update") -> None:
|
||||
"""执行 git add + git commit (仅当有未提交更改时).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
message : str
|
||||
提交信息 (默认: ``chore: update``)
|
||||
"""
|
||||
if not has_files():
|
||||
print("没有文件需要提交")
|
||||
return
|
||||
subprocess.run(["git", "add", "."], check=True)
|
||||
subprocess.run(["git", "commit", "-m", message], check=True)
|
||||
|
||||
|
||||
@px.tool("gittool", subcommand="i", help="初始化并提交")
|
||||
def git_init_add_commit(message: str = "init commit") -> None:
|
||||
"""执行 git init (若需) + git add + git commit (若有更改).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
message : str
|
||||
提交信息 (默认: ``init commit``)
|
||||
"""
|
||||
if not_has_git_repo():
|
||||
subprocess.run(["git", "init"], check=True)
|
||||
if has_files():
|
||||
subprocess.run(["git", "add", "."], check=True)
|
||||
subprocess.run(["git", "commit", "-m", message], check=True)
|
||||
else:
|
||||
print("没有文件需要提交")
|
||||
|
||||
|
||||
@px.tool("gittool", subcommand="isub", help="初始化子目录")
|
||||
def init_sub_dirs() -> None:
|
||||
"""初始化子目录的 Git 仓库."""
|
||||
sub_dirs = [subdir for subdir in Path.cwd().iterdir() if subdir.is_dir()]
|
||||
for subdir in sub_dirs:
|
||||
px.run(
|
||||
px.Graph().chain(
|
||||
px.cmd(["git", "init"], conditions=(lambda _: not_has_git_repo(),), cwd=subdir),
|
||||
px.cmd(["git", "add", "."]),
|
||||
px.cmd(["git", "commit", "-m", "init commit"]),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# cmd 子命令
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool(
|
||||
"gittool",
|
||||
subcommand="clean",
|
||||
help="清理未跟踪文件",
|
||||
cmd=["git", "clean", "-xfd", *EXCLUDE_CMDS],
|
||||
hidden=True,
|
||||
)
|
||||
def clean() -> None:
|
||||
"""清理 Git 未跟踪文件 (隐藏命令, 被 c 依赖)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"gittool",
|
||||
subcommand="c",
|
||||
help="清理并查看状态",
|
||||
cmd=["git", "status", "--porcelain"],
|
||||
needs=["clean"],
|
||||
)
|
||||
def c() -> None:
|
||||
"""清理未跟踪文件并查看 Git 状态."""
|
||||
|
||||
|
||||
@px.tool("gittool", subcommand="p", help="推送", cmd=["git", "push"])
|
||||
def p() -> None:
|
||||
"""推送代码到远程仓库."""
|
||||
|
||||
|
||||
@px.tool("gittool", subcommand="pl", help="拉取", cmd=["git", "pull"])
|
||||
def pl() -> None:
|
||||
"""从远程仓库拉取代码."""
|
||||
@@ -0,0 +1,111 @@
|
||||
"""lscalc - LS-DYNA 计算工具.
|
||||
|
||||
运行 LS-DYNA 计算 (单机/MPI) 与进程状态检查.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
_DEFAULT_NCPU: int = 4
|
||||
|
||||
|
||||
def get_ls_dyna_command(input_file: str, ncpu: int) -> list[str]:
|
||||
"""获取 LS-DYNA 命令.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_file : str
|
||||
输入文件路径
|
||||
ncpu : int
|
||||
CPU 核心数
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[str]
|
||||
LS-DYNA 命令列表
|
||||
"""
|
||||
return ["ls-dyna_mpp", f"i={input_file}", f"ncpu={ncpu}"]
|
||||
|
||||
|
||||
@px.tool("lscalc", subcommand="run", help="运行 LS-DYNA 计算")
|
||||
def run_ls_dyna(input_file: str, ncpu: int = _DEFAULT_NCPU) -> None:
|
||||
"""运行 LS-DYNA 计算.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_file : str
|
||||
输入文件路径
|
||||
ncpu : int
|
||||
CPU 核心数
|
||||
"""
|
||||
input_path = Path(input_file)
|
||||
if not input_path.exists():
|
||||
print(f"输入文件不存在: {input_path}")
|
||||
return
|
||||
|
||||
cmd = get_ls_dyna_command(input_file, ncpu)
|
||||
try:
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"LS-DYNA 计算完成: {input_file}")
|
||||
except FileNotFoundError:
|
||||
print("未找到 ls-dyna_mpp 命令")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"LS-DYNA 计算失败: {e}")
|
||||
|
||||
|
||||
@px.tool("lscalc", subcommand="mpi", help="运行 LS-DYNA MPI 计算")
|
||||
def run_ls_dyna_mpi(input_file: str, ncpu: int = _DEFAULT_NCPU) -> None:
|
||||
"""运行 LS-DYNA MPI 计算.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_file : str
|
||||
输入文件路径
|
||||
ncpu : int
|
||||
CPU 核心数
|
||||
"""
|
||||
input_path = Path(input_file)
|
||||
if not input_path.exists():
|
||||
print(f"输入文件不存在: {input_path}")
|
||||
return
|
||||
|
||||
cmd = ["mpirun", "-np", str(ncpu), "ls-dyna_mpp", f"i={input_file}"]
|
||||
try:
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"LS-DYNA MPI 计算完成: {input_file}")
|
||||
except FileNotFoundError:
|
||||
print("未找到 mpirun 或 ls-dyna_mpp 命令")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"LS-DYNA MPI 计算失败: {e}")
|
||||
|
||||
|
||||
@px.tool("lscalc", subcommand="status", help="检查 LS-DYNA 进程状态")
|
||||
def check_ls_dyna_status() -> None:
|
||||
"""检查 LS-DYNA 进程状态."""
|
||||
try:
|
||||
if Constants.IS_WINDOWS:
|
||||
result = subprocess.run(
|
||||
["tasklist", "/fi", "imagename eq ls-dyna_mpp.exe"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
print(result.stdout)
|
||||
else:
|
||||
result = subprocess.run(
|
||||
["pgrep", "-f", "ls-dyna"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.stdout.strip():
|
||||
print(f"运行中的 LS-DYNA 进程 PID: {result.stdout.strip()}")
|
||||
else:
|
||||
print("没有运行中的 LS-DYNA 进程")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"检查进程状态失败: {e}")
|
||||
@@ -0,0 +1,236 @@
|
||||
"""packtool - Python 打包工具.
|
||||
|
||||
提供源码打包/依赖打包/wheel 构建/嵌入式 Python 安装/zip 包创建/清理 子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.ops._common import IGNORE_PATTERNS
|
||||
|
||||
__all__ = [
|
||||
"clean_build_dir",
|
||||
"create_zip_package",
|
||||
"install_embed_python",
|
||||
"pack_dependencies",
|
||||
"pack_source",
|
||||
"pack_wheel",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
DEFAULT_BUILD_DIR = ".pypack"
|
||||
DEFAULT_DIST_DIR = "dist"
|
||||
DEFAULT_LIB_DIR = "libs"
|
||||
DEFAULT_CACHE_DIR = ".cache/pypack"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 公共函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool("packtool", subcommand="src", help="打包源码")
|
||||
def pack_source(project_dir: Path = Path(), output_dir: Path = Path(".pypack")) -> None:
|
||||
"""打包项目源码.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
project_dir : Path
|
||||
项目目录 (默认: 当前目录)
|
||||
output_dir : Path
|
||||
输出目录 (默认: .pypack)
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pyproject_file = project_dir / "pyproject.toml"
|
||||
project_name = project_dir.name
|
||||
|
||||
if pyproject_file.exists():
|
||||
try:
|
||||
import tomllib
|
||||
|
||||
content = pyproject_file.read_text(encoding="utf-8")
|
||||
data = tomllib.loads(content)
|
||||
project_name = data.get("project", {}).get("name", project_name)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
source_dir = output_dir / "src" / project_name
|
||||
source_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
src_subdir = project_dir / "src"
|
||||
if src_subdir.exists():
|
||||
shutil.copytree(
|
||||
src_subdir,
|
||||
source_dir / "src",
|
||||
ignore=shutil.ignore_patterns(*IGNORE_PATTERNS),
|
||||
dirs_exist_ok=True,
|
||||
)
|
||||
else:
|
||||
for item in project_dir.iterdir():
|
||||
if item.name in IGNORE_PATTERNS or item.name.startswith("."):
|
||||
continue
|
||||
dst_item = source_dir / item.name
|
||||
if item.is_dir():
|
||||
shutil.copytree(
|
||||
item,
|
||||
dst_item,
|
||||
ignore=shutil.ignore_patterns(*IGNORE_PATTERNS),
|
||||
dirs_exist_ok=True,
|
||||
)
|
||||
else:
|
||||
shutil.copy2(item, dst_item)
|
||||
|
||||
print(f"源码打包完成: {source_dir}")
|
||||
|
||||
|
||||
@px.tool("packtool", subcommand="deps", help="打包依赖")
|
||||
def pack_dependencies(lib_dir: Path = Path("libs"), dependencies: list[str] | None = None) -> None:
|
||||
"""打包项目依赖.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lib_dir : Path
|
||||
依赖库目录 (默认: libs)
|
||||
dependencies : list[str] | None
|
||||
依赖列表
|
||||
"""
|
||||
if dependencies is None:
|
||||
dependencies = []
|
||||
|
||||
lib_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if not dependencies:
|
||||
print("没有依赖需要打包")
|
||||
return
|
||||
|
||||
cmd = [
|
||||
"pip",
|
||||
"install",
|
||||
"--target",
|
||||
str(lib_dir),
|
||||
"--no-compile",
|
||||
"--no-warn-script-location",
|
||||
]
|
||||
cmd.extend(dependencies)
|
||||
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"依赖打包完成: {lib_dir}")
|
||||
|
||||
|
||||
@px.tool("packtool", subcommand="wheel", help="构建 wheel")
|
||||
def pack_wheel(project_dir: Path = Path(), output_dir: Path = Path("dist")) -> None:
|
||||
"""打包项目为 wheel 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
project_dir : Path
|
||||
项目目录 (默认: 当前目录)
|
||||
output_dir : Path
|
||||
输出目录 (默认: dist)
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
cmd = [
|
||||
"pip",
|
||||
"wheel",
|
||||
"--no-deps",
|
||||
"--wheel-dir",
|
||||
str(output_dir),
|
||||
str(project_dir),
|
||||
]
|
||||
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"Wheel 打包完成: {output_dir}")
|
||||
|
||||
|
||||
@px.tool("packtool", subcommand="embed", help="安装嵌入式 Python")
|
||||
def install_embed_python(version: str = "3.10", output_dir: Path = Path("python")) -> None:
|
||||
"""安装嵌入式 Python.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
version : str
|
||||
Python 版本 (如: 3.10, 3.11), 默认 3.10
|
||||
output_dir : Path
|
||||
输出目录 (默认: python)
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
arch = platform.machine().lower()
|
||||
if arch in ["x86_64", "amd64"]:
|
||||
arch = "amd64"
|
||||
elif arch in ["arm64", "aarch64"]:
|
||||
arch = "arm64"
|
||||
|
||||
version_map = {
|
||||
"3.8": "3.8.10",
|
||||
"3.9": "3.9.13",
|
||||
"3.10": "3.10.11",
|
||||
"3.11": "3.11.9",
|
||||
"3.12": "3.12.4",
|
||||
}
|
||||
full_version = version_map.get(version, f"{version}.0")
|
||||
|
||||
url = f"https://www.python.org/ftp/python/{full_version}/python-{full_version}-embed-{arch}.zip"
|
||||
|
||||
cache_file = Path(DEFAULT_CACHE_DIR) / f"python-{full_version}-embed-{arch}.zip"
|
||||
cache_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if not cache_file.exists():
|
||||
print(f"正在下载嵌入式 Python {full_version}...")
|
||||
urllib.request.urlretrieve(url, cache_file)
|
||||
print(f"下载完成: {cache_file}")
|
||||
|
||||
with zipfile.ZipFile(cache_file, "r") as zf:
|
||||
zf.extractall(output_dir)
|
||||
|
||||
print(f"嵌入式 Python 安装完成: {output_dir}")
|
||||
|
||||
|
||||
@px.tool("packtool", subcommand="zip", help="创建 zip 包")
|
||||
def create_zip_package(source_dir: Path = Path(), output_file: Path = Path("package.zip")) -> None:
|
||||
"""创建 ZIP 打包文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
source_dir : Path
|
||||
源目录 (默认: 当前目录)
|
||||
output_file : Path
|
||||
输出文件 (默认: package.zip)
|
||||
"""
|
||||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with zipfile.ZipFile(output_file, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
for file in source_dir.rglob("*"):
|
||||
if file.is_file():
|
||||
arcname = file.relative_to(source_dir)
|
||||
zf.write(file, arcname)
|
||||
|
||||
print(f"ZIP 打包完成: {output_file}")
|
||||
|
||||
|
||||
@px.tool("packtool", subcommand="clean", help="清理构建目录")
|
||||
def clean_build_dir(build_dir: Path = Path(".pypack")) -> None:
|
||||
"""清理构建目录.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
build_dir : Path
|
||||
构建目录 (默认: .pypack)
|
||||
"""
|
||||
if build_dir.exists():
|
||||
shutil.rmtree(build_dir)
|
||||
print(f"清理完成: {build_dir}")
|
||||
else:
|
||||
print(f"目录不存在: {build_dir}")
|
||||
@@ -0,0 +1,177 @@
|
||||
"""piptool - pip 包管理工具.
|
||||
|
||||
提供安装/卸载/重装/下载/升级 pip/冻结依赖 子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"pip_download",
|
||||
"pip_freeze",
|
||||
"pip_install",
|
||||
"pip_reinstall",
|
||||
"pip_uninstall",
|
||||
"pip_upgrade",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
PACKAGE_DIR = "packages"
|
||||
REQUIREMENTS_FILE = "requirements.txt"
|
||||
|
||||
_PROTECTED_PACKAGES: frozenset[str] = frozenset(
|
||||
{
|
||||
"pyflowx",
|
||||
"bitool",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 私有辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _get_installed_packages() -> list[str]:
|
||||
"""获取当前环境中所有已安装的包名."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pip", "list", "--format=freeze"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
packages: list[str] = []
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
if line and "==" in line:
|
||||
pkg_name = line.split("==")[0].strip()
|
||||
packages.append(pkg_name)
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
return []
|
||||
return packages
|
||||
|
||||
|
||||
def _expand_wildcard_packages(pattern: str) -> list[str]:
|
||||
"""展开通配符模式为实际的包名列表."""
|
||||
if not any(char in pattern for char in ["*", "?", "[", "]"]):
|
||||
return [pattern]
|
||||
|
||||
installed_packages = _get_installed_packages()
|
||||
matched = [pkg for pkg in installed_packages if fnmatch.fnmatchcase(pkg.lower(), pattern.lower())]
|
||||
return matched
|
||||
|
||||
|
||||
def _filter_protected_packages(packages: list[str]) -> list[str]:
|
||||
"""过滤掉受保护的包名."""
|
||||
safe = [p for p in packages if p.lower() not in {p.lower() for p in _PROTECTED_PACKAGES}]
|
||||
filtered = [p for p in packages if p.lower() in {p.lower() for p in _PROTECTED_PACKAGES}]
|
||||
if filtered:
|
||||
print(f"跳过受保护的包: {', '.join(filtered)}")
|
||||
return safe
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 公共函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool("piptool", subcommand="i", help="安装包")
|
||||
def pip_install(packages: list[str]) -> None:
|
||||
"""安装包.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
packages : list[str]
|
||||
包名列表
|
||||
"""
|
||||
subprocess.run(["pip", "install", *packages], check=True)
|
||||
print(f"安装完成: {', '.join(packages)}")
|
||||
|
||||
|
||||
@px.tool("piptool", subcommand="u", help="卸载包")
|
||||
def pip_uninstall(packages: list[str]) -> None:
|
||||
"""卸载包.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
packages : list[str]
|
||||
包名列表
|
||||
"""
|
||||
packages_to_uninstall: list[str] = []
|
||||
for pattern in packages:
|
||||
packages_to_uninstall.extend(_expand_wildcard_packages(pattern))
|
||||
|
||||
packages_to_uninstall = _filter_protected_packages(packages_to_uninstall)
|
||||
|
||||
if not packages_to_uninstall:
|
||||
return
|
||||
|
||||
subprocess.run(["pip", "uninstall", "-y", *packages_to_uninstall], check=True)
|
||||
|
||||
|
||||
@px.tool("piptool", subcommand="r", help="重装包")
|
||||
def pip_reinstall(packages: list[str], offline: bool = False) -> None:
|
||||
"""重新安装包.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
packages : list[str]
|
||||
包名列表
|
||||
offline : bool
|
||||
离线模式
|
||||
"""
|
||||
safe_ps = _filter_protected_packages(packages)
|
||||
if not safe_ps:
|
||||
print("所有指定的包均为受保护包, 跳过重装")
|
||||
return
|
||||
|
||||
subprocess.run(["pip", "uninstall", "-y", *safe_ps], check=True)
|
||||
|
||||
options = ["--no-index", "--find-links", "."] if offline else []
|
||||
subprocess.run(["pip", "install", *options, *safe_ps], check=True)
|
||||
|
||||
|
||||
@px.tool("piptool", subcommand="d", help="下载包")
|
||||
def pip_download(packages: list[str], offline: bool = False) -> None:
|
||||
"""下载包.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
packages : list[str]
|
||||
包名列表
|
||||
offline : bool
|
||||
离线模式
|
||||
"""
|
||||
options = ["--no-index", "--find-links", "."] if offline else []
|
||||
subprocess.run(
|
||||
["pip", "download", *packages, *options, "-d", PACKAGE_DIR],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
@px.tool("piptool", subcommand="up", help="升级 pip")
|
||||
def pip_upgrade() -> None:
|
||||
"""升级 pip 到最新版本."""
|
||||
subprocess.run(["python", "-m", "pip", "install", "--upgrade", "pip"], check=True)
|
||||
print("pip 升级完成")
|
||||
|
||||
|
||||
@px.tool("piptool", subcommand="f", help="导出依赖")
|
||||
def pip_freeze() -> None:
|
||||
"""冻结依赖到 requirements.txt."""
|
||||
result = subprocess.run(
|
||||
["pip", "freeze", "--exclude-editable"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
Path(REQUIREMENTS_FILE).write_text(result.stdout)
|
||||
print(f"依赖已导出到 {REQUIREMENTS_FILE}")
|
||||
@@ -0,0 +1,248 @@
|
||||
"""pymake - 项目构建工具.
|
||||
|
||||
提供构建/测试/清理/文档/格式化/发布等子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = [
|
||||
"b",
|
||||
"ba",
|
||||
"bc",
|
||||
"bump",
|
||||
"bumpmi",
|
||||
"bumpversion",
|
||||
"c",
|
||||
"cov",
|
||||
"doc",
|
||||
"git_add_all",
|
||||
"git_push",
|
||||
"git_push_tags",
|
||||
"lint",
|
||||
"p",
|
||||
"pb",
|
||||
"publish_python",
|
||||
"pyrefly_check",
|
||||
"sync",
|
||||
"t",
|
||||
"tc",
|
||||
"test_coverage",
|
||||
"tf",
|
||||
"tox",
|
||||
"twine_publish",
|
||||
]
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 单任务别名 (cmd 任务)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="b", help="构建 Python 主包 (uv build)", cmd=["uv", "build"])
|
||||
def b(cwd: Path = Path()) -> None:
|
||||
"""构建 Python 主包."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="bc", help="构建 Rust 核心模块 (maturin build)", cmd=["maturin", "build", "-r"])
|
||||
def bc(cwd: Path = Path()) -> None:
|
||||
"""构建 Rust 核心模块."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="sync", help="同步依赖 (uv sync)", cmd=["uv", "sync"])
|
||||
def sync(cwd: Path = Path()) -> None:
|
||||
"""同步依赖."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="c", help="清理构建产物 (调用 gitt c)", cmd=["pf", "gitt", "c"])
|
||||
def c(cwd: Path = Path()) -> None:
|
||||
"""清理构建产物."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="t",
|
||||
help="运行测试",
|
||||
cmd=["pytest", "-m", "not slow", "-n", "8", "--dist", "loadfile", "--color=yes", "--durations=10"],
|
||||
)
|
||||
def t(cwd: Path = Path()) -> None:
|
||||
"""运行测试 (含并行)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="tf",
|
||||
help="快速测试 (无 slow)",
|
||||
cmd=["pytest", "-m", "not slow", "--dist", "loadfile", "--color=yes", "--durations=10"],
|
||||
)
|
||||
def tf(cwd: Path = Path()) -> None:
|
||||
"""快速测试 (无 slow, 无并行)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="bumpmi",
|
||||
help="升级次版本号 (bumpversion minor)",
|
||||
cmd=["pf", "bumpversion", "--part", "minor"],
|
||||
)
|
||||
def bumpmi(cwd: Path = Path()) -> None:
|
||||
"""升级次版本号."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="doc",
|
||||
help="构建 Sphinx 文档",
|
||||
cmd=["sphinx-build", "-b", "html", "docs", "docs/_build"],
|
||||
)
|
||||
def doc(cwd: Path = Path()) -> None:
|
||||
"""构建 Sphinx 文档."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="lint", help="代码格式化与检查 (ruff)", cmd=["ruff", "check", "--fix", "--unsafe-fixes"])
|
||||
def lint(cwd: Path = Path()) -> None:
|
||||
"""代码格式化与检查."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="tox", help="多版本测试 (tox)", cmd=["tox", "-p", "auto"])
|
||||
def tox(cwd: Path = Path()) -> None:
|
||||
"""多版本测试."""
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 内部 job (hidden, 不暴露为 subcommand)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="bumpversion",
|
||||
help="升级版本号 (patch)",
|
||||
cmd=["pf", "bumpversion", "--part", "patch"],
|
||||
needs=["git_add_all"],
|
||||
hidden=True,
|
||||
)
|
||||
def bumpversion(cwd: Path = Path()) -> None:
|
||||
"""升级版本号 (patch, 内部 job)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="test_coverage",
|
||||
help="测试并生成覆盖率",
|
||||
cmd=["pytest", "--cov", "-n", "8", "--dist", "loadfile", "--tb=short", "-v", "--color=yes", "--durations=10"],
|
||||
needs=["c"],
|
||||
hidden=True,
|
||||
)
|
||||
def test_coverage(cwd: Path = Path()) -> None:
|
||||
"""测试并生成覆盖率 (内部 job)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="pyrefly_check",
|
||||
help="pyrefly 类型检查",
|
||||
cmd=["pyrefly", "check", "."],
|
||||
hidden=True,
|
||||
)
|
||||
def pyrefly_check(cwd: Path = Path()) -> None:
|
||||
"""pyrefly 类型检查 (内部 job)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="git_add_all",
|
||||
help="git add -A",
|
||||
cmd=["git", "add", "-A"],
|
||||
needs=["tc"],
|
||||
hidden=True,
|
||||
)
|
||||
def git_add_all(cwd: Path = Path()) -> None:
|
||||
"""git add -A (内部 job)."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="git_push", help="git push", cmd=["git", "push"], hidden=True)
|
||||
def git_push(cwd: Path = Path()) -> None:
|
||||
"""git push (内部 job)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="git_push_tags",
|
||||
help="git push --tags",
|
||||
cmd=["git", "push", "--tags"],
|
||||
hidden=True,
|
||||
)
|
||||
def git_push_tags(cwd: Path = Path()) -> None:
|
||||
"""git push --tags (内部 job)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="twine_publish",
|
||||
help="twine upload",
|
||||
cmd=["twine", "upload", "--disable-progress-bar"],
|
||||
hidden=True,
|
||||
)
|
||||
def twine_publish(cwd: Path = Path()) -> None:
|
||||
"""twine upload (内部 job)."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="publish_python", help="hatch publish", cmd=["hatch", "publish"], hidden=True)
|
||||
def publish_python(cwd: Path = Path()) -> None:
|
||||
"""hatch publish (内部 job)."""
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 聚合 job (有 needs 无 cmd)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="ba", help="构建所有包 (Python + Rust)", needs=["b", "bc"], strategy="thread")
|
||||
def ba(cwd: Path = Path()) -> None:
|
||||
"""构建所有包 (Python + Rust)."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="bump", help="升级版本号 (清理 + 检查 + add + bumpversion)", needs=["bumpversion"])
|
||||
def bump(cwd: Path = Path()) -> None:
|
||||
"""升级版本号 (聚合)."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="cov", help="测试并生成覆盖率", needs=["test_coverage"])
|
||||
def cov(cwd: Path = Path()) -> None:
|
||||
"""测试并生成覆盖率 (聚合)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="tc",
|
||||
help="类型检查 (pyrefly + ruff)",
|
||||
needs=["c", "pyrefly_check", "lint"],
|
||||
strategy="thread",
|
||||
)
|
||||
def tc(cwd: Path = Path()) -> None:
|
||||
"""类型检查 (聚合)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="p",
|
||||
help="推送代码 (清理 + push + push tags)",
|
||||
needs=["c", "git_push", "git_push_tags"],
|
||||
strategy="thread",
|
||||
)
|
||||
def p(cwd: Path = Path()) -> None:
|
||||
"""推送代码 (聚合)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="pb",
|
||||
help="发布到 PyPI (twine + hatch)",
|
||||
needs=["twine_publish", "publish_python"],
|
||||
strategy="thread",
|
||||
)
|
||||
def pb(cwd: Path = Path()) -> None:
|
||||
"""发布到 PyPI (聚合)."""
|
||||
@@ -0,0 +1,5 @@
|
||||
"""files 子包 — 文件与文档操作工具."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__: list[str] = []
|
||||
@@ -1,18 +1,26 @@
|
||||
"""文件日期处理工具.
|
||||
"""filedate - 文件日期处理工具.
|
||||
|
||||
自动检测文件名的日期前缀,
|
||||
并根据文件的实际创建或修改时间重命名文件.
|
||||
提供添加日期前缀/清除日期前缀 子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"DATE_PATTERN",
|
||||
"SEP",
|
||||
"add_date_prefix",
|
||||
"get_file_timestamp",
|
||||
"process_file_date",
|
||||
"process_files_date",
|
||||
"remove_date_prefix",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
@@ -22,7 +30,7 @@ SEP = "_"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# 公共函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@@ -69,11 +77,34 @@ def process_file_date(filepath: Path, clear: bool = False) -> None:
|
||||
if clear:
|
||||
remove_date_prefix(filepath)
|
||||
else:
|
||||
# 先移除旧日期前缀,再添加新日期前缀
|
||||
new_path = remove_date_prefix(filepath)
|
||||
add_date_prefix(new_path)
|
||||
|
||||
|
||||
@px.tool("filedate", subcommand="add", help="添加日期前缀")
|
||||
def process_files_date_add(files: list[Path]) -> None:
|
||||
"""添加日期前缀.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
files : list[Path]
|
||||
文件路径列表
|
||||
"""
|
||||
process_files_date(files, clear=False)
|
||||
|
||||
|
||||
@px.tool("filedate", subcommand="clear", help="清除日期前缀")
|
||||
def process_files_date_clear(files: list[Path]) -> None:
|
||||
"""清除日期前缀.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
files : list[Path]
|
||||
文件路径列表
|
||||
"""
|
||||
process_files_date(files, clear=True)
|
||||
|
||||
|
||||
def process_files_date(targets: list[Path], clear: bool = False) -> None:
|
||||
"""批量处理文件日期前缀.
|
||||
|
||||
@@ -87,55 +118,3 @@ def process_files_date(targets: list[Path], clear: bool = False) -> None:
|
||||
for target in targets:
|
||||
if target.exists() and not target.name.startswith("."):
|
||||
process_file_date(target, clear)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""文件日期处理工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="FileDate - 文件日期处理工具",
|
||||
usage="filedate <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 添加日期前缀命令
|
||||
add_parser = subparsers.add_parser("add", help="添加日期前缀")
|
||||
add_parser.add_argument("files", nargs="+", help="文件路径")
|
||||
|
||||
# 清除日期前缀命令
|
||||
clear_parser = subparsers.add_parser("clear", help="清除日期前缀")
|
||||
clear_parser.add_argument("files", nargs="+", help="文件路径")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "add":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"process_files_date",
|
||||
fn=process_files_date,
|
||||
args=([Path(f) for f in args.files],),
|
||||
kwargs={"clear": False},
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "clear":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"process_files_date",
|
||||
fn=process_files_date,
|
||||
args=([Path(f) for f in args.files],),
|
||||
kwargs={"clear": True},
|
||||
)
|
||||
]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,16 +1,22 @@
|
||||
"""文件等级重命名工具.
|
||||
"""filelevel - 文件等级重命名工具.
|
||||
|
||||
根据文件等级配置自动重命名文件,
|
||||
支持多种等级标识和括号格式.
|
||||
提供设置文件等级 子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"BRACKETS",
|
||||
"LEVELS",
|
||||
"process_file_level",
|
||||
"process_files_level",
|
||||
"remove_marks",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
@@ -27,7 +33,7 @@ BRACKETS: tuple[str, str] = (" ([_(【-", " )]_)】")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# 公共函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@@ -69,72 +75,34 @@ def process_file_level(filepath: Path, level: int = 0) -> None:
|
||||
filestem = filepath.stem
|
||||
original_stem = filestem
|
||||
|
||||
# 移除所有等级标记
|
||||
for level_names in LEVELS.values():
|
||||
if level_names:
|
||||
filestem = remove_marks(filestem, level_names.split(","))
|
||||
|
||||
# 移除数字标记
|
||||
for digit in map(str, range(1, 10)):
|
||||
filestem = remove_marks(filestem, [digit])
|
||||
|
||||
# 添加等级标记
|
||||
if level > 0:
|
||||
levelstr = LEVELS.get(str(level), "").split(",")[0]
|
||||
if levelstr:
|
||||
filestem = f"{filestem}({levelstr})"
|
||||
|
||||
# 重命名文件
|
||||
if filestem != original_stem:
|
||||
new_path = filepath.with_name(filestem + filepath.suffix)
|
||||
filepath.rename(new_path)
|
||||
print(f"重命名: {filepath} -> {new_path}")
|
||||
|
||||
|
||||
def process_files_level(targets: list[Path], level: int = 0) -> None:
|
||||
@px.tool("filelevel", subcommand="set", help="设置文件等级")
|
||||
def process_files_level(files: list[Path], level: int = 0) -> None:
|
||||
"""批量处理文件等级标记.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
targets : list[Path]
|
||||
files : list[Path]
|
||||
文件路径列表
|
||||
level : int
|
||||
文件等级 (0-4)
|
||||
"""
|
||||
for target in targets:
|
||||
for target in files:
|
||||
process_file_level(target, level)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""文件等级重命名工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="FileLevel - 文件等级重命名工具",
|
||||
usage="filelevel <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 设置等级命令
|
||||
level_parser = subparsers.add_parser("set", help="设置文件等级")
|
||||
level_parser.add_argument("files", nargs="+", help="文件路径")
|
||||
level_parser.add_argument("--level", type=int, choices=[0, 1, 2, 3, 4], required=True, help="文件等级 (0-4)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "set":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"process_files_level", fn=process_files_level, args=([Path(f) for f in args.files], args.level)
|
||||
)
|
||||
]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,7 +1,6 @@
|
||||
"""文件夹备份工具.
|
||||
"""folderback - 文件夹备份工具.
|
||||
|
||||
备份文件和文件夹为 zip 文件,
|
||||
自动删除超过最大数量的旧备份文件.
|
||||
备份当前文件夹到指定目录, 自动清理旧备份.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -12,8 +11,15 @@ from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"backup_folder",
|
||||
"remove_dump",
|
||||
"zip_target",
|
||||
]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# 公共函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@@ -40,17 +46,18 @@ def zip_target(src: Path, dst: Path, max_zip: int) -> None:
|
||||
print(f"备份完成: {target_path}")
|
||||
|
||||
|
||||
def backup_folder(src: str, dst: str, max_zip: int = 5) -> None:
|
||||
@px.tool("folderback", help="备份文件夹")
|
||||
def backup_folder(src: str = ".", dst: str = "./backup", max_zip: int = 5) -> None:
|
||||
"""备份文件夹.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
src : str
|
||||
源文件夹路径
|
||||
源文件夹路径 (默认: 当前目录)
|
||||
dst : str
|
||||
目标文件夹路径
|
||||
目标文件夹路径 (默认: ./backup)
|
||||
max_zip : int
|
||||
最大备份数量
|
||||
最大备份数量 (默认: 5)
|
||||
"""
|
||||
src_path = Path(src)
|
||||
dst_path = Path(dst)
|
||||
@@ -64,31 +71,3 @@ def backup_folder(src: str, dst: str, max_zip: int = 5) -> None:
|
||||
print(f"创建目标文件夹: {dst_path}")
|
||||
|
||||
zip_target(src_path, dst_path, max_zip)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# TaskSpec 定义
|
||||
# ============================================================================
|
||||
|
||||
folderback_default: px.TaskSpec = px.TaskSpec(
|
||||
"folderback_default",
|
||||
fn=lambda: backup_folder(".", "./backup", 5),
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""文件夹备份工具主函数."""
|
||||
runner = px.CliRunner(
|
||||
strategy="thread",
|
||||
description="FolderBack - 文件夹备份工具",
|
||||
graphs={
|
||||
# 备份当前目录到 ./backup
|
||||
"b": px.Graph.from_specs([folderback_default]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
@@ -1,7 +1,6 @@
|
||||
"""文件夹压缩工具.
|
||||
"""folderzip - 文件夹压缩工具.
|
||||
|
||||
压缩目录下的所有文件/文件夹为 zip 文件,
|
||||
默认压缩当前目录下的所有子文件夹.
|
||||
压缩当前目录下的所有子文件夹为 zip 文件.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,18 +10,25 @@ from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"IGNORE_DIRS",
|
||||
"IGNORE_EXT",
|
||||
"IGNORE_FILES",
|
||||
"archive_folder",
|
||||
"zip_folders",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
IGNORE_DIRS: list[str] = [".git", ".idea", ".vscode", "__pycache__"]
|
||||
IGNORE_FILES: list[str] = [".gitignore"]
|
||||
IGNORE: list[str] = [*IGNORE_DIRS, *IGNORE_FILES]
|
||||
IGNORE_EXT: list[str] = [".zip", ".rar", ".7z", ".tar", ".gz"]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# 公共函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@@ -36,13 +42,14 @@ def archive_folder(folder: Path) -> None:
|
||||
print(f"压缩完成: {folder.name}.zip")
|
||||
|
||||
|
||||
@px.tool("folderzip", help="压缩文件夹")
|
||||
def zip_folders(cwd: str = ".") -> None:
|
||||
"""压缩目录下的所有文件夹.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
cwd : str
|
||||
工作目录
|
||||
工作目录 (默认: 当前目录)
|
||||
"""
|
||||
cwd_path = Path(cwd)
|
||||
if not cwd_path.exists():
|
||||
@@ -55,28 +62,3 @@ def zip_folders(cwd: str = ".") -> None:
|
||||
|
||||
for dir_path in dirs:
|
||||
archive_folder(dir_path)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# TaskSpec 定义
|
||||
# ============================================================================
|
||||
|
||||
folderzip_default: px.TaskSpec = px.TaskSpec("folderzip_default", fn=lambda: zip_folders("."))
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""文件夹压缩工具主函数."""
|
||||
runner = px.CliRunner(
|
||||
strategy="thread",
|
||||
description="FolderZip - 文件夹压缩工具",
|
||||
graphs={
|
||||
# 压缩当前目录下的所有文件夹
|
||||
"z": px.Graph.from_specs([folderzip_default]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
@@ -0,0 +1,520 @@
|
||||
"""imagetool - 图片处理工具集.
|
||||
|
||||
提供 resize/crop/rotate/flip/convert/watermark/compress/info/exif/
|
||||
histogram/colors 子命令, 基于 Pillow 实现.
|
||||
|
||||
依赖 ``pyflowx[office]`` extra 中的 ``pillow``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"image_colors",
|
||||
"image_compress",
|
||||
"image_convert",
|
||||
"image_crop",
|
||||
"image_exif",
|
||||
"image_flip",
|
||||
"image_histogram",
|
||||
"image_info",
|
||||
"image_resize",
|
||||
"image_rotate",
|
||||
"image_watermark",
|
||||
]
|
||||
|
||||
try:
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
HAS_PIL = True
|
||||
except ImportError:
|
||||
HAS_PIL = False
|
||||
|
||||
|
||||
def _require_pil() -> bool:
|
||||
"""Pillow 未安装时打印提示, 返回是否可用."""
|
||||
if not HAS_PIL:
|
||||
print("未安装 Pillow 库, 请安装: pip install pyflowx[office]")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _save_image(img: Any, output: Path, fmt: str | None = None, quality: int = 85) -> None:
|
||||
"""保存图片, 自动处理 JPEG 不支持 alpha 通道的情况."""
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
save_fmt = fmt or output.suffix.lstrip(".").upper()
|
||||
if save_fmt == "JPG":
|
||||
save_fmt = "JPEG"
|
||||
if save_fmt == "JPEG":
|
||||
img = img.convert("RGB")
|
||||
if save_fmt in ("JPEG", "WEBP"):
|
||||
img.save(output, format=save_fmt, quality=quality)
|
||||
else:
|
||||
img.save(output, format=save_fmt)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 基础操作
|
||||
# ---------------------------------------------------------------------- #
|
||||
@px.tool("imagetool", subcommand="r", help="调整尺寸")
|
||||
def image_resize(
|
||||
input_path: Path,
|
||||
output_path: Path,
|
||||
width: int,
|
||||
height: int | None = None,
|
||||
keep_ratio: bool = True,
|
||||
) -> None:
|
||||
"""调整图片尺寸.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入图片
|
||||
output_path : Path
|
||||
输出图片
|
||||
width : int
|
||||
目标宽度
|
||||
height : int | None
|
||||
目标高度 (keep_ratio=True 时仅作上限, 默认 None 表示按宽度等比)
|
||||
keep_ratio : bool
|
||||
是否保持宽高比 (默认 True)
|
||||
"""
|
||||
if not _require_pil():
|
||||
return
|
||||
|
||||
img = Image.open(input_path)
|
||||
if keep_ratio:
|
||||
target_height = height if height is not None else width
|
||||
img.thumbnail((width, target_height))
|
||||
else:
|
||||
if height is None:
|
||||
height = width
|
||||
img = img.resize((width, height))
|
||||
_save_image(img, output_path)
|
||||
print(f"调整尺寸完成: {output_path} ({img.size[0]}x{img.size[1]})")
|
||||
|
||||
|
||||
@px.tool("imagetool", subcommand="c", help="裁剪图片")
|
||||
def image_crop(
|
||||
input_path: Path,
|
||||
output_path: Path,
|
||||
left: int,
|
||||
top: int,
|
||||
right: int,
|
||||
bottom: int,
|
||||
) -> None:
|
||||
"""裁剪图片到指定矩形.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入图片
|
||||
output_path : Path
|
||||
输出图片
|
||||
left, top, right, bottom : int
|
||||
裁剪矩形坐标 (左上为原点)
|
||||
"""
|
||||
if not _require_pil():
|
||||
return
|
||||
|
||||
img = Image.open(input_path)
|
||||
cropped = img.crop((left, top, right, bottom))
|
||||
_save_image(cropped, output_path)
|
||||
print(f"裁剪完成: {output_path} ({right - left}x{bottom - top})")
|
||||
|
||||
|
||||
@px.tool("imagetool", subcommand="ro", help="旋转图片")
|
||||
def image_rotate(
|
||||
input_path: Path,
|
||||
output_path: Path,
|
||||
degrees: float,
|
||||
expand: bool = False,
|
||||
) -> None:
|
||||
"""旋转图片.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入图片
|
||||
output_path : Path
|
||||
输出图片
|
||||
degrees : float
|
||||
旋转角度 (正值逆时针)
|
||||
expand : bool
|
||||
是否扩展画布以容纳整个旋转后的图片 (默认 False)
|
||||
"""
|
||||
if not _require_pil():
|
||||
return
|
||||
|
||||
img = Image.open(input_path)
|
||||
rotated = img.rotate(degrees, expand=expand)
|
||||
_save_image(rotated, output_path)
|
||||
print(f"旋转完成: {output_path} ({degrees}度)")
|
||||
|
||||
|
||||
@px.tool("imagetool", subcommand="fl", help="翻转图片")
|
||||
def image_flip(
|
||||
input_path: Path,
|
||||
output_path: Path,
|
||||
direction: str = "horizontal",
|
||||
) -> None:
|
||||
"""翻转图片.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入图片
|
||||
output_path : Path
|
||||
输出图片
|
||||
direction : str
|
||||
翻转方向: "horizontal" (水平镜像) / "vertical" (垂直镜像)
|
||||
"""
|
||||
if not _require_pil():
|
||||
return
|
||||
|
||||
img = Image.open(input_path)
|
||||
method = Image.Transpose.FLIP_LEFT_RIGHT if direction == "horizontal" else Image.Transpose.FLIP_TOP_BOTTOM
|
||||
flipped = img.transpose(method)
|
||||
_save_image(flipped, output_path)
|
||||
print(f"翻转完成: {output_path} ({direction})")
|
||||
|
||||
|
||||
@px.tool("imagetool", subcommand="cv", help="格式转换")
|
||||
def image_convert(
|
||||
input_path: Path,
|
||||
output_path: Path,
|
||||
format: str | None = None,
|
||||
quality: int = 85,
|
||||
) -> None:
|
||||
"""转换图片格式.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入图片
|
||||
output_path : Path
|
||||
输出图片 (后缀决定格式, 除非 format 显式指定)
|
||||
format : str | None
|
||||
目标格式 (如 "PNG"/"JPEG"/"WEBP"), None 时按 output_path 后缀推断
|
||||
quality : int
|
||||
压缩质量 (1-100, 仅对 JPEG/WEBP 有效)
|
||||
"""
|
||||
if not _require_pil():
|
||||
return
|
||||
|
||||
img = Image.open(input_path)
|
||||
_save_image(img, output_path, fmt=format, quality=quality)
|
||||
actual_fmt = format or output_path.suffix.lstrip(".").upper()
|
||||
print(f"格式转换完成: {output_path} ({actual_fmt})")
|
||||
|
||||
|
||||
@px.tool("imagetool", subcommand="wm", help="添加文字水印")
|
||||
def image_watermark(
|
||||
input_path: Path,
|
||||
output_path: Path,
|
||||
text: str,
|
||||
position: str = "bottom-right",
|
||||
opacity: float = 0.5,
|
||||
font_size: int = 32,
|
||||
) -> None:
|
||||
"""添加文字水印.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入图片
|
||||
output_path : Path
|
||||
输出图片
|
||||
text : str
|
||||
水印文字
|
||||
position : str
|
||||
水印位置: top-left/top-right/bottom-left/bottom-right/center
|
||||
opacity : float
|
||||
不透明度 (0.0-1.0)
|
||||
font_size : int
|
||||
字体大小
|
||||
"""
|
||||
if not _require_pil():
|
||||
return
|
||||
|
||||
img = Image.open(input_path).convert("RGBA")
|
||||
overlay = Image.new("RGBA", img.size, (0, 0, 0, 0))
|
||||
draw = ImageDraw.Draw(overlay)
|
||||
|
||||
font = _load_font(font_size)
|
||||
bbox = draw.textbbox((0, 0), text, font=font)
|
||||
text_w = int(bbox[2] - bbox[0])
|
||||
text_h = int(bbox[3] - bbox[1])
|
||||
margin = 10
|
||||
x, y = _resolve_position(position, img.size, text_w, text_h, margin)
|
||||
|
||||
alpha = int(255 * max(0.0, min(1.0, opacity)))
|
||||
draw.text((x, y), text, font=font, fill=(255, 255, 255, alpha))
|
||||
result = Image.alpha_composite(img, overlay)
|
||||
_save_image(result, output_path)
|
||||
print(f"水印添加完成: {output_path}")
|
||||
|
||||
|
||||
def _load_font(size: int) -> Any:
|
||||
"""加载字体, 优先 truetype, 失败回退默认字体."""
|
||||
candidates = ("DejaVuSans.ttf", "Arial.ttf", "LiberationSans-Regular.ttf")
|
||||
for name in candidates:
|
||||
try:
|
||||
return ImageFont.truetype(name, size)
|
||||
except OSError:
|
||||
continue
|
||||
return ImageFont.load_default()
|
||||
|
||||
|
||||
def _resolve_position(
|
||||
position: str,
|
||||
img_size: tuple[int, int],
|
||||
text_w: int,
|
||||
text_h: int,
|
||||
margin: int,
|
||||
) -> tuple[int, int]:
|
||||
"""根据位置描述符计算水印坐标."""
|
||||
w, h = img_size
|
||||
pos_map = {
|
||||
"top-left": (margin, margin),
|
||||
"top-right": (w - text_w - margin, margin),
|
||||
"bottom-left": (margin, h - text_h - margin),
|
||||
"bottom-right": (w - text_w - margin, h - text_h - margin),
|
||||
"center": ((w - text_w) // 2, (h - text_h) // 2),
|
||||
}
|
||||
return pos_map.get(position, pos_map["bottom-right"])
|
||||
|
||||
|
||||
@px.tool("imagetool", subcommand="cp", help="压缩图片")
|
||||
def image_compress(
|
||||
input_path: Path,
|
||||
output_path: Path,
|
||||
quality: int = 85,
|
||||
) -> None:
|
||||
"""压缩图片 (重新编码以减小体积).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入图片
|
||||
output_path : Path
|
||||
输出图片
|
||||
quality : int
|
||||
压缩质量 (1-100)
|
||||
"""
|
||||
if not _require_pil():
|
||||
return
|
||||
|
||||
img = Image.open(input_path)
|
||||
fmt = input_path.suffix.lstrip(".").upper()
|
||||
_save_image(img, output_path, fmt=fmt, quality=quality)
|
||||
in_size = input_path.stat().st_size
|
||||
out_size = output_path.stat().st_size
|
||||
ratio = (1 - out_size / in_size) * 100 if in_size > 0 else 0.0
|
||||
print(f"压缩完成: {output_path} (原 {in_size}B → 新 {out_size}B, 节省 {ratio:.1f}%)")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 元数据与信息
|
||||
# ---------------------------------------------------------------------- #
|
||||
@px.tool("imagetool", subcommand="i", help="查看图片信息")
|
||||
def image_info(input_path: Path, json: bool = False) -> None:
|
||||
"""打印图片信息 (尺寸/格式/模式/EXIF 摘要).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入图片
|
||||
json : bool
|
||||
是否以 JSON 格式输出 (默认纯文本表格)
|
||||
"""
|
||||
if not _require_pil():
|
||||
return
|
||||
|
||||
img = Image.open(input_path)
|
||||
exif = img.getexif()
|
||||
exif_count = len(exif) if exif else 0
|
||||
|
||||
import json as json_mod
|
||||
|
||||
data = {
|
||||
"path": str(input_path),
|
||||
"format": img.format,
|
||||
"mode": img.mode,
|
||||
"width": img.size[0],
|
||||
"height": img.size[1],
|
||||
"exif_tags": exif_count,
|
||||
}
|
||||
if json:
|
||||
print(json_mod.dumps(data, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"文件: {data['path']}")
|
||||
print(f"格式: {data['format']}")
|
||||
print(f"模式: {data['mode']}")
|
||||
print(f"尺寸: {data['width']}x{data['height']}")
|
||||
print(f"EXIF 标签数: {data['exif_tags']}")
|
||||
|
||||
|
||||
@px.tool("imagetool", subcommand="e", help="读取/修改 EXIF")
|
||||
def image_exif(
|
||||
input_path: Path,
|
||||
output_path: Path | None = None,
|
||||
show: bool = True,
|
||||
set: list[str] | None = None,
|
||||
clear: bool = False,
|
||||
) -> None:
|
||||
"""读取或修改 EXIF 元数据.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入图片
|
||||
output_path : Path | None
|
||||
输出路径 (None 时原地覆盖; 仅 show=True 时可省略)
|
||||
show : bool
|
||||
打印全部 EXIF 标签 (默认 True)
|
||||
set : list[str] | None
|
||||
设置标签, 格式 ["KEY=VALUE", ...] (KEY 为数字标签号)
|
||||
clear : bool
|
||||
清空所有 EXIF 标签 (在 set 之前执行)
|
||||
"""
|
||||
if not _require_pil():
|
||||
return
|
||||
|
||||
img = Image.open(input_path)
|
||||
exif = img.getexif()
|
||||
|
||||
if show:
|
||||
_print_exif(exif)
|
||||
|
||||
modified = _apply_exif_modifications(exif, set, clear)
|
||||
if modified:
|
||||
_save_exif(img, exif, output_path if output_path is not None else input_path)
|
||||
|
||||
|
||||
def _print_exif(exif: Any) -> None:
|
||||
"""打印 EXIF 标签."""
|
||||
if exif:
|
||||
for tag, value in exif.items():
|
||||
print(f" {tag}: {value}")
|
||||
else:
|
||||
print(" (无 EXIF 数据)")
|
||||
|
||||
|
||||
def _apply_exif_modifications(exif: Any, set_items: list[str] | None, clear: bool) -> bool:
|
||||
"""应用 EXIF 修改 (clear + set), 返回是否有改动."""
|
||||
if clear:
|
||||
for tag in list(exif.keys()):
|
||||
del exif[tag]
|
||||
if set_items:
|
||||
for item in set_items:
|
||||
_apply_single_exif_set(exif, item)
|
||||
return bool(set_items or clear)
|
||||
|
||||
|
||||
def _apply_single_exif_set(exif: Any, item: str) -> None:
|
||||
"""解析并应用单个 KEY=VALUE 设置项."""
|
||||
if "=" not in item:
|
||||
print(f"跳过无效项 (缺少 =): {item}")
|
||||
return
|
||||
key_str, value = item.split("=", 1)
|
||||
try:
|
||||
tag = int(key_str)
|
||||
except ValueError:
|
||||
print(f"跳过无效标签号: {key_str}")
|
||||
return
|
||||
exif[tag] = value
|
||||
|
||||
|
||||
def _save_exif(img: Any, exif: Any, output_path: Path) -> None:
|
||||
"""保存图片与 EXIF."""
|
||||
exif_bytes = exif.tobytes() if exif else b""
|
||||
img.save(output_path, exif=exif_bytes)
|
||||
print(f"EXIF 已保存: {output_path}")
|
||||
|
||||
|
||||
@px.tool("imagetool", subcommand="hi", help="颜色直方图")
|
||||
def image_histogram(
|
||||
input_path: Path,
|
||||
channel: str = "rgb",
|
||||
) -> None:
|
||||
"""打印颜色直方图统计 (每通道 8 桶).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入图片
|
||||
channel : str
|
||||
通道: "rgb" (R/G/B 三通道) / "luminance" (亮度单通道)
|
||||
"""
|
||||
if not _require_pil():
|
||||
return
|
||||
|
||||
img = Image.open(input_path)
|
||||
hist = img.histogram()
|
||||
buckets = 8
|
||||
|
||||
if channel == "luminance":
|
||||
gray = img.convert("L")
|
||||
gray_hist = gray.histogram()
|
||||
print("亮度直方图 (8 桶):")
|
||||
_print_histogram_buckets(gray_hist, buckets, "L")
|
||||
else:
|
||||
print("RGB 直方图 (8 桶):")
|
||||
if len(hist) == 256:
|
||||
_print_histogram_buckets(hist, buckets, "L")
|
||||
else:
|
||||
for idx, name in enumerate(("R", "G", "B")):
|
||||
start = idx * 256
|
||||
_print_histogram_buckets(hist[start : start + 256], buckets, name)
|
||||
|
||||
|
||||
def _print_histogram_buckets(channel_hist: list[int], buckets: int, name: str) -> None:
|
||||
"""将单通道 256 桶直方图聚合为指定桶数并打印."""
|
||||
bucket_size = max(1, len(channel_hist) // buckets)
|
||||
print(f" {name}:")
|
||||
for i in range(buckets):
|
||||
start = i * bucket_size
|
||||
end = min((i + 1) * bucket_size, len(channel_hist))
|
||||
count = sum(channel_hist[start:end])
|
||||
slice_max = max(channel_hist[start:end] or [1])
|
||||
bar = "#" * min(40, count * 40 // max(1, slice_max))
|
||||
print(f" [{start:3d}-{end:3d}] {count:>8d} {bar}")
|
||||
|
||||
|
||||
@px.tool("imagetool", subcommand="co", help="提取主色调")
|
||||
def image_colors(
|
||||
input_path: Path,
|
||||
count: int = 5,
|
||||
) -> None:
|
||||
"""提取并打印主色调.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入图片
|
||||
count : int
|
||||
提取的颜色数 (默认 5)
|
||||
"""
|
||||
if not _require_pil():
|
||||
return
|
||||
|
||||
img = Image.open(input_path).convert("RGB")
|
||||
quantized = img.quantize(colors=count)
|
||||
palette = quantized.getpalette()
|
||||
if palette is None:
|
||||
print("无法提取调色板")
|
||||
return
|
||||
|
||||
actual_count = len(palette) // 3
|
||||
print(f"主色调 (前 {min(count, actual_count)} 色):")
|
||||
for i in range(min(count, actual_count)):
|
||||
r = palette[i * 3]
|
||||
g = palette[i * 3 + 1]
|
||||
b = palette[i * 3 + 2]
|
||||
hex_color = f"#{r:02X}{g:02X}{b:02X}"
|
||||
print(f" {i + 1}. {hex_color} rgb({r}, {g}, {b})")
|
||||
@@ -0,0 +1,512 @@
|
||||
"""pdftool - PDF 文件工具集.
|
||||
|
||||
提供 PDF 合并/拆分/压缩/加密/解密/提取文本/提取图片/水印/旋转/裁剪/
|
||||
信息/OCR/转图片/修复 等子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"pdf_add_watermark",
|
||||
"pdf_compress",
|
||||
"pdf_crop",
|
||||
"pdf_decrypt",
|
||||
"pdf_encrypt",
|
||||
"pdf_extract_images",
|
||||
"pdf_extract_text",
|
||||
"pdf_info",
|
||||
"pdf_merge",
|
||||
"pdf_ocr",
|
||||
"pdf_reorder",
|
||||
"pdf_repair",
|
||||
"pdf_rotate",
|
||||
"pdf_split",
|
||||
"pdf_to_images",
|
||||
]
|
||||
|
||||
try:
|
||||
import fitz # PyMuPDF
|
||||
|
||||
HAS_PYMUPDF = True
|
||||
except ImportError:
|
||||
HAS_PYMUPDF = False
|
||||
|
||||
try:
|
||||
import pypdf
|
||||
|
||||
HAS_PYPDF = True
|
||||
except ImportError:
|
||||
HAS_PYPDF = False
|
||||
|
||||
|
||||
def _require_pymupdf() -> bool:
|
||||
"""PyMuPDF 未安装时打印提示, 返回是否可用."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _require_pypdf() -> bool:
|
||||
"""pypdf 未安装时打印提示, 返回是否可用."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库, 请安装: pip install pypdf")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="m", help="合并 PDF")
|
||||
def pdf_merge(input_paths: list[Path], output_path: Path = Path("merged.pdf")) -> None:
|
||||
"""合并多个 PDF 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_paths : list[Path]
|
||||
输入 PDF 文件列表
|
||||
output_path : Path
|
||||
输出文件 (默认: merged.pdf)
|
||||
"""
|
||||
if not _require_pypdf():
|
||||
return
|
||||
|
||||
writer = pypdf.PdfWriter()
|
||||
for input_path in input_paths:
|
||||
if input_path.exists():
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"合并完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="s", help="拆分 PDF")
|
||||
def pdf_split(input_path: Path, output_dir: Path = Path("split")) -> None:
|
||||
"""拆分 PDF 文件为单页.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_dir : Path
|
||||
输出目录 (默认: split)
|
||||
"""
|
||||
if not _require_pypdf():
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for i, page in enumerate(reader.pages):
|
||||
writer = pypdf.PdfWriter()
|
||||
writer.add_page(page)
|
||||
output_file = output_dir / f"{input_path.stem}_page_{i + 1}.pdf"
|
||||
with open(output_file, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"拆分完成: {output_dir}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="c", help="压缩 PDF")
|
||||
def pdf_compress(input_path: Path, output_path: Path = Path("compressed.pdf")) -> None:
|
||||
"""压缩 PDF 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: compressed.pdf)
|
||||
"""
|
||||
if not _require_pymupdf():
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path), garbage=4, deflate=True, clean=True)
|
||||
doc.close()
|
||||
|
||||
original_size = input_path.stat().st_size
|
||||
new_size = output_path.stat().st_size
|
||||
ratio = (1 - new_size / original_size) * 100
|
||||
print(f"压缩完成: {output_path} (缩小 {ratio:.1f}%)")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="e", help="加密 PDF")
|
||||
def pdf_encrypt(input_path: Path, output_path: Path = Path("encrypted.pdf"), password: str = "") -> None:
|
||||
"""加密 PDF 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: encrypted.pdf)
|
||||
password : str
|
||||
密码 (必填)
|
||||
"""
|
||||
if not password:
|
||||
print("错误: --password 为必填参数")
|
||||
return
|
||||
if not _require_pypdf():
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
writer = pypdf.PdfWriter()
|
||||
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
writer.encrypt(user_password=password, owner_password=password, use_128bit=True)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"加密完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="d", help="解密 PDF")
|
||||
def pdf_decrypt(input_path: Path, output_path: Path = Path("decrypted.pdf"), password: str = "") -> None:
|
||||
"""解密 PDF 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: decrypted.pdf)
|
||||
password : str
|
||||
密码 (必填)
|
||||
"""
|
||||
if not password:
|
||||
print("错误: --password 为必填参数")
|
||||
return
|
||||
if not _require_pypdf():
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
if reader.is_encrypted:
|
||||
reader.decrypt(password)
|
||||
|
||||
writer = pypdf.PdfWriter()
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"解密完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="xt", help="提取文本")
|
||||
def pdf_extract_text(input_path: Path, output_path: Path = Path("output.txt")) -> None:
|
||||
"""提取 PDF 文本.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: output.txt)
|
||||
"""
|
||||
if not _require_pymupdf():
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
text = ""
|
||||
for page in doc:
|
||||
text += str(page.get_text()) + "\n\n"
|
||||
doc.close()
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(text, encoding="utf-8")
|
||||
print(f"文本提取完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="xi", help="提取图片")
|
||||
def pdf_extract_images(input_path: Path, output_dir: Path = Path("images")) -> None:
|
||||
"""提取 PDF 图片.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_dir : Path
|
||||
输出目录 (默认: images)
|
||||
"""
|
||||
if not _require_pymupdf():
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
image_count = 0
|
||||
# pyrefly: ignore [bad-argument-type]
|
||||
for page_num, page in enumerate(doc):
|
||||
images = page.get_images(full=True)
|
||||
for img_idx, img in enumerate(images):
|
||||
xref = img[0]
|
||||
base_image = doc.extract_image(xref)
|
||||
image_data = base_image["image"]
|
||||
image_ext = base_image["ext"]
|
||||
image_path = output_dir / f"page_{page_num + 1}_img_{img_idx + 1}.{image_ext}"
|
||||
image_path.write_bytes(image_data)
|
||||
image_count += 1
|
||||
|
||||
doc.close()
|
||||
print(f"图片提取完成: {output_dir} (共 {image_count} 张)")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="w", help="添加水印")
|
||||
def pdf_add_watermark(
|
||||
input_path: Path, output_path: Path = Path("watermarked.pdf"), text: str = "CONFIDENTIAL"
|
||||
) -> None:
|
||||
"""添加 PDF 水印.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: watermarked.pdf)
|
||||
text : str
|
||||
水印文字 (默认: CONFIDENTIAL)
|
||||
"""
|
||||
if not _require_pymupdf():
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
for page in doc:
|
||||
rect = page.rect
|
||||
text_width = fitz.get_text_length(text, fontsize=48)
|
||||
x = (rect.width - text_width) / 2
|
||||
y = rect.height / 2
|
||||
page.insert_text((x, y), text, fontsize=48, rotate=45, color=(0, 0, 0))
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"水印添加完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="r", help="旋转 PDF")
|
||||
def pdf_rotate(input_path: Path, output_path: Path = Path("rotated.pdf"), rotation: int = 90) -> None:
|
||||
"""旋转 PDF 页面.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: rotated.pdf)
|
||||
rotation : int
|
||||
旋转角度 (默认: 90)
|
||||
"""
|
||||
if not _require_pymupdf():
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
for page in doc:
|
||||
page.set_rotation(rotation)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"旋转完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="crop", help="裁剪 PDF")
|
||||
def pdf_crop(
|
||||
input_path: Path,
|
||||
output_path: Path = Path("cropped.pdf"),
|
||||
margins: tuple[int, int, int, int] = (10, 10, 10, 10),
|
||||
) -> None:
|
||||
"""裁剪 PDF 页面.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: cropped.pdf)
|
||||
margins : tuple[int, int, int, int]
|
||||
边距 (左, 上, 右, 下), 默认 (10, 10, 10, 10)
|
||||
"""
|
||||
if not _require_pymupdf():
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
left, top, right, bottom = margins
|
||||
|
||||
for page in doc:
|
||||
rect = page.rect
|
||||
new_rect = fitz.Rect(
|
||||
rect.x0 + left,
|
||||
rect.y0 + top,
|
||||
rect.x1 - right,
|
||||
rect.y1 - bottom,
|
||||
)
|
||||
page.set_cropbox(new_rect)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"裁剪完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="i", help="查看 PDF 信息")
|
||||
def pdf_info(input_path: Path) -> None:
|
||||
"""显示 PDF 信息.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
"""
|
||||
if not _require_pymupdf():
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
print(f"文件: {input_path}")
|
||||
print(f"页数: {doc.page_count}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"标题: {doc.metadata.get('title', 'N/A')}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"作者: {doc.metadata.get('author', 'N/A')}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"创建日期: {doc.metadata.get('creationDate', 'N/A')}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"修改日期: {doc.metadata.get('modDate', 'N/A')}")
|
||||
print(f"文件大小: {input_path.stat().st_size / 1024:.1f} KB")
|
||||
doc.close()
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="ocr", help="PDF OCR 识别")
|
||||
def pdf_ocr(input_path: Path, output_path: Path = Path("ocr.pdf"), lang: str = "chi_sim+eng") -> None:
|
||||
"""PDF OCR 识别.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: ocr.pdf)
|
||||
lang : str
|
||||
识别语言 (默认: chi_sim+eng)
|
||||
"""
|
||||
try:
|
||||
import pytesseract
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
print("未安装 OCR 相关库, 请安装: pip install pytesseract pillow")
|
||||
return
|
||||
|
||||
if not _require_pymupdf():
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
new_doc = fitz.open()
|
||||
|
||||
for page in doc:
|
||||
pix = page.get_pixmap()
|
||||
img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
|
||||
ocr_text = pytesseract.image_to_string(img, lang=lang)
|
||||
|
||||
new_page = new_doc.new_page(width=page.rect.width, height=page.rect.height)
|
||||
new_page.insert_image(new_page.rect, pixmap=pix)
|
||||
text_rect = fitz.Rect(0, 0, page.rect.width, page.rect.height)
|
||||
# pyrefly: ignore [bad-argument-type]
|
||||
new_page.insert_textbox(text_rect, ocr_text, fontname="china-ss", fontsize=11)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
new_doc.save(str(output_path))
|
||||
new_doc.close()
|
||||
doc.close()
|
||||
print(f"OCR 识别完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_reorder(input_path: Path, output_path: Path, order: list[int]) -> None:
|
||||
"""重排 PDF 页面顺序.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件
|
||||
order : list[int]
|
||||
页面顺序列表 (0-based)
|
||||
"""
|
||||
if not _require_pypdf():
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
writer = pypdf.PdfWriter()
|
||||
|
||||
for page_num in order:
|
||||
if 0 <= page_num < len(reader.pages):
|
||||
writer.add_page(reader.pages[page_num])
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"重排完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="img", help="PDF 转图片")
|
||||
def pdf_to_images(input_path: Path, output_dir: Path = Path("images"), dpi: int = 300) -> None:
|
||||
"""PDF 转图片.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_dir : Path
|
||||
输出目录 (默认: images)
|
||||
dpi : int
|
||||
DPI (默认: 300)
|
||||
"""
|
||||
if not _require_pymupdf():
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# pyrefly: ignore [bad-argument-type]
|
||||
for page_num, page in enumerate(doc):
|
||||
pix = page.get_pixmap(dpi=dpi)
|
||||
image_path = output_dir / f"{input_path.stem}_page_{page_num + 1}.png"
|
||||
pix.save(str(image_path))
|
||||
|
||||
doc.close()
|
||||
print(f"转换完成: {output_dir}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="repair", help="修复 PDF")
|
||||
def pdf_repair(input_path: Path, output_path: Path = Path("repaired.pdf")) -> None:
|
||||
"""修复 PDF 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: repaired.pdf)
|
||||
"""
|
||||
if not _require_pymupdf():
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path), garbage=4, deflate=True, clean=True)
|
||||
doc.close()
|
||||
print(f"修复完成: {output_path}")
|
||||
@@ -1,11 +1,10 @@
|
||||
"""截图工具.
|
||||
"""screenshot - 截图工具.
|
||||
|
||||
跨平台截图工具, 支持全屏截图和区域截图.
|
||||
跨平台截图: Windows 用 PowerShell, macOS 用 screencapture, Linux 用 gnome-screenshot/scrot.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -13,10 +12,6 @@ from pathlib import Path
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def get_screenshot_path(filename: str | None = None) -> Path:
|
||||
"""获取截图保存路径.
|
||||
@@ -40,6 +35,7 @@ def get_screenshot_path(filename: str | None = None) -> Path:
|
||||
return screenshots_dir / filename
|
||||
|
||||
|
||||
@px.tool("screenshot", subcommand="full", help="全屏截图")
|
||||
def take_screenshot_full(filename: str | None = None) -> None:
|
||||
"""全屏截图.
|
||||
|
||||
@@ -51,7 +47,6 @@ def take_screenshot_full(filename: str | None = None) -> None:
|
||||
output_path = get_screenshot_path(filename)
|
||||
|
||||
if Constants.IS_WINDOWS:
|
||||
# Windows: 使用 PowerShell 截图
|
||||
ps_script = f"""
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
@@ -66,10 +61,8 @@ $bitmap.Dispose()
|
||||
"""
|
||||
subprocess.run(["powershell", "-Command", ps_script], check=True)
|
||||
elif Constants.IS_MACOS:
|
||||
# macOS: 使用 screencapture
|
||||
subprocess.run(["screencapture", "-x", str(output_path)], check=True)
|
||||
else:
|
||||
# Linux: 使用 gnome-screenshot 或 scrot
|
||||
try:
|
||||
subprocess.run(["gnome-screenshot", "-f", str(output_path)], check=True)
|
||||
except FileNotFoundError:
|
||||
@@ -78,6 +71,7 @@ $bitmap.Dispose()
|
||||
print(f"截图已保存: {output_path}")
|
||||
|
||||
|
||||
@px.tool("screenshot", subcommand="area", help="区域截图")
|
||||
def take_screenshot_area(filename: str | None = None) -> None:
|
||||
"""区域截图.
|
||||
|
||||
@@ -89,7 +83,6 @@ def take_screenshot_area(filename: str | None = None) -> None:
|
||||
output_path = get_screenshot_path(filename)
|
||||
|
||||
if Constants.IS_WINDOWS:
|
||||
# Windows: 使用 PowerShell 截图 (需要用户选择区域)
|
||||
ps_script = f"""
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
@@ -113,51 +106,11 @@ $bitmap.Dispose()
|
||||
"""
|
||||
subprocess.run(["powershell", "-Command", ps_script], check=True)
|
||||
elif Constants.IS_MACOS:
|
||||
# macOS: 使用 screencapture 交互模式
|
||||
subprocess.run(["screencapture", "-i", str(output_path)], check=True)
|
||||
else:
|
||||
# Linux: 使用 gnome-screenshot 交互模式
|
||||
try:
|
||||
subprocess.run(["gnome-screenshot", "-a", "-f", str(output_path)], check=True)
|
||||
except FileNotFoundError:
|
||||
subprocess.run(["scrot", "-s", str(output_path)], check=True)
|
||||
|
||||
print(f"截图已保存: {output_path}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""截图工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Screenshot - 截图工具",
|
||||
usage="screenshot <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 全屏截图命令
|
||||
full_parser = subparsers.add_parser("full", help="全屏截图")
|
||||
full_parser.add_argument("--filename", type=str, help="文件名")
|
||||
|
||||
# 区域截图命令
|
||||
area_parser = subparsers.add_parser("area", help="区域截图")
|
||||
area_parser.add_argument("--filename", type=str, help="文件名")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "full":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("screenshot_full", fn=take_screenshot_full, kwargs={"filename": args.filename})]
|
||||
)
|
||||
elif args.command == "area":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("screenshot_area", fn=take_screenshot_area, kwargs={"filename": args.filename})]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,5 @@
|
||||
"""infra 子包 — 基础设施与服务部署工具."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__: list[str] = []
|
||||
@@ -0,0 +1,37 @@
|
||||
"""dockercmd - Docker 镜像登录工具.
|
||||
|
||||
登录腾讯云 Docker 镜像仓库.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import getpass
|
||||
import subprocess
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
_DOCKER_MIRROR_TENCENT: str = "ccr.ccs.tencentyun.com"
|
||||
|
||||
|
||||
@px.tool("dockercmd", subcommand="login", help="登录腾讯云 Docker 镜像仓库")
|
||||
def docker_login_tencent(username: str = "") -> None:
|
||||
"""登录腾讯云 Docker 镜像仓库.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
username : str
|
||||
Docker 用户名 (为空时由 docker 交互式提示输入)
|
||||
"""
|
||||
user = username or getpass.getuser()
|
||||
try:
|
||||
subprocess.run(
|
||||
["docker", "login", "--username", user, _DOCKER_MIRROR_TENCENT],
|
||||
check=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"docker login 失败 (returncode={e.returncode}): {e}")
|
||||
return
|
||||
except FileNotFoundError:
|
||||
print("docker 命令未找到,请确认 Docker 已安装并在 PATH 中")
|
||||
return
|
||||
print(f"已登录腾讯云镜像仓库 (用户: {user})")
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user