Compare commits
128 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c9c7529c58 | |||
| c498d9b1c9 | |||
| b36e279f92 | |||
| 58d6f1faad | |||
| d93da0d8b4 | |||
| 701c455c42 | |||
| e174b64495 | |||
| 3afb25bb5e | |||
| fbd17536fd | |||
| 32ca8c1208 | |||
| a7ff68d279 | |||
| de368ea810 | |||
| 6a3e3a57cd | |||
| 7089944306 | |||
| ec5e348694 | |||
| 12d9f2f647 | |||
| 6ffcbecade | |||
| e76d93187b | |||
| 52e20e3f93 | |||
| 3f966a230e | |||
| 5d0b211a44 | |||
| 6931f36fd1 | |||
| db02443463 | |||
| eb8e1402bc | |||
| c93f45dcb8 | |||
| a0b1814024 | |||
| 3a2826d3f9 | |||
| dbd30689ab | |||
| 5eb59b8a66 | |||
| 8e7b866de2 | |||
| 1b4f9bfa6a | |||
| 2d39272330 | |||
| f699bb9167 | |||
| 35f07e96e1 | |||
| 1f274fe828 | |||
| 85793ff9d5 | |||
| 37ac4b8025 | |||
| 0edeadb846 | |||
| f63db6c71a | |||
| 4d397606e6 | |||
| f24388b151 | |||
| 535b7cba31 | |||
| 3f68bed3fd | |||
| 2e2ca812a1 | |||
| 8de565d0cb | |||
| 5480c48e67 | |||
| c6653d5117 | |||
| d194a991a0 | |||
| 4446658170 | |||
| 1d26f9d3e7 | |||
| d9644ca5d1 | |||
| d3c2d53449 | |||
| 9cfcfb38e4 | |||
| 69db241611 | |||
| 66e6295a24 | |||
| aebb4fce68 | |||
| 7784c8ff86 | |||
| 77918a5568 | |||
| 7e4c615dc7 | |||
| ac5082523e | |||
| 0df6f7c8ac | |||
| 4b66176ce6 | |||
| cf6b6fd059 | |||
| 6f93e6eb6d | |||
| 43e1aad1fe | |||
| 467634f8c7 | |||
| ce31f60441 | |||
| 3d6d769685 | |||
| 3f9c52e6f1 | |||
| 8fadf6edd8 | |||
| abc1152538 | |||
| 5e561b4b3a | |||
| 40f641611b | |||
| 232e7293d9 | |||
| a1bae58e56 | |||
| cbc7cc0a75 | |||
| d0ff7d7b4d | |||
| d154f67ce0 | |||
| 9999071119 | |||
| bdd70e9c43 | |||
| c15b38516a | |||
| 7d4e8a40ce | |||
| 1b2d6d6a2c | |||
| df890f0f16 | |||
| b62a544569 | |||
| d58fc5536e | |||
| c3b86b603d | |||
| 327bd6e069 | |||
| 22f8d2110d | |||
| 2a1f2f7175 | |||
| 9d033e1c0b | |||
| 336f7b7292 | |||
| 65dcbcbf62 | |||
| 7fa97a01e3 | |||
| 83da5135d0 | |||
| 7463a60649 | |||
| 87dd010342 | |||
| bdfee7bee4 | |||
| b954fb1622 | |||
| a7b7a82dff | |||
| 40f0478146 | |||
| b808b880f8 | |||
| e073ff41ee | |||
| ea0c51de5e | |||
| 2b3f4b82d3 | |||
| 1e23c48efc | |||
| 5c8ec281ff | |||
| 6f01cde8ac | |||
| bcd189ae60 | |||
| 20c4fb87c5 | |||
| a98eb6e344 | |||
| 752ff618b2 | |||
| f15f235ecf | |||
| 9d79cddbd6 | |||
| af9aab395a | |||
| 6f334fde73 | |||
| 2ccd84ac3b | |||
| ec30af3edb | |||
| 10bbc07118 | |||
| 194cf3c343 | |||
| 1880cd7a34 | |||
| d43c9e4044 | |||
| 22ac9fc4dd | |||
| 7ded8df05e | |||
| fd282db28f | |||
| 6f64d9d6dc | |||
| a2889fbb08 | |||
| 024b597e44 |
@@ -0,0 +1,47 @@
|
||||
# 版本控制
|
||||
.git
|
||||
.gitignore
|
||||
.github
|
||||
|
||||
# Python 缓存与构建产物
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
*.egg-info
|
||||
*.egg
|
||||
dist
|
||||
build
|
||||
.eggs
|
||||
|
||||
# 测试与覆盖率
|
||||
.pytest_cache
|
||||
.coverage
|
||||
htmlcov
|
||||
.tox
|
||||
coverage.xml
|
||||
|
||||
# 虚拟环境
|
||||
.venv
|
||||
venv
|
||||
env
|
||||
|
||||
# 工具缓存
|
||||
.uv-cache
|
||||
.ruff_cache
|
||||
.pyrefly_cache
|
||||
.mypy_cache
|
||||
|
||||
# IDE 与编辑器
|
||||
.idea
|
||||
.vscode
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# 文档与示例(按需保留)
|
||||
docs
|
||||
examples
|
||||
|
||||
# 系统文件
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
+18
-113
@@ -3,127 +3,32 @@ name: CI
|
||||
on:
|
||||
push:
|
||||
branches: [ main, develop ]
|
||||
pull_request:
|
||||
branches: [ main, develop ]
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# lint:代码风格与格式检查(单平台即可)
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
lint:
|
||||
name: Lint (ruff)
|
||||
ci:
|
||||
name: Lint, Typecheck & Test
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: pyflowx-ci:latest
|
||||
env:
|
||||
UV_LINK_MODE: copy
|
||||
# ---- 国内源 ----
|
||||
PIP_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
PIP_TRUSTED_HOST: pypi.tuna.tsinghua.edu.cn
|
||||
UV_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
UV_TRUSTED_HOST: pypi.tuna.tsinghua.edu.cn
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- uses: http://gitea:3000/zhou/checkout.git@main
|
||||
|
||||
- name: 安装 uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
version: latest
|
||||
enable-cache: true
|
||||
cache-dependency-glob: uv.lock
|
||||
- name: Sync dependencies
|
||||
run: uv sync --frozen
|
||||
|
||||
- name: 设置 Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- name: Ruff check
|
||||
run: ruff check src tests
|
||||
|
||||
- name: 安装依赖
|
||||
run: uv sync --extra dev --frozen
|
||||
|
||||
- name: Ruff 检查
|
||||
run: uv run ruff check src tests
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# typecheck:pyrefly 严格类型检查
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
typecheck:
|
||||
name: Typecheck (pyrefly)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: 安装 uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
version: latest
|
||||
enable-cache: true
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
- name: 设置 Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
|
||||
- name: 安装依赖
|
||||
run: uv sync --extra dev --frozen
|
||||
|
||||
- name: pyrefly 严格类型检查
|
||||
run: uv run pyrefly check .
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# test:多平台 × 多 Python 版本矩阵测试 + 覆盖率
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
test:
|
||||
name: Test (${{ matrix.os }} / py${{ matrix.python-version }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ ubuntu-latest, windows-latest, macos-latest ]
|
||||
python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12', '3.13' ]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: 安装 uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
version: latest
|
||||
enable-cache: true
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
- name: 设置 Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: 安装依赖
|
||||
run: uv sync --extra dev --frozen
|
||||
|
||||
- name: 运行测试
|
||||
run: uv run pytest -v --cov=pyflowx --cov-report=xml --cov-report=term-missing
|
||||
|
||||
- name: 上传覆盖率
|
||||
if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-${{ matrix.os }}-py${{ matrix.python-version }}
|
||||
path: coverage.xml
|
||||
retention-days: 7
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# 聚合:所有检查通过后才标记完成
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
ci-pass:
|
||||
name: CI Pass
|
||||
runs-on: ubuntu-latest
|
||||
needs: [ lint, typecheck, test ]
|
||||
if: always()
|
||||
steps:
|
||||
- name: 检查依赖任务结果
|
||||
if: ${{ needs.lint.result != 'success' || needs.typecheck.result != 'success' || needs.test.result != 'success' }}
|
||||
run: |
|
||||
echo "lint: ${{ needs.lint.result }}"
|
||||
echo "typecheck: ${{ needs.typecheck.result }}"
|
||||
echo "test: ${{ needs.test.result }}"
|
||||
exit 1
|
||||
- name: 全部通过
|
||||
run: echo "✅ 所有 CI 检查通过"
|
||||
- name: Tox test (py38, py313)
|
||||
run: uvx tox run -e py38,py313
|
||||
|
||||
+42
-177
@@ -2,192 +2,57 @@ name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*.*.*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: '发布版本号(如 v0.1.0)'
|
||||
required: true
|
||||
type: string
|
||||
tags: ['v*.*.*']
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
# Trusted Publishing (OIDC) 上传 PyPI 所需
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# 预检:版本号校验 + 与 pyproject.toml 一致性检查
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
pre-check:
|
||||
name: Pre-release Check
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.meta.outputs.version }}
|
||||
tag: ${{ steps.meta.outputs.tag }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: 解析版本号
|
||||
id: meta
|
||||
run: |
|
||||
if [ -n "${{ inputs.tag }}" ]; then
|
||||
TAG="${{ inputs.tag }}"
|
||||
else
|
||||
TAG="${GITHUB_REF#refs/tags/}"
|
||||
fi
|
||||
# 去除前缀 v
|
||||
VERSION="${TAG#v}"
|
||||
echo "tag=$TAG" >> $GITHUB_OUTPUT
|
||||
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
||||
echo "发布版本: $VERSION (tag: $TAG)"
|
||||
|
||||
- name: 校验版本号格式
|
||||
run: |
|
||||
VERSION="${{ steps.meta.outputs.version }}"
|
||||
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$'; then
|
||||
echo "❌ 版本号格式错误: $VERSION(应为 x.y.z 或 x.y.z-rc.n)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: 校验 pyproject.toml 版本一致
|
||||
run: |
|
||||
# 精确提取 [project] 段的 version 字段(避免匹配到依赖的 version)
|
||||
PY_VERSION=$(awk '/^\[project\]/{f=1} f&&/^version[[:space:]]*=/{gsub(/[" ]/,"",$3); print $3; exit}' pyproject.toml)
|
||||
echo "pyproject.toml version: $PY_VERSION"
|
||||
if [ "$PY_VERSION" != "${{ steps.meta.outputs.version }}" ]; then
|
||||
echo "❌ pyproject.toml 版本($PY_VERSION) 与 tag 版本(${{ steps.meta.outputs.version }}) 不一致"
|
||||
echo "请先更新 pyproject.toml 中的 version 字段"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# 构建:wheel + sdist(纯 Python,单平台即可)
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
build:
|
||||
name: Build Artifacts
|
||||
needs: pre-check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: 安装 uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
version: latest
|
||||
enable-cache: true
|
||||
|
||||
- name: 设置 Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
|
||||
- name: 安装依赖
|
||||
run: uv sync --extra dev --frozen
|
||||
|
||||
- name: 构建 wheel + sdist
|
||||
run: uv build
|
||||
|
||||
- name: 校验产物
|
||||
run: |
|
||||
echo "待上传产物:"
|
||||
ls -la dist/
|
||||
if [ -z "$(ls -A dist/*.whl dist/*.tar.gz 2>/dev/null)" ]; then
|
||||
echo "❌ 未找到 wheel 或 sdist 产物"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: 上传构建产物
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: dist
|
||||
path: dist/*
|
||||
retention-days: 30
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# 发布:上传到 PyPI(Trusted Publishing / OIDC)
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
publish-pypi:
|
||||
name: Publish to PyPI
|
||||
needs: [pre-check, build]
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: pypi
|
||||
url: https://pypi.org/project/pyflowx/${{ needs.pre-check.outputs.version }}
|
||||
permissions:
|
||||
id-token: write
|
||||
steps:
|
||||
- name: 下载构建产物
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: dist
|
||||
path: dist
|
||||
|
||||
- name: 上传到 PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
attestations: true
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# 发布:创建 GitHub Release
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
release:
|
||||
name: Publish Release
|
||||
needs: [pre-check, build, publish-pypi]
|
||||
name: Build, Publish & Release
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
container:
|
||||
image: pyflowx-ci:latest
|
||||
env:
|
||||
UV_LINK_MODE: copy
|
||||
# ---- 国内源 ----
|
||||
PIP_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
PIP_TRUSTED_HOST: pypi.tuna.tsinghua.edu.cn
|
||||
UV_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
UV_TRUSTED_HOST: pypi.tuna.tsinghua.edu.cn
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- uses: http://gitea:3000/zhou/checkout.git@v4
|
||||
|
||||
- name: 下载构建产物
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: dist
|
||||
path: assets
|
||||
|
||||
- name: 整理发布产物
|
||||
- name: Build distributions
|
||||
run: uv build
|
||||
|
||||
- name: Publish to pypi
|
||||
run: uv publish --token '${{ secrets.PYPI_TOKEN }}'
|
||||
|
||||
- name: Create Gitea Release & Upload Assets
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TAG_NAME: ${{ github.ref_name }}
|
||||
REPO: ${{ github.repository }}
|
||||
GITEA_URL: http://172.17.0.1:3000
|
||||
run: |
|
||||
ls -la assets/
|
||||
set -e
|
||||
# 1. 创建 Release
|
||||
RELEASE_ID=$(curl -sS -X POST "$GITEA_URL/api/v1/repos/$REPO/releases" \
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"tag_name\":\"$TAG_NAME\",\"name\":\"Release $TAG_NAME\",\"body\":\"Automated release from CI\",\"draft\":false,\"prerelease\":false}" \
|
||||
| python3 -c "import sys,json;print(json.load(sys.stdin)['id'])")
|
||||
|
||||
- name: 生成 Release Notes
|
||||
id: notes
|
||||
run: |
|
||||
{
|
||||
echo "## pyflowx ${{ needs.pre-check.outputs.version }}"
|
||||
echo ""
|
||||
echo "### 下载"
|
||||
echo ""
|
||||
echo "- **Wheel**: \`pyflowx-${{ needs.pre-check.outputs.version }}-py3-none-any.whl\`"
|
||||
echo "- **源码包**: \`pyflowx-${{ needs.pre-check.outputs.version }}.tar.gz\`"
|
||||
echo ""
|
||||
echo "### 安装"
|
||||
echo ""
|
||||
echo '```bash'
|
||||
echo "pip install pyflowx==${{ needs.pre-check.outputs.version }}"
|
||||
echo '```'
|
||||
echo ""
|
||||
echo "### 完整变更日志"
|
||||
} > RELEASE_NOTES.md
|
||||
{
|
||||
echo "content<<EOF"
|
||||
cat RELEASE_NOTES.md
|
||||
echo "EOF"
|
||||
} >> $GITHUB_OUTPUT
|
||||
echo "Created release id=$RELEASE_ID"
|
||||
|
||||
- name: 创建 GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: ${{ needs.pre-check.outputs.tag }}
|
||||
name: pyflowx ${{ needs.pre-check.outputs.version }}
|
||||
body: ${{ steps.notes.outputs.content }}
|
||||
files: assets/*
|
||||
draft: false
|
||||
prerelease: ${{ contains(needs.pre-check.outputs.version, '-') }}
|
||||
generate_release_notes: true
|
||||
# 2. 上传 dist/ 下所有文件作为附件
|
||||
for f in dist/*; do
|
||||
echo "Uploading $f ..."
|
||||
curl -sS -X POST "$GITEA_URL/api/v1/repos/$REPO/releases/$RELEASE_ID/assets?name=$(basename $f)" \
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
-H "Content-Type: application/octet-stream" \
|
||||
--data-binary "@$f"
|
||||
done
|
||||
|
||||
|
||||
|
||||
@@ -10,3 +10,7 @@ wheels/
|
||||
.venv
|
||||
.coverage
|
||||
.idea
|
||||
*_profile.html
|
||||
|
||||
# Sphinx 文档构建输出
|
||||
docs/_build/
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
3.13
|
||||
3.11
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# ReadTheDocs 配置
|
||||
# https://docs.readthedocs.io/en/stable/config-file/v2.html
|
||||
version: 2
|
||||
|
||||
# 构建配置
|
||||
build:
|
||||
os: ubuntu-24.04
|
||||
tools:
|
||||
python: "3.11"
|
||||
|
||||
# Python 依赖与构建命令
|
||||
python:
|
||||
install:
|
||||
- method: pip
|
||||
path: .
|
||||
extra_requirements:
|
||||
- docs
|
||||
|
||||
# Sphinx 构建
|
||||
sphinx:
|
||||
configuration: docs/conf.py
|
||||
builder: html
|
||||
fail_on_warning: false
|
||||
@@ -0,0 +1,15 @@
|
||||
# PYTHON
|
||||
.coverage
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
.tox/
|
||||
.venv/
|
||||
__pycache__/
|
||||
|
||||
# NODEJS
|
||||
node_modules/
|
||||
|
||||
# IDE
|
||||
.idea
|
||||
.trae
|
||||
.vscode
|
||||
@@ -0,0 +1,108 @@
|
||||
# 文档整理与 Sphinx 文档搭建计划
|
||||
|
||||
## Context
|
||||
|
||||
最近完成 CLI 重构:新增 `pf` 统一入口,13 个工具迁移到 YAML 配置并删除了对应 .py 入口脚本,`run()` 的 verbose 统一应用到 spec。但文档未同步:README 仍引用旧命令(`yamlrun`、`python build.py`),模块结构表缺漏;`runner.py` 的 `_apply_verbose_to_graph` 成为死代码;项目缺少可发布的 Sphinx 文档。本次任务整理这些遗留,并搭建 ReadTheDocs 文档站。
|
||||
|
||||
## 任务范围
|
||||
|
||||
### 1. 清理死代码
|
||||
- 删除 `src/pyflowx/runner.py` 的 `_apply_verbose_to_graph` 函数(line 38-68),功能已移入 `executors.py` 的 `run()`。
|
||||
- 删除 `tests/test_runner.py` 中对应测试(line 610-636,`TestApplyVerboseToGraph` 类)。
|
||||
- 清理 `runner.py` 顶部 `from dataclasses import replace` 若变为未使用。
|
||||
|
||||
### 2. 修复版本不一致
|
||||
- `src/pyflowx/__init__.py:105` 硬编码 `__version__ = "0.4.5"`,`pyproject.toml:25` 为 `0.3.5`。
|
||||
- 统一为 `0.4.5`(`__init__.py` 为准,pyproject.toml 是源但 bumpversion 工具应同时更新两者)。
|
||||
|
||||
### 3. 更新 README.md
|
||||
- L304-308:`python build.py clean/build/test` → `pf pymake clean/build/test`。
|
||||
- L335-351、L435:`yamlrun pipeline.yaml ...` → `pf yamlrun pipeline.yaml ...`(6 处)。
|
||||
- L311:`verbose=True(默认)` 描述保留,但 CLI 示例改为 `pf`。
|
||||
- L558-574 模块结构表:补充 `cli/pf.py`(统一入口)、`cli/configs/`(YAML 工具配置)、`cli/_ops/`(工具函数)、`profiling.py`、`registry.py`。
|
||||
- 顶部增加「文档」徽章链接到 ReadTheDocs。
|
||||
|
||||
### 4. 搭建 Sphinx 文档结构
|
||||
新建 `docs/` 目录:
|
||||
|
||||
```
|
||||
docs/
|
||||
├── conf.py # Sphinx 配置
|
||||
├── index.rst # 首页与目录
|
||||
├── installation.rst # 安装
|
||||
├── quickstart.rst # 快速上手(从 README 提炼)
|
||||
├── guide/
|
||||
│ ├── task.rst # TaskSpec 任务描述
|
||||
│ ├── graph.rst # Graph DAG 构建
|
||||
│ ├── execution.rst # 执行策略与 run()
|
||||
│ ├── yaml.rst # YAML 任务编排
|
||||
│ └── cli.rst # pf 统一入口与工具列表
|
||||
├── api.rst # API 参考(automodule 自动生成)
|
||||
└── changelog.rst # 变更日志摘要
|
||||
```
|
||||
|
||||
**conf.py 要点**:
|
||||
- 扩展:`sphinx.ext.autodoc`、`sphinx.ext.napoleon`(支持 Google/NumPy docstring)、`sphinx.ext.viewcode`、`myst_parser`(支持 Markdown)
|
||||
- 主题:`sphinx_rtd_theme`
|
||||
- 项目版本从 `pyflowx.__version__` 动态读取
|
||||
- `autodoc_default_options`:`members: True, undoc-members: True, show-inheritance: True`
|
||||
|
||||
**api.rst**:用 `automodule:: pyflowx` 抓取 `__all__` 的 56 个公共符号。
|
||||
|
||||
### 5. ReadTheDocs 配置
|
||||
- 新建 `.readthedocs.yaml`:Python 3.11,`pip install -e .[docs]`,`sphinx -b html docs/ docs/_build/`。
|
||||
- `.gitignore` 增加 `docs/_build/`。
|
||||
|
||||
### 6. pyproject.toml 补充 docs 依赖
|
||||
```toml
|
||||
docs = [
|
||||
"sphinx>=7.0",
|
||||
"sphinx-rtd-theme>=2.0",
|
||||
"myst-parser>=3.0",
|
||||
]
|
||||
```
|
||||
并在 `[dependency-groups]` 的 dev 中加入 `pyflowx[docs]`。
|
||||
|
||||
## 关键文件
|
||||
|
||||
| 文件 | 操作 |
|
||||
|------|------|
|
||||
| `src/pyflowx/runner.py` | 删除 `_apply_verbose_to_graph` |
|
||||
| `tests/test_runner.py` | 删除 `TestApplyVerboseToGraph` |
|
||||
| `src/pyflowx/__init__.py` | 版本统一(已 0.4.5,确认) |
|
||||
| `pyproject.toml` | 版本 → 0.4.5;加 docs 依赖 |
|
||||
| `README.md` | 更新 CLI 示例与模块结构表 |
|
||||
| `docs/conf.py` | 新建 |
|
||||
| `docs/*.rst` | 新建 |
|
||||
| `.readthedocs.yaml` | 新建 |
|
||||
| `.gitignore` | 加 docs/_build/ |
|
||||
|
||||
## 验证
|
||||
|
||||
1. **测试与 lint**:
|
||||
```bash
|
||||
uv run pytest tests/ -q
|
||||
uv run ruff check src/ tests/ docs/conf.py
|
||||
uv run pyrefly check src/pyflowx/runner.py
|
||||
```
|
||||
|
||||
2. **Sphinx 构建本地验证**:
|
||||
```bash
|
||||
uv sync --extra docs
|
||||
uv run sphinx-build -b html docs/ docs/_build/
|
||||
```
|
||||
确认无 warning,打开 `docs/_build/index.html` 检查页面。
|
||||
|
||||
3. **pf 功能回归**:
|
||||
```bash
|
||||
pf gitt c
|
||||
pf pymake b --dry-run
|
||||
```
|
||||
|
||||
4. **RTD 配置校验**:`.readthedocs.yaml` 语法正确,`docs/conf.py` 能独立构建。
|
||||
|
||||
## 不在范围
|
||||
|
||||
- 不统一各模块 docstring 风格(napoleon 兼容 Google/NumPy,够用)。
|
||||
- 不重构现有 CLI 工具 YAML。
|
||||
- 不新增中文文档翻译(文档用中文撰写,与项目既有风格一致)。
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
alwaysApply: true
|
||||
scene: git_message
|
||||
---
|
||||
|
||||
在此处编写规则,自定义 AI 生成提交信息的风格。
|
||||
|
||||
## 提交信息格式
|
||||
- 提交信息必须使用中文。
|
||||
- 提交信息必须包含变更的类型(例如 "fix"、"feat"、"refactor" 等)。
|
||||
- 提交信息必须尽简洁明了,不要超过一段落。
|
||||
@@ -0,0 +1,157 @@
|
||||
# Python 开发规范
|
||||
|
||||
本规范结合 Python 最佳实践,作为编写与审查 Python 代码的统一标准。
|
||||
详细操作指南见 `.agents/skills/` 下相应技能。
|
||||
|
||||
## 工具链(以 pyproject.toml 为准)
|
||||
|
||||
| 工具 | 用途 | 配置要点 |
|
||||
|------|------|---------|
|
||||
| **ruff** | lint + format | `line-length=120`,`target-version="py38"` |
|
||||
| **pyrefly** | 类型检查 | `preset="strict"`,`python-version="3.8"` |
|
||||
| **pytest** | 测试 | `asyncio_default_fixture_loop_scope="function"`,marker `slow` |
|
||||
| **coverage** | 覆盖率 | `branch=true`,`fail_under=95`,`concurrency=["thread"]` |
|
||||
| **pre-commit** | 提交前检查 | ruff `--fix` + trailing-whitespace + end-of-file-fixer |
|
||||
|
||||
验证(每次修改后必做):
|
||||
|
||||
```bash
|
||||
uvx --from pyflowx pymake tc
|
||||
uvx --from pyflowx pymake cov
|
||||
```
|
||||
|
||||
## 兼容性
|
||||
|
||||
- **最低 Python 3.8**:用 `from __future__ import annotations` 延迟注解求值;
|
||||
按版本用 `typing.List`(3.8) → 内置泛型(3.9) → `X | Y`(3.10) → `typing.override`(3.12)。
|
||||
- **版本守卫**:`if sys.version_info >= (3, X):` 引入高版本 API;低版本回退分支加 `# pragma: no cover`。
|
||||
- **零运行时依赖**:仅依赖标准库(3.8 需 `graphlib_backport`、`typing-extensions`)。
|
||||
新增依赖须审慎,优先用标准库。
|
||||
|
||||
## 类型注解
|
||||
|
||||
- **公共 API 必须有完整类型注解**,包括返回类型;私有函数也应有注解。
|
||||
- 泛型用 `TypeVar`;PEP 696 `default=` 仅 3.13+ 标准库支持,3.8–3.12 用 `typing_extensions.TypeVar`。
|
||||
- `Mapping`/`Sequence` 用于只读参数,`dict`/`list` 用于可变返回。
|
||||
- `Any` 仅用于真正动态场景(如 `Context` 跨任务异构映射);任务内部类型必须完全静态。
|
||||
- 禁用裸 `# type: ignore`;确需时加具体规则码(如 `# type: ignore[union-attr]`)。
|
||||
- **`TYPE_CHECKING` 守卫**:仅类型检查需要的导入放 `if TYPE_CHECKING:` 块内,避免循环依赖。
|
||||
- **类型收窄**:用 `assert isinstance(x, Y)` 辅助 pyrefly 推断;`cast()` 仅用于类型系统无法表达的场景。
|
||||
|
||||
## 数据结构
|
||||
|
||||
- **不可变优先**:配置/描述类用 `@dataclass(frozen=True)`;可变类属性标注 `RUF012` 豁免。
|
||||
- **缓存**:实例级用 `functools.cached_property`,按参数键控用 `functools.lru_cache`;
|
||||
不可哈希参数需 try/except 回退。修改被缓存数据源后必须手动清空缓存。
|
||||
- **抽象基类**:接口用 `abc.ABC` + `@abstractmethod`(如 `StateBackend`)。
|
||||
- **枚举**:状态/标志值用 `enum.Enum`(如 `TaskStatus`),禁止裸字符串/魔术数字;枚举值用 `UPPER_SNAKE`。
|
||||
- **`__repr__`**:可变类实现 `__repr__`(含关键字段);`frozen=True` dataclass 自动生成。
|
||||
|
||||
## 模块与导入
|
||||
|
||||
- **单一职责**:每模块只做一件事(`task.py` 数据结构、`executors.py` 执行、`command.py` 命令、`compose.py` 组合)。禁止跨职责边界。
|
||||
- **导入顺序**(ruff isort):`__future__` → 标准库 → 第三方 → 本地,各组间空行。
|
||||
- **惰性导入**:仅为打破循环依赖时使用,函数体内导入并注释说明;顶层导入是默认。
|
||||
- **`__all__`**:定义 `__all__` 显式声明导出符号,位置仅次于 `__future__` 之后。
|
||||
- **禁用 star imports**:`from x import *` 污染命名空间、破坏类型检查(`__init__.py` 聚合经 `__all__` 控制为例外)。
|
||||
- **避免 `utils.py`/`helpers.py`**:按职责归入对应模块。
|
||||
|
||||
## 函数设计
|
||||
|
||||
- **模块级函数优于 Mixin**:共享逻辑用模块级函数,类只持有状态与薄方法。
|
||||
- **静态方法慎用**:纯函数直接放模块级。
|
||||
- **参数 ≤ 5 个**为宜;超出用 dataclass 封装参数对象。
|
||||
- **单一职责**:一个函数做一件事;过长函数考虑拆分。
|
||||
- **异常范围要窄**:只捕获预期异常(如 `(TypeError, ValueError, KeyError, AttributeError)`),
|
||||
**禁止** `except Exception` 掩盖 bug;捕获后至少 `logger.warning` 记录。
|
||||
- **可变默认参数**:`def f(x=[])` 是经典坑;用 `None` 哨兵或 `field(default_factory=list)`。
|
||||
|
||||
## 异常处理
|
||||
|
||||
- **自定义异常家族**:继承公共基类(如 `PyFlowXError`),按错误场景分类。
|
||||
- **异常包装**:`raise NewError(...) from exc` 保留因果链。
|
||||
- **不要吞异常**:捕获后必须处理(记录/包装/重抛),禁止空 `except: pass`。
|
||||
- **钩子/回调异常**:第三方回调异常仅记录,不影响主流程。
|
||||
|
||||
## 并发与线程安全
|
||||
|
||||
- **进程全局状态**(`os.environ`/`os.chdir`)在并发场景下必须用全局锁(`threading.RLock`)序列化。
|
||||
- **条件评估不可有可变状态**:组合条件(NOT/AND/OR)不得修改共享 `_reason`,避免竞态。
|
||||
- **批量 I/O**:循环内多次写盘改为批量一次(`contextmanager` 包裹延迟落盘)。
|
||||
- **信号量限流**:`concurrency_key` + `Semaphore` 按组限流。
|
||||
|
||||
## 测试
|
||||
|
||||
详细操作指南见 `.agents/skills/pyflowx-testing` 技能。硬约束:
|
||||
|
||||
- **覆盖率 ≥ 95%**(branch coverage),不得下降。
|
||||
- **公共 API 优先测试**:用公共接口(`has`/`get`),不访问私有方法;
|
||||
故障注入等场景可临时访问私有属性,docstring 注明原因。
|
||||
- **命名**:`test_<被测对象>_<场景>`。
|
||||
- **断言**:原生 `assert x == 1`,禁用 `self.assertEqual`;`pytest.raises` 必填 `match=`。
|
||||
- **Mock 优先级**:`monkeypatch` > 内联 stub > `unittest.mock` > `pytest-mock`。
|
||||
禁用 `@patch` 装饰器、`mock.patch.object` 上下文、`pytest-mock` 的 `mocker` fixture。
|
||||
- **fixture**:`tmp_path`/`monkeypatch`/`capsys` 优先;autouse 仅全局必需时用。
|
||||
- **slow 标记**:耗时测试加 `@pytest.mark.slow`,CI 可 `-m "not slow"` 跳过。
|
||||
- **测试代码也跑 ruff**:`tests/**` 忽略 `ARG001`/`ARG002`。
|
||||
|
||||
## 代码风格
|
||||
|
||||
- **行宽 120**(ruff formatter 处理)。
|
||||
- **docstring**:公共 API 必须有;中文叙述 + 中文注释是本项目既有风格。
|
||||
- **打印和日志**:使用中文打印和日志,避免使用英文。
|
||||
- **命名**:`snake_case` 函数/变量,`PascalCase` 类,`UPPER_SNAKE` 常量,`_leading_underscore` 私有。
|
||||
- **字符串引号**:ruff 默认双引号。
|
||||
- **末尾单 `\n`**、**无尾随空格**(pre-commit 强制)。
|
||||
- **不用 emoji**:除非用户明确要求。
|
||||
|
||||
## Pythonic 风格
|
||||
|
||||
- **`is` 比较 `None`/`True`/`False`**:单例用 `is`,值用 `==`(PEP 8 E711/E712)。
|
||||
- **EAFP 优于 LBYL**:先尝试再处理异常,而非先检查再执行(避免竞态窗口)。
|
||||
- **truthiness**:`if items:` 优于 `if len(items) > 0:`。
|
||||
- **字符串格式化**:首选 f-string;`%` 仅用于 `logging` 延迟格式化。
|
||||
- **推导式**优于 `map`+`filter`;> 2 层拆为显式循环。
|
||||
- **`enumerate`** 替代 `range(len())`;**`zip`** 并行迭代(3.10+ 用 `strict=True`)。
|
||||
- **解包** `a, b = pair` 优于索引访问;忽略值用 `_`。
|
||||
- **海象运算符 `:=`**(3.8+):赋值+判断合一,但不滥用。
|
||||
|
||||
## 日志
|
||||
|
||||
- **`logging.getLogger(__name__)`**:每模块独立 logger,禁用 `print` 调试残留。
|
||||
- **结构化上下文**:`extra={...}` 传字段;`logger.warning("task %r failed: %s", name, exc)` 优于 f-string(延迟格式化)。
|
||||
- **日志级别**:`DEBUG` 诊断 / `INFO` 关键流程 / `WARNING` 可恢复异常 / `ERROR` 需人工介入。
|
||||
- **禁止日志密码/密钥**:脱敏后再记录。
|
||||
|
||||
## 路径与资源
|
||||
|
||||
- **优先 `pathlib.Path`**:`Path("a") / "b"` 而非 `os.path.join`(ruff `PTH` 强制);
|
||||
禁止字符串拼接路径。类型注解用 `Path`,边界 `str` 立即包装。
|
||||
- **`with` 语句**:文件、锁、连接、临时目录一律用 `with` 或 `contextlib.contextmanager`;
|
||||
多资源用 `contextlib.ExitStack`。
|
||||
- **显式关闭**:长生命周期对象(连接池、线程池)实现 `close()`,但优先 `with`。
|
||||
- **批量操作**:循环内多次 acquire/release 改为批量一次。
|
||||
|
||||
## 安全
|
||||
|
||||
- **禁用 `eval`/`exec`**:处理不可信输入时绝不使用;用 `ast.literal_eval` 或专用解析器。
|
||||
- **`subprocess`**:禁用 `shell=True` 除非命令完全可信;优先 `list[str]` 形式。
|
||||
- **凭证不入仓**:密钥/token/密码放 `.env` 或环境变量,`.gitignore` 必须包含 `.env`。
|
||||
- **日志脱敏**:记录请求/响应时移除 `Authorization`、`password` 等字段。
|
||||
- **依赖审计**:`uv lock` 后审阅新增依赖,避免引入已知 CVE 的包。
|
||||
|
||||
## 性能要点
|
||||
|
||||
- **避免重复计算**:循环内查询应缓存或预构建映射(如 `{name: spec}`)。
|
||||
- **避免双重查找**:`has(k)` + `get(k)` 改为单次 `get(k)` + `KeyError` 回退。
|
||||
- **统一校验**:入口校验一次,下游路径不重复(如 `run()` 统一 `validate()`,`layers()` 不再重复)。
|
||||
- **事件 emit**:任务生命周期必须 emit `RUNNING` → `SUCCESS`/`FAILED`/`SKIPPED`,
|
||||
不要留死分支(`# pragma: no cover` 是清理信号,应激活或删除)。
|
||||
|
||||
## Git 与提交
|
||||
|
||||
- **自动提交**:任务完成后自动 `git add`(按文件名)+ `git commit` + `git push`(仅当分支已跟踪远程时执行 push;新分支跳过 push 并在总结中说明)。
|
||||
- **不修改 git config**。
|
||||
- **不运行破坏性命令**(`push --force`/`reset --hard`/`clean -f`)除非用户明确要求。
|
||||
- **staging**:按文件名添加,不用 `git add -A`/`git add .`,避免误加敏感文件。
|
||||
- **commit message**:简洁,聚焦"为什么"而非"是什么";遵循仓库既有风格。
|
||||
@@ -0,0 +1,134 @@
|
||||
---
|
||||
alwaysApply: true
|
||||
---
|
||||
|
||||
# 自驱动开发规则
|
||||
|
||||
本规则定义一种"目标驱动、闭环执行"的工作模式:仅在任务开始时与用户确认一次目标与边界,后续由 Agent 自主完成"计划 → 编码 → 测试 → 文档 → 验证"的迭代循环,直到用户目标达成。
|
||||
|
||||
## 核心原则
|
||||
|
||||
- **目标导向**:始终以用户最终目标为准绳,所有阶段产出都应服务于该目标。
|
||||
- **闭环执行**:每个子任务必须走完"计划 → 实现 → 测试 → 文档 → 验证"五步;禁止跳步留半成品。
|
||||
- **自主决策**:初始确认之后,实现路径、API 形态、重构范围、文件命名、测试组织、错误修复策略等由 Agent 自行决断,不再逐项请示。**可逆操作(编辑文件、运行测试、修复 lint、调整实现)直接执行,不询问**;只有不可逆/高风险操作才暂停。
|
||||
- **透明沟通**:每个阶段开始前用一句话说明意图;关键节点(完成、阻塞、转向)给简短更新;不复述内部思考,**不在收尾时停下询问"是否继续"或"是否提交"**——直接输出总结并结束。
|
||||
- **安全边界**:仅在高风险、不可逆操作或真正阻塞时才暂停找用户。
|
||||
|
||||
## 初始确认(一次性,仅在最开始)
|
||||
|
||||
任务启动时,用 `AskUserQuestion` 一次性确认以下信息(已由项目规范覆盖的不必重复确认):
|
||||
|
||||
1. **目标与范围**:要解决什么问题?交付物是什么?显式列出不在范围内的内容。
|
||||
2. **验收标准**:怎样算"完成"?可观测的判定条件(功能、性能、覆盖率阈值)。
|
||||
3. **特殊约束**:除 `python-standards.md` 之外的约束(兼容性、依赖限制、API 兼容策略等)。
|
||||
4. **测试要求**:覆盖率门槛(项目默认 ≥95%,branch);是否需要新增 `slow` 标记。
|
||||
|
||||
**git commit/push 不在确认范围内**:任务完成后自动 commit + push(仅当分支已跟踪远程时执行 push;新分支跳过 push 并在总结中说明),遵循 `.trae/rules/git-commit-message.md` 风格。仅 force-push、reset --hard、clean -f、修改 git config 等真正破坏性操作才需暂停确认。
|
||||
|
||||
确认后,将目标与验收标准固化进 `TaskCreate` 任务列表,后续不再就同一信息反复询问。
|
||||
|
||||
## 迭代循环
|
||||
|
||||
下列五个阶段构成一个完整闭环。未达验收标准时,回到「计划」开启下一轮;达标准时,进入「收尾」。
|
||||
|
||||
### 1. 计划(Plan)
|
||||
|
||||
- 用 Explore/Glob/Grep 研究相关代码与既有模式,避免凭空设计。
|
||||
- 用 `TaskCreate` 把目标拆为可独立验证的子任务;每完成一项立即 `TaskUpdate` 为 completed。
|
||||
- 优先复用现有抽象;不为本轮假想需求设计接口。
|
||||
- 不过早抽象:三处相似才考虑提取,否则就地写。
|
||||
|
||||
### 2. 实现(Code)
|
||||
|
||||
- 严格遵守 `.trae/rules/python-standards.md` 与既有代码风格。
|
||||
- 优先 Edit 现有文件;新增文件需有明确职责边界。
|
||||
- 不引入运行时依赖(项目零依赖原则);确需引入须在计划阶段说明。
|
||||
- 公共 API 必须有完整类型注解与中文 docstring。
|
||||
- 不写未被要求的功能、不为未来场景预留扩展点。
|
||||
|
||||
### 3. 测试(Test)
|
||||
|
||||
- 新增/修改的公共 API 必须配套测试;优先通过公共接口测试,故障注入可访问私有属性并在 docstring 注明。
|
||||
- Mock 优先级:`monkeypatch` > 内联 stub > `unittest.mock` > `pytest-mock`;禁用 `@patch` 装饰器。
|
||||
- 必跑校验(每次修改后):
|
||||
|
||||
```bash
|
||||
uvx --from pyflowx pymake tc
|
||||
uvx --from pyflowx pymake cov
|
||||
```
|
||||
|
||||
- 测试失败时定位根因再修复,不通过放宽断言或 `# pragma: no cover` 绕过。
|
||||
- 覆盖率不得低于上一次的值(项目门槛 95%,branch)。
|
||||
|
||||
### 4. 文档(Docs)
|
||||
|
||||
- 同步更新 docstring、README、模块结构说明。
|
||||
- 行为变更须同步更新 `.agents/skills/pyflowx-development/SKILL.md` 中的对应章节。
|
||||
- 跨会话有价值的设计决策、约束、陷阱,追加到 memory(`project_memory.md` 或对应 `topics.md`)。
|
||||
- 不主动新建 `*.md` 文档;除非用户明确要求。
|
||||
|
||||
### 5. 验证(Verify)
|
||||
|
||||
- 逐条对照初始确认的「验收标准」核验;未满足则回到「计划」继续下一轮。
|
||||
- 全套门禁通过:ruff、pyrefly、pytest、coverage。
|
||||
- 给出本轮变更清单(改了哪些文件、为什么)。
|
||||
|
||||
## 暂停条件(仅在以下情况中断自驱动找用户)
|
||||
|
||||
1. **歧义无法自决**:需求存在多种合理解读且无既有约定可循。
|
||||
2. **高风险/不可逆操作**:删除非临时文件、`git push --force`、`reset --hard`、删表、修改 CI 配置、修改 git config、卸载依赖等。**普通 `git commit`/`push` 不属于此类**(任务完成后自动执行)。
|
||||
3. **不可恢复的失败**:根因不在本仓库、需外部环境/权限配合、或经两轮尝试仍无法定位。
|
||||
4. **超出初始确认范围**:用户目标在执行中发现需要显著扩大范围或改变方向。
|
||||
5. **用户主动询问**:用户在对话中提出新问题或要求澄清。
|
||||
|
||||
**注意**:"目标已达成"**不是**暂停条件——验收标准全部满足后直接进入收尾并结束任务,不询问"是否扩展范围"或"是否提交"。
|
||||
|
||||
非以上情况,一律继续自驱动,不要为"求确认"而暂停。
|
||||
|
||||
## 决策判据:该问还是自决
|
||||
|
||||
遇到不确定时,按以下顺序判断:
|
||||
|
||||
1. **是否不可逆/高风险?** 是 → 暂停确认(如删除文件、`push --force`、修改 CI 配置、卸载依赖)。否 → 继续。
|
||||
2. **是否在初始确认范围内?** 是 → 按确认执行,不询问。否 → 视为"超出初始确认范围",暂停。
|
||||
3. **是否有既有约定可循?** 是 → 按约定执行(参考 `python-standards.md`、`project_memory.md`)。否 → 视为"歧义无法自决",暂停。
|
||||
4. **是否可逆?** 是 → 直接执行,即使结果可能不完美(可在后续迭代修正)。否 → 暂停。
|
||||
|
||||
**可直接自决(不询问)的典型情况**:
|
||||
|
||||
- 测试失败、覆盖率不达标、lint/类型检查报错 → 定位根因并修复。
|
||||
- 代码风格选择(命名、模块划分、参数顺序)→ 自决。
|
||||
- 文件编辑、运行测试、运行校验命令 → 直接执行。
|
||||
- 任务完成后输出收尾总结 → 直接输出,不询问下一步。
|
||||
- 显式指定 `name` 参数以保持测试兼容性 → 自决。
|
||||
- 重命名局部变量以避免遮蔽 → 自决。
|
||||
|
||||
**必须暂停询问的典型情况**:
|
||||
|
||||
- 删除非临时文件、重命名公共模块/包。
|
||||
- `git push --force`、`reset --hard`、`clean -f`、修改 git config(普通 commit/push 自动执行,无需询问)。
|
||||
- 引入新的运行时依赖(违反项目零依赖原则)。
|
||||
- 修改 CI 配置、pre-commit 钩子、pyproject.toml 的工具链配置。
|
||||
- 卸载或降级既有依赖。
|
||||
|
||||
## 沟通风格
|
||||
|
||||
- 阶段切换时一句话说明即可;不要把内部推理写给用户看。
|
||||
- 完成子任务后用一两句总结改了什么、下一步做什么。
|
||||
- 遇到阻塞时直接说明:卡在哪、试了什么、需要用户做什么。
|
||||
- **不在收尾时询问"是否需要提交"或"是否扩展范围"**——直接输出总结并结束。用户后续若有新需求,由用户主动提出。
|
||||
- 不使用 emoji,除非用户明确要求。
|
||||
|
||||
## 工具使用
|
||||
|
||||
- 独立操作尽量并行调用(多个 Read/Grep/Glob 一批发出)。
|
||||
- 用 `TaskCreate`/`TaskUpdate` 维护进度,不批量推迟标记。
|
||||
- 长命令用后台运行(`run_in_background`),完成会自动通知。
|
||||
- 文件操作一律用专用工具:Read/Edit/Write/Glob/Grep,不用 `cat`/`sed`/`grep`/`find`。
|
||||
|
||||
## 收尾
|
||||
|
||||
- 验收标准全部满足后,**直接输出最终总结并结束任务**:交付物、关键决策、遗留事项。
|
||||
- **自动提交**:收尾时自动 `git add`(按文件名)+ `git commit`(遵循 `.trae/rules/git-commit-message.md` 风格)+ `git push`(仅当分支已跟踪远程时执行;新分支跳过 push 并在总结中说明);**不询问**"是否需要提交"或"是否扩展范围"。
|
||||
- 若验收标准未全部满足,回到「计划」继续下一轮,不停下询问。
|
||||
- 将本次会话的关键产出与决策更新到 memory,便于后续会话续接。
|
||||
@@ -0,0 +1,135 @@
|
||||
---
|
||||
name: "pyflowx-testing"
|
||||
description: "PyFlowX 项目的测试编写规范与 mock 使用指南。在编写或审查测试、选择 mock 工具、设计 fixture、处理 asyncio 测试时调用。"
|
||||
---
|
||||
|
||||
# PyFlowX 测试规范
|
||||
|
||||
本技能是 `.trae/rules/python-standards.md` 测试章节的详细展开。
|
||||
规则文件仅保留硬约束指针,本文件提供完整操作指南。
|
||||
|
||||
## 总则
|
||||
|
||||
- **覆盖率 ≥ 95%**(branch coverage),不得下降。
|
||||
- **公共 API 优先测试**:测试用公共接口(`has`/`get`),不访问私有方法
|
||||
(如 `_expired`)。兼容旧测试的私有方法应删除并迁移测试。
|
||||
例外:`_store`/`_flush` 等内部状态在无法用公共 API 触发时(如模拟过期、
|
||||
故障注入),可临时访问私有属性,并在 docstring 注明原因。
|
||||
- **命名**:`test_<被测对象>_<场景>`,如 `test_storage_key_cache_key_exception_returns_name`。
|
||||
- **每个测试一个断言重点**;多个断言要语义相关。
|
||||
- **slow 标记**:耗时测试加 `@pytest.mark.slow`,CI 可 `-m "not slow"` 跳过。
|
||||
- **测试代码也跑 ruff**:`tests/**` 忽略 `ARG001`/`ARG002`(未用 fixture 参数)。
|
||||
- **断言风格**:用原生 `assert` + 比较运算符(`assert x == 1`),
|
||||
不用 `self.assertEqual`;pytest 会生成更清晰的 diff。
|
||||
|
||||
## Mock 工具选择(强制)
|
||||
|
||||
**优先级**:`monkeypatch` > 内联 stub > `unittest.mock` > `pytest-mock`。
|
||||
|
||||
| 场景 | 工具 | 示例 |
|
||||
|------|------|------|
|
||||
| 替换模块属性 / 环境变量 / 工作目录 | `monkeypatch` | `monkeypatch.setattr(subprocess, "run", fake_run)` |
|
||||
| `os.environ["KEY"]` 临时设置 | `monkeypatch.setenv` | `monkeypatch.setenv("LOCALAPPDATA", "C:\\...")` |
|
||||
| 切换 cwd | `monkeypatch.chdir` | `monkeypatch.chdir(tmp_path)` |
|
||||
| 一次性 stub 函数 | 内联 lambda / 闭包 | `ran = []; monkeypatch.setattr(subprocess, "run", lambda *c, **__: ran.append(c))` |
|
||||
| 复杂 spy(记录调用次数/参数/返回序列) | `unittest.mock.MagicMock` | 仅当 lambda 不足以表达时 |
|
||||
| `with patch(...)` 上下文 | **禁用**(用 monkeypatch) | monkeypatch 自动 teardown 更安全 |
|
||||
|
||||
**禁止**:
|
||||
- 不用 `pytest-mock` 的 `mocker` fixture(项目虽在 dev 依赖声明,但实际
|
||||
测试代码未使用;为保持风格统一,新代码继续用 `monkeypatch`)。
|
||||
- 不用 `unittest.mock.patch` 装饰器(`@patch("x.y")`),它隐藏依赖且
|
||||
与 pytest fixture 模式不兼容;用 `monkeypatch.setattr` 替代。
|
||||
- 不用 `mock.patch.object` 作为上下文管理器,除非被测代码本身就是
|
||||
contextmanager(此时用 `monkeypatch.setattr` 仍更简单)。
|
||||
|
||||
## monkeypatch 使用规范
|
||||
|
||||
- **类型注解**:fixture 参数标注 `monkeypatch: pytest.MonkeyPatch`。
|
||||
- **作用域**:monkeypatch 自动在测试结束时撤销,**禁止**手动
|
||||
`monkeypatch.setattr(x, "y", original)` 恢复(多余且容易遗漏)。
|
||||
例外:在单个测试内需要中途恢复时,用 `monkeypatch.undo()` 全量撤销。
|
||||
- **替换目标**:替换"被测代码看到的对象",而非全局对象本身。
|
||||
- 错误:`monkeypatch.setattr("os.path.exists", fake)` —— 替换全局,影响其他模块。
|
||||
- 正确:`monkeypatch.setattr(pyflowx.command.shutil, "which", fake)` ——
|
||||
替换被测模块引用的 `shutil.which`。
|
||||
- **属性 vs 字符串路径**:优先属性访问形式 `monkeypatch.setattr(obj, "attr", val)`
|
||||
而非字符串路径 `monkeypatch.setattr("pkg.mod.obj.attr", val)`,
|
||||
前者有 IDE 跳转与重构支持。
|
||||
- **记录调用**:用闭包 `ran: list[tuple] = []` + `lambda *a, **k: ran.append((a, k))`
|
||||
替代 `MagicMock`,可读性更好且无需导入。
|
||||
|
||||
## Stub 与 Spy 模式
|
||||
|
||||
- **轻量 stub**:内联定义 `class MockResult: returncode = 0; stdout = ""`,
|
||||
替代 `MagicMock(return_value=...)`,类型明确且不引入 mock 依赖。
|
||||
- **状态收集**:闭包 + list 比 `mock.call_args_list` 更易断言:
|
||||
```python
|
||||
calls: list[list[str]] = []
|
||||
|
||||
|
||||
def fake_run(cmd: list[str], **_: Any) -> MockResult:
|
||||
calls.append(cmd)
|
||||
return MockResult()
|
||||
|
||||
|
||||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||||
assert calls == [["clear"]]
|
||||
```
|
||||
- **副作用序列**:需要按调用次数返回不同值时,用 `itertools.cycle` 或
|
||||
手动计数器,而非 `side_effect=[...]`(mock 专有 API)。
|
||||
- **异常注入**:`def raise_oserror(*a, **k): raise OSError("...")`,
|
||||
用 `pytest.raises(OSError)` 验证,而非 `side_effect=OSError`。
|
||||
|
||||
## 异常断言
|
||||
|
||||
- **`pytest.raises`**:必填 `match=` 正则(除非异常消息完全不可预测),
|
||||
避免误捕获同类异常:
|
||||
```python
|
||||
with pytest.raises(StorageError, match="cannot write"):
|
||||
b.save("a", 1)
|
||||
```
|
||||
- **异常链**:验证 `__cause__` 时用 `exc_info.value.__cause__`,
|
||||
确认 `raise X from Y` 因果链完整。
|
||||
- **禁止** `try/except + assert False`:用 `pytest.raises` 替代。
|
||||
|
||||
## Fixture 规范
|
||||
|
||||
- **`tmp_path`**:处理临时文件,自动清理,禁止 `tempfile.mkdtemp()` 手动管理。
|
||||
- **`monkeypatch`**:环境变量、cwd、模块属性 mock(见上)。
|
||||
- **`capsys`/`capfd`**:捕获 stdout/stderr,验证日志或命令输出。
|
||||
- **autouse fixture**:仅在全局必需时用(如 `conftest.py` 的
|
||||
`packtool_tmp_workdir` 自动切到 tmp_path);否则显式声明参数。
|
||||
- **fixture 命名**:`snake_case`,描述"提供什么"而非"测试什么"
|
||||
(`sample_graph` 优于 `test_data`)。
|
||||
- **fixture 作用域**:默认 `function`;`module`/`session` 仅当构造昂贵且
|
||||
只读时,并加注释说明无副作用。
|
||||
|
||||
## asyncio 测试
|
||||
|
||||
- **fixture `loop_scope="function"`**(pyproject 已配置默认值)。
|
||||
- **async 测试**:`async def test_x():`,pytest-asyncio 自动驱动。
|
||||
- **await 检查**:测试异步函数必须 `await` 结果,禁止仅验证返回 coroutine 对象。
|
||||
- **异步 mock**:用 `AsyncMock`(3.8+ 在 `unittest.mock`)或
|
||||
`async def fake(): return value`,禁用 `MagicMock(return_value=coro)`。
|
||||
|
||||
## 参数化
|
||||
|
||||
- **`@pytest.mark.parametrize`**:用 `ids` 参数提供可读标识:
|
||||
```python
|
||||
@pytest.mark.parametrize(
|
||||
("strategy", "expected_workers"),
|
||||
[("sequential", 1), ("thread", 8), ("async", 1)],
|
||||
ids=["seq", "thread-8", "async"],
|
||||
)
|
||||
```
|
||||
- **参数命名**:参数元组用有意义名称,而非 `("a", "b")`。
|
||||
- **组合爆炸**:参数组合 > 20 时拆分测试,避免单个测试函数臃肿。
|
||||
|
||||
## 测试组织
|
||||
|
||||
- **文件命名**:`test_<被测模块>.py`(`test_storage.py` 对应 `storage.py`)。
|
||||
- **类分组**:仅在测试逻辑强相关时用 `class TestXxx:` 分组;默认用模块级函数。
|
||||
- **docstring**:每个测试函数一句话说明"测试什么场景",复杂场景补充"为什么"。
|
||||
- **setup/teardown**:优先 fixture;`setup_method`/`teardown_method` 仅在
|
||||
无法用 fixture 表达时(罕见)。
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
# 使用国内镜像源拉取基础镜像
|
||||
# 备选镜像源前缀:docker.1ms.run / dockerpull.com / docker.xuanyuan.me
|
||||
FROM docker.m.daocloud.io/python:3.13-slim
|
||||
|
||||
# 国内镜像源(清华)
|
||||
ENV PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_TRUSTED_HOST=pypi.tuna.tsinghua.edu.cn
|
||||
ENV UV_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV UV_TRUSTED_HOST=pypi.tuna.tsinghua.edu.cn
|
||||
|
||||
# 环境变量:非交互 + 路径配置
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
LANG=C.UTF-8 \
|
||||
LC_ALL=C.UTF-8 \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_CACHE_DIR=/uv-cache \
|
||||
UV_PYTHON_INSTALL_DIR=/uv-python \
|
||||
UV_PROJECT_ENVIRONMENT=/opt/venv \
|
||||
PATH="/opt/venv/bin:${PATH}"
|
||||
|
||||
# 配置 apt 国内镜像(阿里云)并安装系统依赖
|
||||
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git \
|
||||
jq \
|
||||
build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 配置 pip 国内镜像(阿里云)
|
||||
RUN mkdir -p /etc/pip \
|
||||
&& printf '[global]\nindex-url = https://mirrors.aliyun.com/pypi/simple/\ntrusted-host = mirrors.aliyun.com\n' \
|
||||
> /etc/pip/pip.conf \
|
||||
&& mkdir -p /root/.config/pip \
|
||||
&& ln -sf /etc/pip/pip.conf /root/.config/pip/pip.conf
|
||||
|
||||
# 安装 uv 并预装 Python 3.8 / 3.13
|
||||
RUN pip install --no-cache-dir uv -i https://mirrors.aliyun.com/pypi/simple/ \
|
||||
&& uv python install 3.8 3.13
|
||||
|
||||
# 安装 Node.js 20.x(actions/checkout 需要)
|
||||
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
|
||||
apt-get install -y nodejs && \
|
||||
node --version
|
||||
|
||||
# 预装项目 dev 依赖(仅复制依赖描述文件,利用 Docker 层缓存)
|
||||
WORKDIR /workspace
|
||||
COPY pyproject.toml tox.ini README.md ./
|
||||
COPY src/ ./src/
|
||||
|
||||
# 同步依赖到 /opt/venv(CI 时直接复用)
|
||||
RUN uv sync --frozen --no-install-project 2>/dev/null || uv sync --no-install-project
|
||||
|
||||
# 预装 tox 环境(py38 + py313)
|
||||
RUN uvx tox run -e py38,py313 --notest 2>/dev/null || true
|
||||
|
||||
# 持久化 uv 缓存目录(CI 可挂载到宿主机加速)
|
||||
VOLUME ["/uv-cache"]
|
||||
|
||||
# 默认入口
|
||||
CMD ["/bin/bash"]
|
||||
@@ -5,6 +5,7 @@
|
||||
[](https://github.com/gookeryoung/pyflowx/actions/workflows/ci.yml)
|
||||
[](https://pypi.org/project/pyflowx/)
|
||||
[](https://pypi.org/project/pyflowx/)
|
||||
[](https://pyflowx.readthedocs.io/zh/latest/)
|
||||
[](https://github.com/gookeryoung/pyflowx)
|
||||
[](https://github.com/gookeryoung/pyflowx/blob/main/LICENSE)
|
||||
|
||||
@@ -14,18 +15,26 @@ PyFlowX 把"任务依赖"这件事做到极致简单:**参数名就是依赖
|
||||
## 特性
|
||||
|
||||
- **零样板** —— 参数名即依赖,框架自动注入上游结果
|
||||
- **三种执行策略** —— `sequential`(调试)/ `thread`(I/O 密集同步)/ `async`(I/O 密集异步)
|
||||
- **四种执行策略** —— `sequential`(调试)/ `thread`(I/O 密集同步)/ `async`(I/O 密集异步)/ `dependency`(依赖驱动,最大化并行)
|
||||
- **类型安全** —— `TaskSpec[T]` 把返回类型一路传到 `RunReport`,mypy strict 通过
|
||||
- **DAG 校验** —— 构建时即时校验重名、缺失依赖、环
|
||||
- **自动分层** —— Kahn 算法分组,同层任务可并行
|
||||
- **重试与超时** —— 每个任务独立配置 `retries` 与 `timeout`
|
||||
- **断点续跑** —— `MemoryBackend` / `JSONBackend`,成功结果可缓存复用
|
||||
- **重试与超时** —— 每个任务独立配置 `RetryPolicy`(max_attempts/delay/backoff/jitter/retry_on)与 `timeout`
|
||||
- **软依赖** —— `soft_depends_on` 仅用于上下文注入,不参与拓扑分层
|
||||
- **并发限制** —— `concurrency_key` + `concurrency_limits` 按组限流
|
||||
- **任务钩子** —— `TaskHooks`(pre_run/post_run/on_failure)生命周期回调
|
||||
- **断点续跑** —— `MemoryBackend` / `JSONBackend`,成功结果可缓存复用;`batch()` 批量落盘
|
||||
- **缓存键** —— `cache_key` 函数基于输入计算稳定键,使不同输入产生独立缓存
|
||||
- **命令任务** —— `cmd` 参数直接执行外部命令,支持列表/shell/可调用对象
|
||||
- **条件执行** —— `conditions` 参数按平台、环境变量、应用安装等条件跳过任务
|
||||
- **图组合** —— `compose` / `GraphComposer` 编程式展开多图字符串引用
|
||||
- **任务模板** —— `task_template` 工厂批量生成相似 TaskSpec
|
||||
- **图级默认值** —— `GraphDefaults` 统一配置 retry/timeout/concurrency 等
|
||||
- **CLI 运行器** —— `CliRunner` 把多个图映射为命令行子命令,替代 Makefile
|
||||
- **可观测** —— `on_event` 回调、`dry_run` 预览、`verbose` 生命周期日志、Mermaid 可视化
|
||||
- **零运行时依赖** —— 仅依赖标准库(3.8 需 `graphlib_backport`)
|
||||
- **95% 测试覆盖** —— 分支覆盖率>= 95%
|
||||
- **可观测** —— `on_event` 回调(RUNNING/SUCCESS/FAILED/SKIPPED)、`dry_run` 预览、`verbose` 生命周期日志、Mermaid 可视化
|
||||
- **YAML 任务编排** —— GitHub Actions 风格的声明式任务图,支持 `jobs`/`needs`/`strategy.matrix`/`if` 等 CI/CD 概念,从 YAML 文件直接加载执行
|
||||
- **最小依赖** —— 仅依赖标准库 + PyYAML(3.8 需 `graphlib_backport`、`typing-extensions`)
|
||||
- **97% 测试覆盖** —— 分支覆盖率 >= 95%
|
||||
|
||||
## 安装
|
||||
|
||||
@@ -67,23 +76,31 @@ print(report["double"]) # [2, 4, 6]
|
||||
|
||||
### TaskSpec —— 任务描述
|
||||
|
||||
`TaskSpec` 是不可变的任务描述符,是唯一需要配置的东西:
|
||||
`TaskSpec` 是不可变的任务描述符(`Generic[T]`,返回类型一路传到 `RunReport`),是唯一需要配置的东西:
|
||||
|
||||
```python
|
||||
px.TaskSpec(
|
||||
name="fetch_user", # 唯一标识
|
||||
fn=fetch_user, # 同步或异步函数
|
||||
cmd=["curl", "..."], # 或: 执行命令(覆盖 fn)
|
||||
depends_on=("auth",), # 依赖的任务名
|
||||
depends_on=("auth",), # 硬依赖(参与拓扑分层)
|
||||
soft_depends_on=("cache",), # 软依赖(仅注入,不参与分层)
|
||||
args=(uid,), # 静态位置参数(追加在注入参数后)
|
||||
kwargs={"timeout": 30}, # 静态关键字参数
|
||||
retries=3, # 失败重试次数(0 = 仅一次)
|
||||
retry=px.RetryPolicy(max_attempts=3, delay=1.0, backoff=2.0), # 重试策略
|
||||
timeout=30.0, # 超时秒数(None = 不限制)
|
||||
tags=("api", "user"), # 自由标签,用于子图过滤
|
||||
conditions=(is_prod,), # 条件函数列表(全部为 True 才执行)
|
||||
priority=10, # 同层内优先级(高优先执行,默认 0)
|
||||
concurrency_key="db", # 并发分组键(配合 concurrency_limits 限流)
|
||||
cache_key=lambda ctx: str(ctx.get("uid")), # 缓存键函数(不同输入独立缓存)
|
||||
hooks=px.TaskHooks(pre_run=..., post_run=..., on_failure=...), # 生命周期钩子
|
||||
cwd=Path("/tmp"), # 命令工作目录(仅 cmd 模式)
|
||||
env={"DEBUG": "1"}, # 环境变量覆盖(fn 与 cmd 模式均生效)
|
||||
verbose=True, # 打印命令输出(仅 cmd 模式)
|
||||
skip_if_missing=True, # 命令不存在时自动跳过(仅 list[str] cmd)
|
||||
allow_upstream_skip=False, # 上游 SKIPPED/FAILED 时是否仍执行
|
||||
continue_on_error=False, # 本任务失败是否不中断整体
|
||||
)
|
||||
```
|
||||
|
||||
@@ -97,18 +114,54 @@ px.TaskSpec(
|
||||
### Graph —— DAG 构建
|
||||
|
||||
```python
|
||||
graph = px.Graph.from_specs([...]) # 整批校验(推荐)
|
||||
# 图级默认值:TaskSpec 字段为 None 时回退
|
||||
defaults = px.GraphDefaults(retry=px.RetryPolicy(max_attempts=2), timeout=60.0)
|
||||
|
||||
graph = px.Graph.from_specs([...], defaults=defaults) # 整批校验(推荐)
|
||||
# 或增量构建
|
||||
graph = px.Graph()
|
||||
graph = px.Graph(defaults=defaults)
|
||||
graph.add(px.TaskSpec("a", fn_a))
|
||||
graph.add(px.TaskSpec("b", fn_b, ("a",)))
|
||||
|
||||
graph.validate() # 显式校验(环检测)
|
||||
graph.layers() # 拓扑分层
|
||||
graph.layers() # 拓扑分层(run() 入口已统一校验,直接调用需自行先 validate)
|
||||
graph.to_mermaid() # Mermaid 可视化
|
||||
graph.describe() # 人类可读摘要
|
||||
graph.subgraph(("api",)) # 按标签切片
|
||||
graph.subgraph_by_names(("a", "b")) # 按名称切片
|
||||
graph.map("fetch", [1, 2, 3], lambda i: TaskSpec(f"fetch_{i}", ...)) # 批量 fan-out
|
||||
```
|
||||
|
||||
### 图组合 —— compose
|
||||
|
||||
`compose` / `GraphComposer` 把带字符串引用的多个图展开为纯 `Graph`:
|
||||
|
||||
```python
|
||||
graphs = {
|
||||
"build": px.Graph.from_specs([px.TaskSpec("b", cmd=["echo", "b"])]),
|
||||
"all": px.Graph.from_specs(["build", px.TaskSpec("t", cmd=["echo", "t"])]),
|
||||
}
|
||||
resolved = px.compose(graphs) # "all" 图中的 "build" 引用被展开
|
||||
```
|
||||
|
||||
引用格式:`"command_name"`(整个图)或 `"command_name.task_name"`(特定任务)。
|
||||
`CliRunner` 内部自动调用 `compose`。
|
||||
|
||||
### 任务模板 —— task_template
|
||||
|
||||
`task_template` 工厂批量生成相似 TaskSpec:
|
||||
|
||||
```python
|
||||
fetch = px.task_template(
|
||||
fn=fetch_url,
|
||||
retry=px.RetryPolicy(max_attempts=5),
|
||||
timeout=30.0,
|
||||
tags=("api",),
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
fetch("users", url="https://api.example.com/users"),
|
||||
fetch("posts", url="https://api.example.com/posts"),
|
||||
])
|
||||
```
|
||||
|
||||
### run —— 执行
|
||||
@@ -116,12 +169,14 @@ graph.subgraph_by_names(("a", "b")) # 按名称切片
|
||||
```python
|
||||
report = px.run(
|
||||
graph,
|
||||
strategy="async", # sequential | thread | async
|
||||
strategy="async", # sequential | thread | async | dependency
|
||||
max_workers=8, # thread 策略的线程池大小
|
||||
concurrency_limits={"db": 2}, # 按 concurrency_key 限流
|
||||
dry_run=False, # True = 仅打印计划
|
||||
verbose=False, # True = 打印任务生命周期日志
|
||||
on_event=callback, # 状态转换回调
|
||||
on_event=callback, # 状态转换回调(RUNNING/SUCCESS/FAILED/SKIPPED)
|
||||
state=px.JSONBackend("state.json"), # 断点续跑后端
|
||||
continue_on_error=False, # True = 单任务失败不中断整体
|
||||
)
|
||||
```
|
||||
|
||||
@@ -141,7 +196,7 @@ report.describe() # 人类可读报告
|
||||
按顺序求值:
|
||||
|
||||
1. **标注为 `Context`** 的参数 → 接收完整上游结果映射
|
||||
2. **名称匹配依赖** 的参数 → 接收该依赖的结果
|
||||
2. **名称匹配依赖** 的参数 → 接收该依赖的结果(含软依赖,缺失时注入默认值)
|
||||
3. **`**kwargs`** 参数 → 接收所有依赖结果(dict)
|
||||
4. **`TaskSpec.args` / `kwargs`** → 为非依赖参数提供静态值
|
||||
|
||||
@@ -170,8 +225,11 @@ def fetch_user(uid: int) -> dict: # uid 来自 TaskSpec.args
|
||||
| `sequential` | 串行 | 调试、CPU 密集 | 直接调用 | 事件循环 |
|
||||
| `thread` | 线程池 | I/O 密集同步 | 线程池 | 不支持 |
|
||||
| `async` | 事件循环 | I/O 密集异步 | 卸载到线程池 | 事件循环 |
|
||||
| `dependency` | 依赖驱动 | 最大化并行度 | 卸载到线程池 | 事件循环 |
|
||||
|
||||
所有策略都遵循 `retries`、`timeout`、上下文注入、状态后端,并发出 `TaskEvent`。
|
||||
所有策略都遵循 `RetryPolicy`、`timeout`、上下文注入、状态后端、`concurrency_limits`,
|
||||
并发出 `TaskEvent`(RUNNING/SUCCESS/FAILED/SKIPPED)。`dependency` 策略无层屏障:
|
||||
任务在其所有硬依赖完成后立即启动。
|
||||
|
||||
## 命令任务
|
||||
|
||||
@@ -244,15 +302,123 @@ runner.run_cli() # 解析 sys.argv 并执行
|
||||
命令行用法:
|
||||
|
||||
```bash
|
||||
python build.py clean # 执行 clean 图
|
||||
python build.py build --strategy thread # 覆盖执行策略
|
||||
python build.py test --dry-run # 仅打印执行计划
|
||||
python build.py --list # 列出所有命令
|
||||
python build.py --quiet # 静默模式
|
||||
pf pymake clean # 执行 clean 图
|
||||
pf pymake build --strategy thread # 覆盖执行策略
|
||||
pf pymake test --dry-run # 仅打印执行计划
|
||||
pf pymake --list # 列出所有命令
|
||||
pf pymake --quiet # 静默模式
|
||||
```
|
||||
|
||||
`verbose=True`(默认)时打印任务生命周期(开始/成功/失败/跳过)与命令输出;`--quiet` 关闭。
|
||||
|
||||
## YAML 任务编排
|
||||
|
||||
PyFlowX 支持 GitHub Actions 风格的声明式 YAML 任务编排,从 YAML 文件直接加载任务图。
|
||||
|
||||
### 编程式 API
|
||||
|
||||
```python
|
||||
import pyflowx as px
|
||||
|
||||
# 从 YAML 文件加载任务图
|
||||
graph = px.Graph.from_yaml("pipeline.yaml")
|
||||
report = px.run(graph, strategy="thread")
|
||||
|
||||
# 或用函数式 API
|
||||
graph = px.load_yaml("pipeline.yaml")
|
||||
graph = px.parse_yaml_string("""
|
||||
jobs:
|
||||
hello:
|
||||
cmd: ["echo", "hello"]
|
||||
""")
|
||||
```
|
||||
|
||||
### CLI 入口
|
||||
|
||||
通过 `pf` 统一入口调用(详见 [pf 工具](#cli-工具) 章节):
|
||||
|
||||
```bash
|
||||
# 执行 YAML 任务图
|
||||
pf yamlrun pipeline.yaml
|
||||
|
||||
# 指定执行策略
|
||||
pf yamlrun pipeline.yaml --strategy thread
|
||||
|
||||
# 仅打印任务分层,不执行
|
||||
pf yamlrun pipeline.yaml --dry-run
|
||||
|
||||
# 列出所有任务名
|
||||
pf yamlrun pipeline.yaml --list
|
||||
|
||||
# 静默模式
|
||||
pf yamlrun pipeline.yaml --quiet
|
||||
```
|
||||
|
||||
### YAML Schema(GitHub Actions 风格)
|
||||
|
||||
```yaml
|
||||
strategy: thread # 图级默认策略
|
||||
defaults: # 图级默认值
|
||||
retry: {max_attempts: 3}
|
||||
verbose: true
|
||||
env: {CI: "true"}
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
cmd: ["git", "clone", "..."]
|
||||
runs-on: linux
|
||||
|
||||
build:
|
||||
needs: [setup] # 依赖列表
|
||||
cmd: ["python", "-m", "build"]
|
||||
timeout: 300
|
||||
retry: {max_attempts: 2, delay: 1.0}
|
||||
|
||||
test:
|
||||
needs: [build]
|
||||
cmd: ["python${{ matrix.version }}", "-m", "pytest"] # 矩阵占位符
|
||||
strategy:
|
||||
matrix: # 笛卡尔积展开为 6 个任务
|
||||
version: ["3.8", "3.9", "3.10"]
|
||||
os: ["linux", "macos"]
|
||||
if: "env.CI" # 条件: 环境变量存在
|
||||
|
||||
lint:
|
||||
needs: [build]
|
||||
cmd: ["ruff", "check"]
|
||||
if: "env.CI == 'true'" # 条件: 环境变量等于
|
||||
|
||||
deploy:
|
||||
needs: [test, lint] # 矩阵依赖自动展开
|
||||
cmd: ["twine", "upload"]
|
||||
if: "env.DEPLOY_TOKEN != ''"
|
||||
allow-upstream-skip: true
|
||||
concurrency-key: deploy_lock
|
||||
```
|
||||
|
||||
### 字段映射
|
||||
|
||||
| YAML 字段 | TaskSpec 字段 | 说明 |
|
||||
|-----------|---------------|------|
|
||||
| `jobs.<id>` | `name` | job ID 作为任务名 |
|
||||
| `cmd` / `run` | `cmd` | `cmd` 为列表形式,`run` 为 shell 字符串 |
|
||||
| `needs` | `depends_on` | 依赖列表(矩阵任务自动展开) |
|
||||
| `if` | `conditions` | `success()` / `always()` / `env.VAR` / `env.VAR == 'x'` |
|
||||
| `strategy.matrix` | 矩阵扇出 | 笛卡尔积展开为多个任务 |
|
||||
| `${{ matrix.key }}` | 占位符 | 在 cmd/run/cwd/env 中替换 |
|
||||
| `timeout` | `timeout` | 超时秒数 |
|
||||
| `retry` | `retry` | `{max_attempts, delay, backoff, jitter}` |
|
||||
| `cwd` | `cwd` | 工作目录 |
|
||||
| `env` | `env` | 环境变量 |
|
||||
| `verbose` | `verbose` | 详细输出 |
|
||||
| `continue-on-error` | `continue_on_error` | 失败不中止整图 |
|
||||
| `skip-if-missing` | `skip_if_missing` | 命令不存在时跳过 |
|
||||
| `allow-upstream-skip` | `allow_upstream_skip` | 上游跳过时仍执行 |
|
||||
| `priority` | `priority` | 同层优先级 |
|
||||
| `concurrency-key` | `concurrency_key` | 并发限制键 |
|
||||
| `tags` | `tags` | 自由标签 |
|
||||
| `runs-on` | `tags`(追加) | 运行环境标签 |
|
||||
|
||||
## 示例
|
||||
|
||||
仓库 `examples/` 目录包含完整示例:
|
||||
@@ -260,6 +426,7 @@ python build.py --quiet # 静默模式
|
||||
- [`etl_pipeline.py`](examples/etl_pipeline.py) —— ETL 流水线(sequential)
|
||||
- [`parallel_run.py`](examples/parallel_run.py) —— 并行执行对比(thread vs sequential)
|
||||
- [`async_aggregation.py`](examples/async_aggregation.py) —— 异步聚合 + Context 注入
|
||||
- [`yaml_pipeline.yaml`](examples/yaml_pipeline.yaml) + [`yaml_pipeline.py`](examples/yaml_pipeline.py) —— YAML 声明式 CI/CD 流水线(矩阵扇出 + 条件执行)
|
||||
|
||||
运行:
|
||||
|
||||
@@ -267,6 +434,8 @@ python build.py --quiet # 静默模式
|
||||
python examples/etl_pipeline.py
|
||||
python examples/parallel_run.py
|
||||
python examples/async_aggregation.py
|
||||
python -m pyflowx.examples.yaml_pipeline
|
||||
pf yamlrun src/pyflowx/examples/yaml_pipeline.yaml --dry-run
|
||||
```
|
||||
|
||||
## 断点续跑
|
||||
@@ -275,12 +444,25 @@ python examples/async_aggregation.py
|
||||
from pyflowx import JSONBackend
|
||||
|
||||
# 第一次运行:成功结果写入 state.json
|
||||
backend = JSONBackend("state.json")
|
||||
backend = JSONBackend("state.json", ttl=3600) # ttl 秒数,过期条目自动忽略
|
||||
report = px.run(graph, strategy="sequential", state=backend)
|
||||
|
||||
# 第二次运行:已缓存任务自动跳过
|
||||
# 第二次运行:已缓存任务自动跳过(状态为 SKIPPED)
|
||||
report = px.run(graph, strategy="sequential", state=backend)
|
||||
# report.results 中缓存任务状态为 SKIPPED
|
||||
```
|
||||
|
||||
`run()` 内部以 `backend.batch()` 包裹整个执行:所有 `save` 延迟到运行结束时统一落盘一次
|
||||
(`JSONBackend` 从 O(N²) 降为 O(N) 磁盘写入;`MemoryBackend` 为 no-op)。
|
||||
|
||||
**缓存键**:默认存储键为任务名。配置 `cache_key` 函数后,键为 `"name:cache_key_value"`,
|
||||
使不同输入产生独立缓存条目:
|
||||
|
||||
```python
|
||||
px.TaskSpec(
|
||||
"fetch_user",
|
||||
fn=fetch_user,
|
||||
cache_key=lambda ctx: str(ctx.get("uid")), # 不同 uid 独立缓存
|
||||
)
|
||||
```
|
||||
|
||||
## 错误处理
|
||||
@@ -321,14 +503,52 @@ except px.PyFlowXError:
|
||||
|
||||
PyFlowX 专注于**单机 DAG 调度**的极致简洁,适合 ETL、数据处理、CI 流水线等场景。
|
||||
|
||||
## 高级特性
|
||||
|
||||
### 并发限制
|
||||
|
||||
按 `concurrency_key` 分组限流,避免压垮下游资源:
|
||||
|
||||
```python
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("q1", fn=query_db, concurrency_key="db"),
|
||||
px.TaskSpec("q2", fn=query_db, concurrency_key="db"),
|
||||
px.TaskSpec("q3", fn=query_db, concurrency_key="db"),
|
||||
])
|
||||
# 同一时刻最多 2 个 "db" 组任务运行
|
||||
px.run(graph, strategy="async", concurrency_limits={"db": 2})
|
||||
```
|
||||
|
||||
### 任务钩子
|
||||
|
||||
`TaskHooks` 在任务生命周期触发(异常仅记录,不影响任务状态):
|
||||
|
||||
```python
|
||||
hooks = px.TaskHooks(
|
||||
pre_run=lambda spec: print(f"start {spec.name}"),
|
||||
post_run=lambda spec, value: print(f"done {spec.name}"),
|
||||
on_failure=lambda spec, exc: alert(spec.name, exc),
|
||||
)
|
||||
px.TaskSpec("task", fn=work, hooks=hooks)
|
||||
```
|
||||
|
||||
### 优先级
|
||||
|
||||
同层内按 `priority` 降序执行(稳定排序):
|
||||
|
||||
```python
|
||||
px.TaskSpec("low", fn=work, priority=0)
|
||||
px.TaskSpec("high", fn=work, priority=10) # 同层内先执行
|
||||
```
|
||||
|
||||
## 开发
|
||||
|
||||
```bash
|
||||
# 安装开发依赖
|
||||
uv sync --extra dev
|
||||
|
||||
# 运行测试(含覆盖率)
|
||||
uv run pytest --cov=pyflowx --cov-fail-under=100
|
||||
# 运行测试(含覆盖率,阈值 95%)
|
||||
uv run pytest --cov=pyflowx --cov-fail-under=95
|
||||
|
||||
# 类型检查
|
||||
uv run mypy
|
||||
@@ -338,6 +558,42 @@ uv run ruff check src tests examples
|
||||
uv run ruff format --check src tests examples
|
||||
```
|
||||
|
||||
## 模块结构
|
||||
|
||||
### 核心
|
||||
|
||||
| 模块 | 职责 |
|
||||
|------|------|
|
||||
| `task.py` | 纯数据结构:`TaskSpec`、`RetryPolicy`、`TaskHooks`、`TaskStatus` |
|
||||
| `graph.py` | DAG 构建、校验、分层、可视化 |
|
||||
| `compose.py` | 多图组合:`GraphComposer` / `compose` |
|
||||
| `context.py` | 上下文注入:参数名→依赖解析 |
|
||||
| `command.py` | 命令执行:`run_command`(list/shell/Callable) |
|
||||
| `conditions.py` | 条件执行:内置条件与组合器 |
|
||||
| `executors.py` | 执行器与 `run` 入口:四种策略共享模块级辅助;verbose 统一应用到 spec |
|
||||
| `storage.py` | 状态后端:`MemoryBackend` / `JSONBackend`(batch flush) |
|
||||
| `runner.py` | CLI 运行器:`CliRunner` |
|
||||
| `report.py` | 运行结果:`RunReport` / `TaskResult` |
|
||||
| `yaml_loader.py` | YAML 任务编排:GitHub Actions 风格 schema 解析(`load_yaml` / `parse_yaml_string` / `run_cli`) |
|
||||
| `registry.py` | 函数注册中心:`register_fn` / `get_fn` / `has_fn`(YAML 的 `fn:` 引用) |
|
||||
| `profiling.py` | 性能分析:`Profiler` 任务耗时统计 |
|
||||
| `errors.py` | 错误家族:`PyFlowXError` 子类 |
|
||||
| `ops/` | 工具函数(dev/files/media/system),被 YAML 的 `fn:` 引用 |
|
||||
|
||||
### CLI 工具
|
||||
|
||||
| 模块 | 职责 |
|
||||
|------|------|
|
||||
| `cli/pf.py` | 统一入口:`pf <tool> [command]`,自动发现 `configs/*.yaml` 并路由 |
|
||||
| `cli/configs/` | YAML 工具配置(gittool/filedate/pdftool 等 13 个工具的 `cli:` schema) |
|
||||
| `cli/pymake.py` | 构建工具(替代 Makefile),`pf pymake b` 调用 |
|
||||
| `cli/yamlrun.py` | YAML pipeline 执行器,`pf yamlrun pipeline.yaml` 调用 |
|
||||
| `cli/profiler.py` | 性能分析 CLI |
|
||||
| `cli/emlmanager.py` | 邮件管理 CLI |
|
||||
| `cli/dev/` | 开发工具(dockercmd/envdev) |
|
||||
| `cli/llm/` | LLM 工具(msdownload/sglang) |
|
||||
| `cli/system/` | 系统工具(clearscreen/taskkill/which) |
|
||||
|
||||
## 许可证
|
||||
|
||||
MIT
|
||||
|
||||
Vendored
+106
@@ -0,0 +1,106 @@
|
||||
API 参考
|
||||
========
|
||||
|
||||
任务描述
|
||||
--------
|
||||
|
||||
.. autoclass:: pyflowx.TaskSpec
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
:exclude-members: args, kwargs
|
||||
|
||||
.. autoclass:: pyflowx.RetryPolicy
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.TaskHooks
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.TaskStatus
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
图构建
|
||||
------
|
||||
|
||||
.. autoclass:: pyflowx.Graph
|
||||
:members:
|
||||
:undoc-members:
|
||||
:exclude-members: from_specs, from_yaml
|
||||
|
||||
.. autoclass:: pyflowx.GraphDefaults
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autofunction:: pyflowx.compose
|
||||
.. autofunction:: pyflowx.task_template
|
||||
|
||||
执行
|
||||
----
|
||||
|
||||
.. autofunction:: pyflowx.run
|
||||
|
||||
.. autoclass:: pyflowx.RunReport
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.TaskResult
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
YAML 编排
|
||||
---------
|
||||
|
||||
.. autofunction:: pyflowx.load_yaml
|
||||
.. autofunction:: pyflowx.parse_yaml_string
|
||||
.. autofunction:: pyflowx.run_yaml
|
||||
.. autofunction:: pyflowx.run_cli
|
||||
.. autofunction:: pyflowx.build_cli_parser
|
||||
|
||||
函数注册
|
||||
--------
|
||||
|
||||
.. autofunction:: pyflowx.register_fn
|
||||
.. autofunction:: pyflowx.get_fn
|
||||
.. autofunction:: pyflowx.has_fn
|
||||
|
||||
命令执行
|
||||
--------
|
||||
|
||||
.. autofunction:: pyflowx.run_command
|
||||
|
||||
CLI 运行器
|
||||
----------
|
||||
|
||||
.. autoclass:: pyflowx.CliRunner
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
状态后端
|
||||
--------
|
||||
|
||||
.. autoclass:: pyflowx.StateBackend
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.MemoryBackend
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.JSONBackend
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
错误家族
|
||||
--------
|
||||
|
||||
.. autoexception:: pyflowx.PyFlowXError
|
||||
.. autoexception:: pyflowx.DuplicateTaskError
|
||||
.. autoexception:: pyflowx.MissingDependencyError
|
||||
.. autoexception:: pyflowx.CycleError
|
||||
.. autoexception:: pyflowx.TaskFailedError
|
||||
.. autoexception:: pyflowx.TaskTimeoutError
|
||||
.. autoexception:: pyflowx.InjectionError
|
||||
.. autoexception:: pyflowx.StorageError
|
||||
@@ -0,0 +1,45 @@
|
||||
变更日志
|
||||
========
|
||||
|
||||
0.4.5
|
||||
-----
|
||||
|
||||
CLI 重构
|
||||
~~~~~~~~
|
||||
|
||||
- 新增 ``pf`` 统一入口:通过 ``pf <tool> [command] [options]`` 调用所有工具
|
||||
- 13 个工具迁移到 YAML 配置(filedate/filelevel/folderback/folderzip/screenshot/sshcopyid/lscalc/bumpversion/autofmt/piptool/packtool/pdftool/gittool)
|
||||
- YAML 配置支持 ``cli:`` 段声明命令行参数 schema,由 ``build_cli_parser`` 自动生成 argparse
|
||||
- 删除 13 个冗余 ``.py`` 入口脚本,统一通过 ``pf`` 调用
|
||||
- ``run()`` 在 ``verbose=True`` 时自动把 verbose 标记应用到所有 spec
|
||||
- 全局选项 ``--verbose`` 改为 ``--quiet``(默认显示执行过程)
|
||||
- ``cmd`` 任务成功时打印 stdout(此前被静默丢弃)
|
||||
- ``gittool`` 用 ``CLEAN_EXCLUDES`` 数组变量配置 ``git clean -e`` 参数
|
||||
|
||||
YAML 任务编排
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- 支持 ``variables`` 变量定义,``${VAR}`` 在 cmd/env/cwd 中替换
|
||||
- 列表变量展开为 cmd 数组多个元素
|
||||
- ``cli:`` 段支持 subcommands/positional/options 三级 schema
|
||||
- 支持 ``type: path`` 自动转为 ``pathlib.Path``
|
||||
|
||||
文档
|
||||
~~~~
|
||||
|
||||
- 搭建 Sphinx 文档,发布到 ReadTheDocs
|
||||
- 更新 README:CLI 示例改为 ``pf`` 统一入口,模块结构表补全
|
||||
|
||||
0.3.x
|
||||
-----
|
||||
|
||||
- 新增 YAML 任务编排(GitHub Actions 风格 schema)
|
||||
- 新增 ``fn:`` 函数引用与 ``register_fn`` / ``get_fn`` 注册中心
|
||||
- 新增 ``compose`` / ``GraphComposer`` 多图组合
|
||||
- 新增 ``task_template`` 任务模板工厂
|
||||
- 新增 ``concurrency_key`` + ``concurrency_limits`` 并发限制
|
||||
- 新增 ``JSONBackend`` 断点续跑与 ``batch()`` 批量落盘
|
||||
- 新增 ``cache_key`` 缓存键函数
|
||||
- 新增条件执行(``IS_WINDOWS`` / ``HAS_INSTALLED`` / ``ENV_VAR_EQUALS`` 等)
|
||||
- 四种执行策略:``sequential`` / ``thread`` / ``async`` / ``dependency``
|
||||
- 参数名即依赖的上下文注入机制
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Sphinx 配置.
|
||||
|
||||
ReadTheDocs 构建 PyFlowX 文档站。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 确保 src/ 在 sys.path 中, autodoc 能导入 pyflowx
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
|
||||
|
||||
from pyflowx import __version__
|
||||
|
||||
# -- 项目信息 --------------------------------------------------------------
|
||||
project = "PyFlowX"
|
||||
author = "pyflowx"
|
||||
copyright = "2024, pyflowx"
|
||||
release = __version__
|
||||
version = __version__
|
||||
|
||||
# -- Sphinx 配置 -----------------------------------------------------------
|
||||
extensions = [
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.napoleon",
|
||||
"sphinx.ext.viewcode",
|
||||
"sphinx.ext.intersphinx",
|
||||
"myst_parser",
|
||||
]
|
||||
|
||||
# -- 主题 ------------------------------------------------------------------
|
||||
html_theme = "sphinx_rtd_theme"
|
||||
html_static_path = ["_static"]
|
||||
|
||||
# -- autodoc 配置 ----------------------------------------------------------
|
||||
autodoc_default_options = {
|
||||
"members": True,
|
||||
"undoc-members": True,
|
||||
"show-inheritance": True,
|
||||
"member-order": "bysource",
|
||||
}
|
||||
autodoc_type_hints = "description"
|
||||
autodoc_typehints_format = "short"
|
||||
|
||||
# -- napoleon 配置 (Google/NumPy docstring 兼容) --------------------------
|
||||
napoleon_google_docstring = True
|
||||
napoleon_numpy_docstring = True
|
||||
napoleon_include_init_with_doc = False
|
||||
napoleon_include_private_with_doc = False
|
||||
napoleon_include_special_with_doc = True
|
||||
|
||||
# -- intersphinx -----------------------------------------------------------
|
||||
intersphinx_mapping = {
|
||||
"python": ("https://docs.python.org/3", None),
|
||||
}
|
||||
|
||||
# -- 全局选项 ---------------------------------------------------------------
|
||||
language = "zh_CN"
|
||||
master_doc = "index"
|
||||
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
|
||||
source_suffix = {
|
||||
".rst": "restructuredtext",
|
||||
".md": "markdown",
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
pf 统一 CLI 入口
|
||||
================
|
||||
|
||||
所有工具通过 ``pf <tool> [command] [options]`` 调用。工具定义在 ``cli/configs/`` 目录下的 YAML 文件中。
|
||||
|
||||
基本用法
|
||||
--------
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pf # 列出所有可用工具
|
||||
pf filedate # 查看 filedate 工具帮助
|
||||
pf filedate add a.txt # 调用 filedate 的 add 子命令
|
||||
pf gitt c # 调用 gittool 的 c 子命令
|
||||
pf pymake b # 调用 pymake 的 b 别名
|
||||
|
||||
全局选项
|
||||
--------
|
||||
|
||||
所有 YAML 工具支持以下全局选项:
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 25 75
|
||||
|
||||
* - 选项
|
||||
- 说明
|
||||
* - ``--dry-run``
|
||||
- 仅打印执行计划,不执行
|
||||
* - ``--quiet`` / ``-q``
|
||||
- 减少输出,不显示执行过程
|
||||
* - ``--strategy``
|
||||
- 执行策略(``sequential`` / ``thread`` / ``async`` / ``dependency``)
|
||||
* - ``--list``
|
||||
- 列出所有任务名后退出
|
||||
|
||||
默认 ``verbose`` 开启,显示执行过程(任务开始/命令/返回码/任务成功)。``--quiet`` 关闭。
|
||||
|
||||
YAML 配置工具
|
||||
--------------
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 20 15 65
|
||||
|
||||
* - 工具
|
||||
- 别名
|
||||
- 说明
|
||||
* - ``filedate``
|
||||
- ``fd``
|
||||
- 文件日期处理
|
||||
* - ``filelevel``
|
||||
- ``fl``
|
||||
- 文件等级重命名
|
||||
* - ``folderback``
|
||||
- ``fb``
|
||||
- 文件夹备份
|
||||
* - ``folderzip``
|
||||
- ``fz``
|
||||
- 文件夹压缩
|
||||
* - ``gittool``
|
||||
- ``gitt``
|
||||
- Git 执行工具
|
||||
* - ``lscalc``
|
||||
- ``ls``
|
||||
- LS-DYNA 计算工具
|
||||
* - ``packtool``
|
||||
- ``pack``
|
||||
- Python 打包工具
|
||||
* - ``pdftool``
|
||||
- ``pdf``
|
||||
- PDF 文件工具集
|
||||
* - ``piptool``
|
||||
- ``pip``
|
||||
- pip 包管理工具
|
||||
* - ``screenshot``
|
||||
- ``ss``
|
||||
- 截图工具
|
||||
* - ``sshcopyid``
|
||||
- ``ssh``
|
||||
- SSH 密钥部署工具
|
||||
* - ``autofmt``
|
||||
- ``af``
|
||||
- 自动格式化工具
|
||||
* - ``bumpversion``
|
||||
- ``bump``
|
||||
- 版本号自动管理工具
|
||||
|
||||
传统工具
|
||||
--------
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 20 80
|
||||
|
||||
* - 工具
|
||||
- 说明
|
||||
* - ``pymake``
|
||||
- 构建工具(替代 Makefile),如 ``pf pymake b`` 构建
|
||||
* - ``yamlrun``
|
||||
- YAML pipeline 执行器,``pf yamlrun pipeline.yaml``
|
||||
* - ``profiler``
|
||||
- 性能分析
|
||||
* - ``emlman``
|
||||
- 邮件管理
|
||||
* - ``reseticon``
|
||||
- 重置图标缓存
|
||||
|
||||
自定义工具
|
||||
----------
|
||||
|
||||
在 ``cli/configs/`` 目录新建 ``<tool>.yaml`` 即可被 ``pf`` 自动发现:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
# cli/configs/mytool.yaml
|
||||
strategy: sequential
|
||||
variables:
|
||||
MSG: "hello"
|
||||
cli:
|
||||
description: "我的工具"
|
||||
usage: "pf mytool [command]"
|
||||
subcommands:
|
||||
greet:
|
||||
help: "打招呼"
|
||||
jobs:
|
||||
greet:
|
||||
cmd: ["echo", "${MSG}"]
|
||||
|
||||
执行::
|
||||
|
||||
pf mytool greet
|
||||
|
||||
CliRunner(编程式)
|
||||
-------------------
|
||||
|
||||
``CliRunner`` 把多个 Graph 映射为命令行子命令,适合构建项目专属构建工具:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
runner = px.CliRunner(
|
||||
strategy="sequential",
|
||||
description="My Build Tool",
|
||||
graphs={
|
||||
"clean": clean_graph,
|
||||
"build": build_graph,
|
||||
"test": test_graph,
|
||||
},
|
||||
)
|
||||
runner.run_cli() # 解析 sys.argv 并执行
|
||||
|
||||
命令行::
|
||||
|
||||
pf pymake clean
|
||||
pf pymake build --strategy thread
|
||||
pf pymake test --dry-run
|
||||
pf pymake --list
|
||||
pf pymake --quiet
|
||||
@@ -0,0 +1,93 @@
|
||||
执行策略与 run()
|
||||
=================
|
||||
|
||||
``run()`` 是执行入口,支持四种策略:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
report = px.run(
|
||||
graph,
|
||||
strategy="async", # sequential | thread | async | dependency
|
||||
max_workers=8, # thread 策略的线程池大小
|
||||
concurrency_limits={"db": 2}, # 按 concurrency_key 限流
|
||||
dry_run=False, # True = 仅打印计划
|
||||
verbose=True, # True = 打印执行过程
|
||||
on_event=callback, # 状态转换回调
|
||||
state=px.JSONBackend("state.json"), # 断点续跑后端
|
||||
continue_on_error=False, # True = 单任务失败不中断整体
|
||||
)
|
||||
|
||||
策略对比
|
||||
--------
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 18 18 30 16 18
|
||||
|
||||
* - 策略
|
||||
- 并发模型
|
||||
- 适用场景
|
||||
- 同步任务
|
||||
- 异步任务
|
||||
* - ``sequential``
|
||||
- 串行
|
||||
- 调试、CPU 密集
|
||||
- 直接调用
|
||||
- 事件循环
|
||||
* - ``thread``
|
||||
- 线程池
|
||||
- I/O 密集同步
|
||||
- 线程池
|
||||
- 不支持
|
||||
* - ``async``
|
||||
- 事件循环
|
||||
- I/O 密集异步
|
||||
- 卸载到线程池
|
||||
- 事件循环
|
||||
* - ``dependency``
|
||||
- 依赖驱动
|
||||
- 最大化并行度
|
||||
- 卸载到线程池
|
||||
- 事件循环
|
||||
|
||||
所有策略都遵循 ``RetryPolicy``、``timeout``、上下文注入、状态后端、``concurrency_limits``,
|
||||
并发出 ``TaskEvent``(RUNNING/SUCCESS/FAILED/SKIPPED)。``dependency`` 策略无层屏障:
|
||||
任务在其所有硬依赖完成后立即启动。
|
||||
|
||||
上下文注入规则
|
||||
--------------
|
||||
|
||||
按顺序求值:
|
||||
|
||||
1. **标注为 ``Context``** 的参数 → 接收完整上游结果映射
|
||||
2. **名称匹配依赖** 的参数 → 接收该依赖的结果(含软依赖,缺失时注入默认值)
|
||||
3. **``**kwargs``** 参数 → 接收所有依赖结果(dict)
|
||||
4. **``TaskSpec.args`` / ``kwargs``** → 为非依赖参数提供静态值
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
def aggregate(ctx: px.Context) -> Dict[str, Any]:
|
||||
"""ctx 包含所有 depends_on 任务的返回值。"""
|
||||
return dict(ctx)
|
||||
|
||||
def merge(fetch_a: str, fetch_b: str) -> str:
|
||||
"""fetch_a / fetch_b 自动注入。"""
|
||||
return fetch_a + fetch_b
|
||||
|
||||
断点续跑
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from pyflowx import JSONBackend
|
||||
|
||||
backend = JSONBackend("state.json", ttl=3600)
|
||||
report = px.run(graph, strategy="sequential", state=backend)
|
||||
|
||||
``run()`` 内部以 ``backend.batch()`` 包裹整个执行:所有 ``save`` 延迟到运行结束时统一落盘一次。
|
||||
|
||||
缓存键:默认存储键为任务名。配置 ``cache_key`` 函数后,键为 ``"name:cache_key_value"``。
|
||||
|
||||
完整 API 说明详见 :doc:`/api`。
|
||||
@@ -0,0 +1,50 @@
|
||||
Graph —— DAG 构建
|
||||
=================
|
||||
|
||||
``Graph`` 管理任务集合,提供建构建、校验、分层、可视化能力。
|
||||
|
||||
构建方式
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# 图级默认值:TaskSpec 字段为 None 时回退
|
||||
defaults = px.GraphDefaults(retry=px.RetryPolicy(max_attempts=2), timeout=60.0)
|
||||
|
||||
graph = px.Graph.from_specs([...], defaults=defaults) # 整批校验(推荐)
|
||||
|
||||
# 或增量构建
|
||||
graph = px.Graph(defaults=defaults)
|
||||
graph.add(px.TaskSpec("a", fn_a))
|
||||
graph.add(px.TaskSpec("b", fn_b, ("a",)))
|
||||
|
||||
常用方法
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
graph.validate() # 显式校验(环检测)
|
||||
graph.layers() # 拓扑分层(Kahn 算法)
|
||||
graph.to_mermaid() # Mermaid 可视化
|
||||
graph.describe() # 人类可读摘要
|
||||
graph.subgraph(("api",)) # 按标签切片
|
||||
graph.subgraph_by_names(("a", "b")) # 按名称切片
|
||||
graph.map("fetch", [1, 2, 3], lambda i: TaskSpec(f"fetch_{i}", ...)) # 批量 fan-out
|
||||
|
||||
图组合
|
||||
------
|
||||
|
||||
``compose`` / ``GraphComposer`` 把带字符串引用的多个图展开为纯 ``Graph``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
graphs = {
|
||||
"build": px.Graph.from_specs([px.TaskSpec("b", cmd=["echo", "b"])]),
|
||||
"all": px.Graph.from_specs(["build", px.TaskSpec("t", cmd=["echo", "t"])]),
|
||||
}
|
||||
resolved = px.compose(graphs) # "all" 图中的 "build" 引用被展开
|
||||
|
||||
引用格式:``"command_name"``(整个图)或 ``"command_name.task_name"``(特定任务)。
|
||||
``CliRunner`` 内部自动调用 ``compose``。
|
||||
|
||||
完整方法说明详见 :doc:`/api`。
|
||||
@@ -0,0 +1,89 @@
|
||||
TaskSpec —— 任务描述
|
||||
=====================
|
||||
|
||||
``TaskSpec`` 是不可变的任务描述符(``Generic[T]``,返回类型一路传到 ``RunReport``),是唯一需要配置的东西。
|
||||
|
||||
主要参数说明:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
px.TaskSpec(
|
||||
name="fetch_user", # 唯一标识
|
||||
fn=fetch_user, # 同步或异步函数
|
||||
cmd=["curl", "..."], # 或: 执行命令(覆盖 fn)
|
||||
depends_on=("auth",), # 硬依赖(参与拓扑分层)
|
||||
soft_depends_on=("cache",), # 软依赖(仅注入,不参与分层)
|
||||
args=(uid,), # 静态位置参数(追加在注入参数后)
|
||||
kwargs={"timeout": 30}, # 静态关键字参数
|
||||
retry=px.RetryPolicy(max_attempts=3, delay=1.0, backoff=2.0),
|
||||
timeout=30.0, # 超时秒数(None = 不限制)
|
||||
tags=("api", "user"), # 自由标签,用于子图过滤
|
||||
conditions=(is_prod,), # 条件函数列表(全部为 True 才执行)
|
||||
priority=10, # 同层内优先级(高优先执行,默认 0)
|
||||
concurrency_key="db", # 并发分组键(配合 concurrency_limits 限流)
|
||||
cache_key=lambda ctx: str(ctx.get("uid")), # 缓存键函数
|
||||
hooks=px.TaskHooks(pre_run=..., post_run=..., on_failure=...),
|
||||
cwd=Path("/tmp"), # 命令工作目录(仅 cmd 模式)
|
||||
env={"DEBUG": "1"}, # 环境变量覆盖
|
||||
verbose=True, # 打印命令输出(仅 cmd 模式)
|
||||
skip_if_missing=True, # 命令不存在时自动跳过(仅 list[str] cmd)
|
||||
allow_upstream_skip=False, # 上游 SKIPPED/FAILED 时是否仍执行
|
||||
continue_on_error=False, # 本任务失败是否不中断整体
|
||||
)
|
||||
|
||||
两种任务形态
|
||||
------------
|
||||
|
||||
- **函数任务**(``fn``):普通 Python 函数,参数名驱动自动注入
|
||||
- **命令任务**(``cmd``):执行外部命令,支持 ``list[str]``、``str``(shell)、``Callable`` 三种形态
|
||||
|
||||
``skip_if_missing=True`` 时,``list[str]`` 类型的 ``cmd`` 会通过 ``shutil.which`` 检查命令是否存在,不存在则跳过任务(标记为 ``SKIPPED``)而非失败。
|
||||
|
||||
重试策略
|
||||
--------
|
||||
|
||||
``RetryPolicy`` 配置重试次数、延迟、退避:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
retry = px.RetryPolicy(
|
||||
max_attempts=3, # 最大尝试次数
|
||||
delay=1.0, # 初始延迟秒数
|
||||
backoff=2.0, # 退避倍数
|
||||
jitter=0.1, # 随机抖动(避免惊群)
|
||||
retry_on=(ConnectionError,), # 仅对这些异常重试
|
||||
)
|
||||
|
||||
任务钩子
|
||||
--------
|
||||
|
||||
``TaskHooks`` 在任务生命周期触发(异常仅记录,不影响任务状态):
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
hooks = px.TaskHooks(
|
||||
pre_run=lambda spec: print(f"start {spec.name}"),
|
||||
post_run=lambda spec, value: print(f"done {spec.name}"),
|
||||
on_failure=lambda spec, exc: alert(spec.name, exc),
|
||||
)
|
||||
px.TaskSpec("task", fn=work, hooks=hooks)
|
||||
|
||||
任务模板
|
||||
--------
|
||||
|
||||
``task_template`` 工厂批量生成相似 TaskSpec:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
fetch = px.task_template(
|
||||
fn=fetch_url,
|
||||
retry=px.RetryPolicy(max_attempts=5),
|
||||
timeout=30.0,
|
||||
tags=("api",),
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
fetch("users", url="https://api.example.com/users"),
|
||||
fetch("posts", url="https://api.example.com/posts"),
|
||||
])
|
||||
|
||||
完整字段说明详见 :doc:`/api`。
|
||||
@@ -0,0 +1,164 @@
|
||||
YAML 任务编排
|
||||
=============
|
||||
|
||||
PyFlowX 支持 GitHub Actions 风格的声明式 YAML 任务编排,从 YAML 文件直接加载任务图。
|
||||
|
||||
编程式 API
|
||||
----------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# 从 YAML 文件加载任务图
|
||||
graph = px.Graph.from_yaml("pipeline.yaml")
|
||||
report = px.run(graph, strategy="thread")
|
||||
|
||||
# 或用函数式 API
|
||||
graph = px.load_yaml("pipeline.yaml")
|
||||
|
||||
# 从字符串解析
|
||||
graph = px.parse_yaml_string("""
|
||||
jobs:
|
||||
hello:
|
||||
cmd: ["echo", "hello"]
|
||||
""")
|
||||
|
||||
YAML Schema
|
||||
-----------
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
strategy: thread # 图级默认策略
|
||||
defaults: # 图级默认值
|
||||
retry: {max_attempts: 3}
|
||||
verbose: true
|
||||
env: {CI: "true"}
|
||||
|
||||
variables: # 变量定义 (可在 cmd/env 中 ${VAR} 引用)
|
||||
OUTPUT: "dist"
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
cmd: ["git", "clone", "..."]
|
||||
runs-on: linux
|
||||
|
||||
build:
|
||||
needs: [setup] # 依赖列表
|
||||
cmd: ["python", "-m", "build"]
|
||||
timeout: 300
|
||||
retry: {max_attempts: 2, delay: 1.0}
|
||||
|
||||
test:
|
||||
needs: [build]
|
||||
cmd: ["python${{ matrix.version }}", "-m", "pytest"]
|
||||
strategy:
|
||||
matrix: # 笛卡尔积展开为 6 个任务
|
||||
version: ["3.8", "3.9", "3.10"]
|
||||
os: ["linux", "macos"]
|
||||
if: "env.CI" # 条件: 环境变量存在
|
||||
|
||||
lint:
|
||||
needs: [build]
|
||||
cmd: ["ruff", "check"]
|
||||
if: "env.CI == 'true'"
|
||||
|
||||
deploy:
|
||||
needs: [test, lint] # 矩阵依赖自动展开
|
||||
cmd: ["twine", "upload"]
|
||||
if: "env.DEPLOY_TOKEN != ''"
|
||||
allow-upstream-skip: true
|
||||
concurrency-key: deploy_lock
|
||||
|
||||
字段映射
|
||||
--------
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 30 30 40
|
||||
|
||||
* - YAML 字段
|
||||
- TaskSpec 字段
|
||||
- 说明
|
||||
* - ``jobs.<id>``
|
||||
- ``name``
|
||||
- job ID 作为任务名
|
||||
* - ``cmd`` / ``run``
|
||||
- ``cmd``
|
||||
- ``cmd`` 为列表形式,``run`` 为 shell 字符串
|
||||
* - ``needs``
|
||||
- ``depends_on``
|
||||
- 依赖列表(矩阵任务自动展开)
|
||||
* - ``if``
|
||||
- ``conditions``
|
||||
- ``success()`` / ``always()`` / ``env.VAR`` / ``env.VAR == 'x'``
|
||||
* - ``strategy.matrix``
|
||||
- 矩阵扇出
|
||||
- 笛卡尔积展开为多个任务
|
||||
* - ``${{ matrix.key }}``
|
||||
- 占位符
|
||||
- 在 cmd/run/cwd/env 中替换
|
||||
* - ``timeout``
|
||||
- ``timeout``
|
||||
- 超时秒数
|
||||
* - ``retry``
|
||||
- ``retry``
|
||||
- ``{max_attempts, delay, backoff, jitter}``
|
||||
* - ``cwd``
|
||||
- ``cwd``
|
||||
- 工作目录
|
||||
* - ``env``
|
||||
- ``env``
|
||||
- 环境变量
|
||||
* - ``verbose``
|
||||
- ``verbose``
|
||||
- 详细输出
|
||||
* - ``continue-on-error``
|
||||
- ``continue_on_error``
|
||||
- 失败不中止整图
|
||||
* - ``skip-if-missing``
|
||||
- ``skip_if_missing``
|
||||
- 命令不存在时跳过
|
||||
* - ``allow-upstream-skip``
|
||||
- ``allow_upstream_skip``
|
||||
- 上游跳过时仍执行
|
||||
* - ``priority``
|
||||
- ``priority``
|
||||
- 同层优先级
|
||||
* - ``concurrency-key``
|
||||
- ``concurrency_key``
|
||||
- 并发限制键
|
||||
* - ``tags``
|
||||
- ``tags``
|
||||
- 自由标签
|
||||
* - ``runs-on``
|
||||
- ``tags``(追加)
|
||||
- 运行环境标签
|
||||
|
||||
CLI 配置段(``cli:``)
|
||||
----------------------
|
||||
|
||||
工具 YAML 还可定义 ``cli:`` 段,声明命令行参数 schema,由 ``pf`` 自动解析:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
cli:
|
||||
description: "FileDate - 文件日期处理工具"
|
||||
usage: "pf filedate <command> [files...]"
|
||||
subcommands:
|
||||
add:
|
||||
help: "添加日期前缀"
|
||||
positional:
|
||||
- name: FILES
|
||||
nargs: "+"
|
||||
type: path
|
||||
help: "文件路径"
|
||||
options:
|
||||
- name: CLEAR
|
||||
flag: "--clear"
|
||||
action: store_true
|
||||
help: "清除已有日期前缀"
|
||||
|
||||
支持的 ``type``:``str`` / ``int`` / ``float`` / ``path``。
|
||||
|
||||
完整 API 说明详见 :doc:`/api`。
|
||||
@@ -0,0 +1,56 @@
|
||||
PyFlowX 文档
|
||||
============
|
||||
|
||||
PyFlowX 是一个轻量、类型安全的 DAG 任务调度器:**参数名就是依赖声明**。
|
||||
无需装饰器、无需样板包装器,写一个普通函数,框架按参数名自动注入上游结果。
|
||||
|
||||
特性
|
||||
----
|
||||
|
||||
- **零样板** —— 参数名即依赖,框架自动注入上游结果
|
||||
- **四种执行策略** —— sequential(串行)、thread(线程池)、async(事件循环)、dependency(依赖驱动,最大化并行)
|
||||
- **类型安全** —— ``TaskSpec[T]`` 把返回类型一路传到 ``RunReport``
|
||||
- **DAG 校验** —— 构建时即时校验重名、缺失依赖、环
|
||||
- **自动分层** —— Kahn 算法分组,同层任务可并行
|
||||
- **重试与超时** —— 每个任务独立配置 ``RetryPolicy`` 与 ``timeout``
|
||||
- **并发限制** —— ``concurrency_key`` + ``concurrency_limits`` 按组限流
|
||||
- **断点续跑** —— ``MemoryBackend`` / ``JSONBackend``,成功结果可缓存复用
|
||||
- **命令任务** —— ``cmd`` 参数直接执行外部命令
|
||||
- **条件执行** —— ``conditions`` 按平台、环境变量等条件跳过任务
|
||||
- **YAML 任务编排** —— GitHub Actions 风格声明式任务图
|
||||
- **pf 统一 CLI** —— ``pf <tool> [command]`` 调用所有工具
|
||||
- **最小依赖** —— 仅依赖标准库 + PyYAML
|
||||
|
||||
文档导航
|
||||
--------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: 入门
|
||||
|
||||
installation
|
||||
quickstart
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: 用户指南
|
||||
|
||||
guide/task
|
||||
guide/graph
|
||||
guide/execution
|
||||
guide/yaml
|
||||
guide/cli
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: 参考
|
||||
|
||||
api
|
||||
changelog
|
||||
|
||||
索引
|
||||
----
|
||||
|
||||
* :ref:`genindex`
|
||||
* :ref:`modindex`
|
||||
* :ref:`search`
|
||||
@@ -0,0 +1,51 @@
|
||||
安装
|
||||
====
|
||||
|
||||
PyFlowX 支持 Python 3.8+,仅依赖标准库与 PyYAML(3.8 额外需要 ``graphlib_backport`` 和 ``typing-extensions``)。
|
||||
|
||||
pip 安装
|
||||
--------
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install pyflowx
|
||||
|
||||
uv 安装
|
||||
-------
|
||||
|
||||
推荐使用 `uv <https://docs.astral.sh/uv/>`_:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
uv add pyflowx
|
||||
|
||||
可选依赖
|
||||
--------
|
||||
|
||||
``office`` —— PDF/图片处理(pdftool、screenshot 等工具需要):
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install pyflowx[office]
|
||||
|
||||
``dev`` —— 开发工具链(ruff、pyrefly、pytest、tox 等):
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install pyflowx[dev]
|
||||
|
||||
验证安装
|
||||
--------
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pf --version
|
||||
|
||||
输出示例::
|
||||
|
||||
PyFlowX 0.4.5
|
||||
|
||||
下一步
|
||||
------
|
||||
|
||||
前往 :doc:`quickstart` 开始使用。
|
||||
@@ -0,0 +1,87 @@
|
||||
快速上手
|
||||
========
|
||||
|
||||
核心思想:**参数名即依赖**。写一个普通函数,参数名匹配上游任务名,框架自动注入结果。
|
||||
|
||||
最小示例
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
def extract() -> list[int]:
|
||||
return [1, 2, 3]
|
||||
|
||||
# 参数名 extract 自动匹配上游任务名 → 自动注入
|
||||
def double(extract: list[int]) -> list[int]:
|
||||
return [x * 2 for x in extract]
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("extract", extract),
|
||||
px.TaskSpec("double", double, ("extract",)),
|
||||
])
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
print(report["double"]) # [2, 4, 6]
|
||||
|
||||
三种任务形态
|
||||
------------
|
||||
|
||||
1. **函数任务**(``fn``):普通 Python 函数,参数名驱动自动注入
|
||||
2. **命令任务**(``cmd``):执行外部命令,支持 ``list[str]`` / ``str``(shell)/ ``Callable``
|
||||
3. **YAML 声明式**:从 YAML 文件加载任务图
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("list", cmd=["ls", "-la"]),
|
||||
px.TaskSpec("greet", fn=lambda: "hello"),
|
||||
])
|
||||
|
||||
执行策略
|
||||
--------
|
||||
|
||||
PyFlowX 提供四种执行策略:
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 20 20 60
|
||||
|
||||
* - 策略
|
||||
- 并发模型
|
||||
- 适用场景
|
||||
* - ``sequential``
|
||||
- 串行
|
||||
- 调试、CPU 密集
|
||||
* - ``thread``
|
||||
- 线程池
|
||||
- I/O 密集同步
|
||||
* - ``async``
|
||||
- 事件循环
|
||||
- I/O 密集异步
|
||||
* - ``dependency``
|
||||
- 依赖驱动
|
||||
- 最大化并行度(默认推荐)
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
report = px.run(graph, strategy="dependency")
|
||||
|
||||
结果访问
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
report["task_name"] # 任务返回值
|
||||
report.result_of("task_name") # 完整 TaskResult
|
||||
report.success # 整体是否成功
|
||||
report.summary() # 统计字典
|
||||
report.failed_tasks() # 失败任务名列表
|
||||
|
||||
下一步
|
||||
------
|
||||
|
||||
- :doc:`guide/task` —— TaskSpec 详细配置
|
||||
- :doc:`guide/yaml` —— YAML 声明式任务编排
|
||||
- :doc:`guide/cli` —— ``pf`` 统一 CLI 入口
|
||||
+29
-29
@@ -6,43 +6,38 @@ classifiers = [
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3.14",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Topic :: Software Development :: Libraries :: Application Frameworks",
|
||||
]
|
||||
dependencies = ["graphlib_backport >= 1.0.0; python_version < '3.9'"]
|
||||
dependencies = [
|
||||
"graphlib_backport >= 1.0.0; python_version < '3.9'",
|
||||
"pyyaml>=6.0.1",
|
||||
"typing-extensions>=4.13.2; python_version < '3.13'",
|
||||
]
|
||||
description = "Lightweight, type-safe DAG task scheduler with multi-strategy execution."
|
||||
keywords = ["async", "dag", "scheduler", "task", "workflow"]
|
||||
license = { text = "MIT" }
|
||||
name = "pyflowx"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.8"
|
||||
version = "0.2.2"
|
||||
version = "0.4.6"
|
||||
|
||||
[project.scripts]
|
||||
autofmt = "pyflowx.cli.autofmt:main"
|
||||
bumpversion = "pyflowx.cli.bumpversion:main"
|
||||
clr = "pyflowx.cli.clearscreen:main"
|
||||
emlman = "pyflowx.cli.emlmanager:main"
|
||||
envlinux = "pyflowx.cli.envlinux:main"
|
||||
envpy = "pyflowx.cli.envpy:main"
|
||||
envqt = "pyflowx.cli.envqt:main"
|
||||
envrs = "pyflowx.cli.envrs:main"
|
||||
filedate = "pyflowx.cli.filedate:main"
|
||||
filelvl = "pyflowx.cli.filelevel:main"
|
||||
foldback = "pyflowx.cli.folderback:main"
|
||||
foldzip = "pyflowx.cli.folderzip:main"
|
||||
gitt = "pyflowx.cli.gittool:main"
|
||||
hfdown = "pyflowx.cli.hfdownload:main"
|
||||
lscalc = "pyflowx.cli.lscalc:main"
|
||||
packtool = "pyflowx.cli.packtool:main"
|
||||
pdftool = "pyflowx.cli.pdftool:main"
|
||||
piptool = "pyflowx.cli.piptool:main"
|
||||
pymake = "pyflowx.cli.pymake:main"
|
||||
scrcap = "pyflowx.cli.screenshot:main"
|
||||
sshcopy = "pyflowx.cli.sshcopyid:main"
|
||||
taskk = "pyflowx.cli.taskkill:main"
|
||||
wch = "pyflowx.cli.which:main"
|
||||
dockercmd = "pyflowx.cli.dev.dockercmd:main"
|
||||
emlman = "pyflowx.cli.emlmanager:main"
|
||||
msdown = "pyflowx.cli.llm.msdownload:main"
|
||||
pf = "pyflowx.cli.pf:main"
|
||||
pxp = "pyflowx.cli.profiler:main"
|
||||
sglang = "pyflowx.cli.llm.sglang:main"
|
||||
yamlrun = "pyflowx.cli.yamlrun:main"
|
||||
# dev
|
||||
envdev = "pyflowx.cli.dev.envdev:main"
|
||||
# system
|
||||
clr = "pyflowx.cli.system.clearscreen:main"
|
||||
taskk = "pyflowx.cli.system.taskkill:main"
|
||||
wch = "pyflowx.cli.system.which:main"
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
@@ -59,7 +54,9 @@ dev = [
|
||||
"ruff>=0.8.0",
|
||||
"tox-uv>=1.13.1",
|
||||
"tox>=4.25.0",
|
||||
"types-PyYAML>=6.0.12",
|
||||
]
|
||||
docs = ["myst-parser>=3.0", "sphinx-rtd-theme>=2.0", "sphinx>=7.0"]
|
||||
office = [
|
||||
"pillow>=10.4.0",
|
||||
"pymupdf>=1.24.11",
|
||||
@@ -71,6 +68,9 @@ office = [
|
||||
build-backend = "hatchling.build"
|
||||
requires = ["hatchling"]
|
||||
|
||||
[tool.uv]
|
||||
required-version = ">=0.5.0"
|
||||
|
||||
[[tool.uv.index]]
|
||||
default = true
|
||||
url = "https://mirrors.aliyun.com/pypi/simple/"
|
||||
@@ -85,12 +85,12 @@ packages = ["src/pyflowx"]
|
||||
pyflowx = { workspace = true }
|
||||
|
||||
[dependency-groups]
|
||||
dev = ["pyflowx[dev,office]"]
|
||||
dev = ["pyflowx[dev,docs,office]"]
|
||||
|
||||
[tool.coverage.run]
|
||||
branch = true
|
||||
concurrency = ["thread"]
|
||||
omit = ["src/pyflowx/examples/*", "tests/*"]
|
||||
omit = ["src/pyflowx/cli/*", "src/pyflowx/examples/*", "tests/*"]
|
||||
source = ["pyflowx"]
|
||||
|
||||
[tool.coverage.report]
|
||||
@@ -100,7 +100,7 @@ exclude_lines = [
|
||||
"pragma: no cover",
|
||||
"raise NotImplementedError",
|
||||
]
|
||||
fail_under = 80
|
||||
fail_under = 95
|
||||
show_missing = true
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
@@ -146,6 +146,6 @@ select = [
|
||||
"**/tests/**" = ["ARG001", "ARG002"]
|
||||
|
||||
[tool.pyrefly]
|
||||
preset = "basic"
|
||||
preset = "strict"
|
||||
project-includes = ["**/*.ipynb", "**/*.py*"]
|
||||
python-version = "3.8"
|
||||
|
||||
+50
-17
@@ -4,9 +4,15 @@
|
||||
--------
|
||||
* :class:`TaskSpec` —— 不可变任务描述符(唯一需要配置的东西)。
|
||||
* :class:`Graph` —— 由一组 spec 构建的 DAG;负责校验、分层、可视化。
|
||||
* :func:`run` —— 以 ``sequential`` / ``thread`` / ``async`` 策略执行图。
|
||||
* :func:`run` ——以 ``sequential`` / ``thread`` / ``async`` / ``dependency``
|
||||
策略执行图。
|
||||
* :class:`RunReport` —— 类型化、可查询的运行结果。
|
||||
* :class:`Context` —— 整体上下文注入的标注标记。
|
||||
* :class:`RetryPolicy` —— 重试策略(max_attempts/delay/backoff/jitter/retry_on)。
|
||||
* :class:`TaskHooks` —— 任务生命周期钩子(pre_run/post_run/on_failure)。
|
||||
* :class:`GraphDefaults` —— 图级默认值。
|
||||
* :func:`compose` —— 编程式组合多图。
|
||||
* :func:`task_template` —— 批量生成相似 TaskSpec 的工厂。
|
||||
* 状态后端::class:`StateBackend`、:class:`MemoryBackend`、:class:`JSONBackend`。
|
||||
|
||||
快速上手
|
||||
@@ -18,7 +24,7 @@
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("extract", extract),
|
||||
px.TaskSpec("double", double, ("extract",)),
|
||||
px.TaskSpec("double", double, depends_on=("extract",)),
|
||||
])
|
||||
report = px.run(graph, strategy="sequential")
|
||||
print(report["double"]) # [2, 4, 6]
|
||||
@@ -29,23 +35,18 @@
|
||||
from pyflowx.conditions import IS_WINDOWS, BuiltinConditions
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
# 使用命令列表
|
||||
px.TaskSpec("list_files", cmd=["ls", "-la"]),
|
||||
# 使用 shell 命令
|
||||
px.TaskSpec("check_git", cmd="git status"),
|
||||
# 条件执行:仅在 Windows 上运行
|
||||
px.TaskSpec(
|
||||
"win_only",
|
||||
cmd=["dir"],
|
||||
conditions=(IS_WINDOWS,)
|
||||
),
|
||||
# 条件执行:仅在 git 已安装时运行
|
||||
px.TaskSpec(
|
||||
"git_check",
|
||||
cmd=["git", "--version"],
|
||||
conditions=(BuiltinConditions.HAS_INSTALLED("git"),)
|
||||
),
|
||||
# 命令不存在时自动跳过(而非失败)
|
||||
px.TaskSpec(
|
||||
"optional_build",
|
||||
cmd=["maturin", "build"],
|
||||
@@ -57,6 +58,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .command import run_command
|
||||
from .compose import GraphComposer, compose
|
||||
from .conditions import (
|
||||
IS_LINUX,
|
||||
IS_MACOS,
|
||||
@@ -78,13 +81,28 @@ from .errors import (
|
||||
TaskTimeoutError,
|
||||
)
|
||||
from .executors import Strategy, run
|
||||
from .graph import Graph
|
||||
from .graph import Graph, GraphDefaults
|
||||
from .profiling import ProfileReport, TaskProfile
|
||||
from .registry import FnRegistry, get_fn, has_fn, register_fn
|
||||
from .report import RunReport
|
||||
from .runner import CliExitCode, CliRunner
|
||||
from .storage import JSONBackend, MemoryBackend, StateBackend
|
||||
from .task import TaskCmd, TaskEvent, TaskResult, TaskSpec, TaskStatus
|
||||
from .task import (
|
||||
CacheKeyFn,
|
||||
RetryPolicy,
|
||||
TaskCmd,
|
||||
TaskEvent,
|
||||
TaskHooks,
|
||||
TaskResult,
|
||||
TaskSpec,
|
||||
TaskStatus,
|
||||
cmd,
|
||||
task,
|
||||
task_template,
|
||||
)
|
||||
from .yaml_loader import YamlLoadError, build_cli_parser, load_yaml, parse_yaml_string, run_cli, run_yaml
|
||||
|
||||
__version__ = "0.2.2"
|
||||
__version__ = "0.4.6"
|
||||
|
||||
__all__ = [
|
||||
"IS_LINUX",
|
||||
@@ -92,38 +110,53 @@ __all__ = [
|
||||
"IS_POSIX",
|
||||
"IS_WINDOWS",
|
||||
"BuiltinConditions",
|
||||
"CacheKeyFn",
|
||||
"CliExitCode",
|
||||
# CLI 运行器
|
||||
"CliRunner",
|
||||
# 条件判断
|
||||
"Condition",
|
||||
"Constants",
|
||||
"Context",
|
||||
"CycleError",
|
||||
"DuplicateTaskError",
|
||||
"FnRegistry",
|
||||
"Graph",
|
||||
"GraphComposer",
|
||||
"GraphDefaults",
|
||||
"InjectionError",
|
||||
"JSONBackend",
|
||||
"MemoryBackend",
|
||||
"MissingDependencyError",
|
||||
# 错误
|
||||
"ProfileReport",
|
||||
"PyFlowXError",
|
||||
"RetryPolicy",
|
||||
"RunReport",
|
||||
# 状态后端
|
||||
"StateBackend",
|
||||
"StorageError",
|
||||
"Strategy",
|
||||
"TaskCmd",
|
||||
"TaskEvent",
|
||||
"TaskFailedError",
|
||||
"TaskHooks",
|
||||
"TaskProfile",
|
||||
"TaskResult",
|
||||
# 核心类型
|
||||
"TaskSpec",
|
||||
"TaskStatus",
|
||||
"TaskTimeoutError",
|
||||
# 辅助(高级)
|
||||
"YamlLoadError",
|
||||
"build_call_args",
|
||||
"build_cli_parser",
|
||||
"cmd",
|
||||
"compose",
|
||||
"describe_injection",
|
||||
# 执行
|
||||
"get_fn",
|
||||
"has_fn",
|
||||
"load_yaml",
|
||||
"parse_yaml_string",
|
||||
"register_fn",
|
||||
"run",
|
||||
"run_cli",
|
||||
"run_command",
|
||||
"run_yaml",
|
||||
"task",
|
||||
"task_template",
|
||||
]
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
"""CLI 工具模块.
|
||||
|
||||
提供各种命令行工具的入口点.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# 自动格式化工具
|
||||
from pyflowx.cli.autofmt import main as autofmt_main
|
||||
from pyflowx.cli.bumpversion import main as bumpversion_main
|
||||
from pyflowx.cli.clearscreen import main as clearscreen_main
|
||||
|
||||
# EML 邮件管理工具
|
||||
from pyflowx.cli.emlmanager import main as emlmanager_main
|
||||
|
||||
# EML 邮件管理工具
|
||||
from pyflowx.cli.emlmanager import main as emlmanager_web_main
|
||||
from pyflowx.cli.envpy import main as envpy_main
|
||||
from pyflowx.cli.envqt import main as envqt_main
|
||||
from pyflowx.cli.envrs import main as envrs_main
|
||||
|
||||
# 文件工具
|
||||
from pyflowx.cli.filedate import main as filedate_main
|
||||
from pyflowx.cli.filelevel import main as filelevel_main
|
||||
from pyflowx.cli.folderback import main as folderback_main
|
||||
from pyflowx.cli.folderzip import main as folderzip_main
|
||||
|
||||
# Git 工具
|
||||
from pyflowx.cli.gittool import main as gittool_main
|
||||
|
||||
# 仿真工具
|
||||
from pyflowx.cli.lscalc import main as lscalc_main
|
||||
|
||||
# 打包工具
|
||||
from pyflowx.cli.packtool import main as packtool_main
|
||||
|
||||
# PDF 工具
|
||||
from pyflowx.cli.pdftool import main as pdftool_main
|
||||
|
||||
# 开发工具
|
||||
from pyflowx.cli.piptool import main as piptool_main
|
||||
from pyflowx.cli.pymake import main as pymake_main
|
||||
from pyflowx.cli.screenshot import main as screenshot_main
|
||||
from pyflowx.cli.sshcopyid import main as sshcopyid_main
|
||||
|
||||
__all__ = [
|
||||
# 自动格式化工具
|
||||
"autofmt_main",
|
||||
"bumpversion_main",
|
||||
"clearscreen_main",
|
||||
# EML 邮件管理工具
|
||||
"emlmanager_main",
|
||||
"emlmanager_web_main",
|
||||
"envpy_main",
|
||||
"envqt_main",
|
||||
"envrs_main",
|
||||
# 文件工具
|
||||
"filedate_main",
|
||||
"filelevel_main",
|
||||
"folderback_main",
|
||||
"folderzip_main",
|
||||
# Git 工具
|
||||
"gittool_main",
|
||||
# 仿真工具
|
||||
"lscalc_main",
|
||||
# 打包工具
|
||||
"packtool_main",
|
||||
# PDF 工具
|
||||
"pdftool_main",
|
||||
# 开发工具
|
||||
"piptool_main",
|
||||
"pymake_main",
|
||||
"screenshot_main",
|
||||
"sshcopyid_main",
|
||||
# 系统工具
|
||||
"taskkill_main",
|
||||
"which_main",
|
||||
]
|
||||
|
||||
@@ -1,282 +0,0 @@
|
||||
"""自动格式化工具模块.
|
||||
|
||||
提供 Python 代码自动格式化的常用功能封装,
|
||||
支持 docstring 自动生成、pyproject.toml 配置同步等功能.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
try:
|
||||
import tomllib # noqa: F401
|
||||
|
||||
HAS_TOMLLIB = True
|
||||
except ImportError:
|
||||
HAS_TOMLLIB = False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
IGNORE_PATTERNS = [
|
||||
"__pycache__",
|
||||
"*.pyc",
|
||||
"*.pyo",
|
||||
".git",
|
||||
".venv",
|
||||
".idea",
|
||||
".vscode",
|
||||
"*.egg-info",
|
||||
"dist",
|
||||
"build",
|
||||
".pytest_cache",
|
||||
".tox",
|
||||
".mypy_cache",
|
||||
]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def format_with_ruff(target: Path, fix: bool = True) -> None:
|
||||
"""使用 ruff 格式化代码.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
target : Path
|
||||
目标路径
|
||||
fix : bool
|
||||
是否自动修复
|
||||
"""
|
||||
cmd = ["ruff", "format", str(target)]
|
||||
if fix:
|
||||
cmd.append("--fix")
|
||||
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"ruff format 完成: {target}")
|
||||
|
||||
|
||||
def lint_with_ruff(target: Path, fix: bool = True) -> None:
|
||||
"""使用 ruff 检查代码.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
target : Path
|
||||
目标路径
|
||||
fix : bool
|
||||
是否自动修复
|
||||
"""
|
||||
cmd = ["ruff", "check", str(target)]
|
||||
if fix:
|
||||
cmd.extend(["--fix", "--unsafe-fixes"])
|
||||
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"ruff check 完成: {target}")
|
||||
|
||||
|
||||
def add_docstring(file_path: Path, docstring: str) -> bool:
|
||||
"""为文件添加 docstring.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_path : Path
|
||||
文件路径
|
||||
docstring : str
|
||||
docstring 内容
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
是否成功添加
|
||||
"""
|
||||
try:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
tree = ast.parse(content)
|
||||
|
||||
# 检查是否已有 docstring
|
||||
first_node = tree.body[0] if tree.body else None
|
||||
if first_node and isinstance(first_node, ast.Expr) and isinstance(first_node.value, ast.Constant):
|
||||
return False
|
||||
|
||||
# 添加 docstring
|
||||
lines = content.splitlines()
|
||||
doc_lines = docstring.splitlines()
|
||||
doc_lines.append("")
|
||||
new_content = "\n".join(doc_lines + lines)
|
||||
|
||||
file_path.write_text(new_content, encoding="utf-8")
|
||||
print(f"添加 docstring: {file_path}")
|
||||
return True
|
||||
|
||||
except (OSError, UnicodeDecodeError, SyntaxError) as e:
|
||||
print(f"处理失败: {file_path} - {e}")
|
||||
return False
|
||||
|
||||
|
||||
def generate_module_docstring(file_path: Path) -> str:
|
||||
"""生成模块 docstring.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_path : Path
|
||||
文件路径
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
生成的 docstring
|
||||
"""
|
||||
stem = file_path.stem
|
||||
parent = file_path.parent.name
|
||||
|
||||
# 关键词匹配
|
||||
keywords = {
|
||||
"cli": f"Command-line interface for {parent}",
|
||||
"gui": f"Graphical user interface for {parent}",
|
||||
"core": f"Core functionality for {parent}",
|
||||
"util": f"Utility functions for {parent}",
|
||||
"model": f"Data models for {parent}",
|
||||
"test": f"Tests for {parent}",
|
||||
}
|
||||
|
||||
for key, desc in keywords.items():
|
||||
if key in stem.lower():
|
||||
return f'"""{desc}."""'
|
||||
|
||||
return f'"""{stem.replace("_", " ").title()} module."""'
|
||||
|
||||
|
||||
def auto_add_docstrings(root_dir: Path) -> int:
|
||||
"""自动为所有 Python 文件添加 docstring.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root_dir : Path
|
||||
根目录
|
||||
|
||||
Returns
|
||||
-------
|
||||
int
|
||||
添加的 docstring 数量
|
||||
"""
|
||||
count = 0
|
||||
for py_file in root_dir.rglob("*.py"):
|
||||
# 跳过忽略的文件
|
||||
if any(pattern in str(py_file) for pattern in IGNORE_PATTERNS):
|
||||
continue
|
||||
|
||||
docstring = generate_module_docstring(py_file)
|
||||
if add_docstring(py_file, docstring):
|
||||
count += 1
|
||||
|
||||
print(f"共添加 {count} 个 docstring")
|
||||
return count
|
||||
|
||||
|
||||
def sync_pyproject_config(root_dir: Path) -> None:
|
||||
"""同步 pyproject.toml 配置到子项目.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root_dir : Path
|
||||
根目录
|
||||
"""
|
||||
main_toml = root_dir / "pyproject.toml"
|
||||
if not main_toml.exists():
|
||||
print(f"主项目配置文件不存在: {main_toml}")
|
||||
return
|
||||
|
||||
# 查找所有子项目的 pyproject.toml
|
||||
sub_tomls = [p for p in root_dir.rglob("pyproject.toml") if p != main_toml and ".venv" not in str(p)]
|
||||
|
||||
if not sub_tomls:
|
||||
print("没有找到子项目的 pyproject.toml")
|
||||
return
|
||||
|
||||
print(f"找到 {len(sub_tomls)} 个子项目配置文件")
|
||||
|
||||
# 对每个子项目调用 ruff format
|
||||
for sub_toml in sub_tomls:
|
||||
subprocess.run(["ruff", "format", str(sub_toml)], check=False)
|
||||
|
||||
print("配置同步完成")
|
||||
|
||||
|
||||
def format_all(root_dir: Path) -> None:
|
||||
"""格式化所有 Python 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root_dir : Path
|
||||
根目录
|
||||
"""
|
||||
# 使用 ruff format
|
||||
subprocess.run(["ruff", "format", str(root_dir)], check=True)
|
||||
|
||||
# 使用 ruff check
|
||||
subprocess.run(["ruff", "check", "--fix", "--unsafe-fixes", str(root_dir)], check=True)
|
||||
|
||||
print(f"格式化完成: {root_dir}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""自动格式化工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="AutoFmt - 自动格式化工具",
|
||||
usage="autofmt <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# ruff format 命令
|
||||
format_parser = subparsers.add_parser("fmt", help="使用 ruff 格式化代码")
|
||||
format_parser.add_argument("--target", type=str, default=".", help="目标路径")
|
||||
|
||||
# ruff check 命令
|
||||
lint_parser = subparsers.add_parser("lint", help="使用 ruff 检查代码")
|
||||
lint_parser.add_argument("--target", type=str, default=".", help="目标路径")
|
||||
lint_parser.add_argument("--fix", action="store_true", help="自动修复")
|
||||
|
||||
# 自动添加 docstring 命令
|
||||
doc_parser = subparsers.add_parser("doc", help="自动添加 docstring")
|
||||
doc_parser.add_argument("--root-dir", type=str, default=".", help="根目录")
|
||||
|
||||
# 同步配置命令
|
||||
sync_parser = subparsers.add_parser("sync", help="同步 pyproject.toml 配置")
|
||||
sync_parser.add_argument("--root-dir", type=str, default=".", help="根目录")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "fmt":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("ruff_format", cmd=["ruff", "format", args.target], verbose=True)])
|
||||
elif args.command == "lint":
|
||||
cmd = ["ruff", "check", args.target]
|
||||
if args.fix:
|
||||
cmd.extend(["--fix", "--unsafe-fixes"])
|
||||
graph = px.Graph.from_specs([px.TaskSpec("ruff_check", cmd=cmd, verbose=True)])
|
||||
elif args.command == "doc":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("auto_docstring", fn=auto_add_docstrings, args=(Path(args.root_dir),), verbose=True)]
|
||||
)
|
||||
elif args.command == "sync":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("sync_config", fn=sync_pyproject_config, args=(Path(args.root_dir),), verbose=True)]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,261 +0,0 @@
|
||||
"""版本号自动管理工具.
|
||||
|
||||
使用 TaskSpec 模式实现, 支持语义化版本管理和多文件格式的版本号更新.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
BumpVersionType = Literal["patch", "minor", "major"]
|
||||
|
||||
# 针对不同文件类型的版本号匹配模式
|
||||
# pyproject.toml: version = "X.Y.Z" 或 version = 'X.Y.Z'
|
||||
_PYPROJECT_VERSION_PATTERN = re.compile(
|
||||
r'(?:^|\n)\s*version\s*=\s*["\']'
|
||||
r"(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
|
||||
r"(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
|
||||
r"(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?"
|
||||
r'["\']',
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
# __init__.py: __version__ = "X.Y.Z" 或 __version__ = 'X.Y.Z'
|
||||
_INIT_VERSION_PATTERN = re.compile(
|
||||
r'(?:^|\n)\s*__version__\s*=\s*["\']'
|
||||
r"(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
|
||||
r"(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
|
||||
r"(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?"
|
||||
r'["\']',
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def _get_pattern_for_file(file_name: str) -> re.Pattern[str] | None:
|
||||
"""根据文件类型获取对应的正则表达式.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_name : str
|
||||
文件名
|
||||
|
||||
Returns
|
||||
-------
|
||||
re.Pattern[str] | None
|
||||
对应的正则表达式,如果无法确定则返回 None
|
||||
"""
|
||||
if file_name == "pyproject.toml":
|
||||
return _PYPROJECT_VERSION_PATTERN
|
||||
if file_name == "__init__.py":
|
||||
return _INIT_VERSION_PATTERN
|
||||
return None
|
||||
|
||||
|
||||
def _calculate_new_version(major: int, minor: int, patch: int, part: BumpVersionType) -> str:
|
||||
"""计算新版本号.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
major : int
|
||||
当前主版本号
|
||||
minor : int
|
||||
当前次版本号
|
||||
patch : int
|
||||
当前补丁版本号
|
||||
part : BumpVersionType
|
||||
要更新的部分
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
新版本号
|
||||
"""
|
||||
if part == "major":
|
||||
return f"{major + 1}.0.0"
|
||||
if part == "minor":
|
||||
return f"{major}.{minor + 1}.0"
|
||||
return f"{major}.{minor}.{patch + 1}"
|
||||
|
||||
|
||||
def _build_replacement_string(original_match: str, new_version: str, file_name: str) -> str:
|
||||
"""构建替换字符串,保留原始格式.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
original_match : str
|
||||
原始匹配的字符串
|
||||
new_version : str
|
||||
新版本号
|
||||
file_name : str
|
||||
文件名
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
替换字符串
|
||||
"""
|
||||
quote_char = '"' if '"' in original_match else "'"
|
||||
|
||||
if file_name == "pyproject.toml":
|
||||
prefix_match = re.match(r'(\s*version\s*=\s*)["\']', original_match)
|
||||
prefix = prefix_match.group(1) if prefix_match else "version = "
|
||||
return f"{prefix}{quote_char}{new_version}{quote_char}"
|
||||
|
||||
if file_name == "__init__.py":
|
||||
prefix_match = re.match(r'(\s*__version__\s*=\s*)["\']', original_match)
|
||||
prefix = prefix_match.group(1) if prefix_match else "__version__ = "
|
||||
return f"{prefix}{quote_char}{new_version}{quote_char}"
|
||||
|
||||
return new_version
|
||||
|
||||
|
||||
def bump_file_version(file_path: Path, part: BumpVersionType = "patch") -> str | None:
|
||||
"""更新文件中的版本号.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_path : Path
|
||||
要更新的文件路径
|
||||
part : BumpVersionType
|
||||
版本部分: patch, minor, major
|
||||
|
||||
Returns
|
||||
-------
|
||||
str | None
|
||||
更新后的新版本号,如果文件中未找到版本号则返回 None
|
||||
"""
|
||||
try:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
except Exception as e:
|
||||
print(f"读取文件 {file_path} 时出错: {e}")
|
||||
raise
|
||||
|
||||
# 获取文件对应的正则表达式
|
||||
pattern = _get_pattern_for_file(file_path.name)
|
||||
|
||||
# 对于未知文件类型,尝试两种模式
|
||||
if pattern:
|
||||
match = pattern.search(content)
|
||||
else:
|
||||
match = _PYPROJECT_VERSION_PATTERN.search(content) or _INIT_VERSION_PATTERN.search(content)
|
||||
|
||||
if not match:
|
||||
print(f"文件 {file_path} 中未找到版本号模式")
|
||||
return None
|
||||
|
||||
# 提取当前版本号
|
||||
major = int(match.group("major"))
|
||||
minor = int(match.group("minor"))
|
||||
patch = int(match.group("patch"))
|
||||
|
||||
# 计算新版本号
|
||||
new_version = _calculate_new_version(major, minor, patch, part)
|
||||
|
||||
# 构建替换字符串
|
||||
original_match = match.group(0)
|
||||
replacement = _build_replacement_string(original_match, new_version, file_path.name)
|
||||
|
||||
# 更新文件内容
|
||||
content = content.replace(original_match, replacement)
|
||||
|
||||
try:
|
||||
file_path.write_text(content, encoding="utf-8")
|
||||
except Exception as e:
|
||||
print(f"更新文件 {file_path} 版本号时出错: {e}")
|
||||
raise
|
||||
|
||||
return new_version
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""版本号管理工具主函数."""
|
||||
parser = argparse.ArgumentParser(description="BumpVersion - 版本号自动管理工具")
|
||||
parser.add_argument(
|
||||
"part",
|
||||
type=str,
|
||||
nargs="?",
|
||||
default="patch",
|
||||
choices=get_args(BumpVersionType),
|
||||
help=f"版本部分: {get_args(BumpVersionType)}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-tag",
|
||||
action="store_true",
|
||||
help="提交后不创建 git tag",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
part = args.part
|
||||
|
||||
# 搜索文件,排除常见的虚拟环境和缓存目录
|
||||
ignore_dirs = {".venv", "venv", ".git", "__pycache__", ".tox", "node_modules", "build", "dist", ".eggs"}
|
||||
all_files = set()
|
||||
|
||||
for pattern in ["__init__.py", "pyproject.toml"]:
|
||||
for file in Path.cwd().rglob(pattern):
|
||||
# 检查路径中是否包含需要忽略的目录
|
||||
if not any(ignore_dir in file.parts for ignore_dir in ignore_dirs):
|
||||
all_files.add(file)
|
||||
|
||||
if not all_files:
|
||||
print("未找到包含版本号的文件")
|
||||
return
|
||||
|
||||
print(f"找到 {len(all_files)} 个文件需要更新版本号")
|
||||
for file in sorted(all_files):
|
||||
print(f" - {file.relative_to(Path.cwd())}")
|
||||
|
||||
# 更新所有文件的版本号(使用顺序执行避免竞争条件)
|
||||
# 使用相对于 cwd 的路径作为任务名,确保唯一性
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
f"bump_{file.relative_to(Path.cwd())}".replace("\\", "_").replace("/", "_").replace(".", "_"),
|
||||
fn=bump_file_version,
|
||||
args=(file, part),
|
||||
)
|
||||
for file in all_files
|
||||
]
|
||||
)
|
||||
report = px.run(graph, strategy="sequential")
|
||||
|
||||
# 收集新版本号(取第一个成功的结果)
|
||||
new_version = None
|
||||
for task_name in report:
|
||||
result = report[task_name]
|
||||
if result is not None:
|
||||
new_version = result
|
||||
break
|
||||
|
||||
if not new_version:
|
||||
print("未能获取新版本号")
|
||||
return
|
||||
|
||||
print(f"版本号已更新为: {new_version}")
|
||||
|
||||
# 提交修改
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("git_add", cmd=["git", "add", "."]),
|
||||
px.TaskSpec(
|
||||
"git_commit", cmd=["git", "commit", "-m", f"bump version to {new_version}"], depends_on=["git_add"]
|
||||
),
|
||||
]
|
||||
)
|
||||
px.run(graph, strategy="sequential")
|
||||
|
||||
# 创建 git tag
|
||||
if not args.no_tag:
|
||||
tag_name = f"v{new_version}"
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("git_tag", cmd=["git", "tag", "-a", tag_name, "-m", f"Release {tag_name}"]),
|
||||
]
|
||||
)
|
||||
px.run(graph, strategy="sequential")
|
||||
print(f"已创建标签: {tag_name}")
|
||||
@@ -1,27 +0,0 @@
|
||||
"""清屏工具.
|
||||
|
||||
跨平台清屏工具, 支持终端和控制台清屏.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
def clear_screen() -> None:
|
||||
"""使用系统命令清屏."""
|
||||
if Constants.IS_WINDOWS:
|
||||
subprocess.run(["cmd", "/c", "cls"], check=False)
|
||||
else:
|
||||
subprocess.run(["clear"], check=False)
|
||||
|
||||
print("\033[2J\033[H", end="")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""清屏工具主函数."""
|
||||
graph = px.Graph.from_specs([px.TaskSpec("clearscreen", fn=clear_screen)])
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,26 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
DockerMirrorType = Literal["tencent"]
|
||||
|
||||
DOCKER_MIRROR_URLS: dict[DockerMirrorType, str] = {"tencent": "ccr.ccs.tencentyun.com"}
|
||||
|
||||
|
||||
def main():
|
||||
# parser = argparse.ArgumentParser(description="Docker 命令行工具")
|
||||
# parser.add_argument("--username", nargs="?", default="", type=str, help="Docker 用户名")
|
||||
# args = parser.parse_args()
|
||||
|
||||
tasks: list[px.TaskSpec] = [
|
||||
px.cmd(["docker", "login", "--username", "xxx", DOCKER_MIRROR_URLS["tencent"]], name="docker_login_tencent"),
|
||||
]
|
||||
|
||||
alias: dict[str, str | list[str | px.TaskSpec] | px.TaskSpec | px.Graph] = {
|
||||
"login": "docker_login_tencent",
|
||||
}
|
||||
|
||||
runner = px.CliRunner(strategy="sequential", tasks=tasks, aliases=alias)
|
||||
runner.run_cli()
|
||||
@@ -0,0 +1,366 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import getpass
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import BuiltinConditions
|
||||
from pyflowx.tasks.system import setenv_group, write_file
|
||||
|
||||
# ============================================================================
|
||||
# Mirror 配置
|
||||
# ============================================================================
|
||||
DOWNLOAD_MIRROR_SCRIPT: str = "curl -sSL https://linuxmirrors.cn/main.sh -o /tmp/linuxmirrors.sh"
|
||||
INSTALL_MIRROR_SCRIPT: str = "sudo bash /tmp/linuxmirrors.sh"
|
||||
|
||||
# ============================================================================
|
||||
# Python 配置
|
||||
# ============================================================================
|
||||
PyMirrorType = Literal["tsinghua", "aliyun", "huaweicloud", "ustc", "zju"]
|
||||
|
||||
PIP_INDEX_URLS: dict[PyMirrorType, str] = {
|
||||
"tsinghua": "https://pypi.tuna.tsinghua.edu.cn/simple",
|
||||
"aliyun": "https://mirrors.aliyun.com/pypi/simple/",
|
||||
"huaweicloud": "https://mirrors.huaweicloud.com/repository/pypi/simple/",
|
||||
"ustc": "https://pypi.mirrors.ustc.edu.cn/simple/",
|
||||
"zju": "https://mirrors.zju.edu.cn/pypi/simple/",
|
||||
}
|
||||
|
||||
PIP_TRUSTED_HOSTS: dict[PyMirrorType, str] = {
|
||||
"tsinghua": "pypi.tuna.tsinghua.edu.cn",
|
||||
"aliyun": "mirrors.aliyun.com",
|
||||
"huaweicloud": "mirrors.huaweicloud.com",
|
||||
"ustc": "pypi.mirrors.ustc.edu.cn",
|
||||
"zju": "mirrors.zju.edu.cn",
|
||||
}
|
||||
PIP_CONFIG_PATH = Path.home() / ".pip" / "pip.conf" if BuiltinConditions.IS_LINUX() else Path.home() / "pip" / "pip.ini"
|
||||
|
||||
UV_INDEX_URLS = PIP_INDEX_URLS
|
||||
UV_PYTHON_INSTALL_MIRROR: str = "https://registry.npmmirror.com/-/binary/python-build-standalone"
|
||||
|
||||
# ============================================================================
|
||||
# Conda 配置
|
||||
# ============================================================================
|
||||
CondaMirrorType = Literal["tsinghua", "ustc", "bsfu", "aliyun"]
|
||||
|
||||
CONDA_MIRROR_URLS: dict[CondaMirrorType, list[str]] = {
|
||||
"tsinghua": [
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/pro/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/bioconda/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/menpo/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/pytorch/",
|
||||
],
|
||||
"ustc": [
|
||||
"https://mirrors.ustc.edu.cn/anaconda/pkgs/main/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/pkgs/free/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/pkgs/r/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/pkgs/msys2/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/pkgs/pro/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/pkgs/dev/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/cloud/conda-forge/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/cloud/bioconda/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/cloud/menpo/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/cloud/pytorch/",
|
||||
],
|
||||
"bsfu": [
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/pkgs/main/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/pkgs/free/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/pkgs/r/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/pkgs/msys2/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/pkgs/pro/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/pkgs/dev/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/cloud/conda-forge/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/cloud/bioconda/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/cloud/menpo/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/cloud/pytorch/",
|
||||
],
|
||||
"aliyun": [
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/main/",
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/free/",
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/r/",
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/msys2/",
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/pro/",
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/dev/",
|
||||
"https://mirrors.aliyun.com/anaconda/cloud/conda-forge/",
|
||||
"https://mirrors.aliyun.com/anaconda/cloud/bioconda/",
|
||||
"https://mirrors.aliyun.com/anaconda/cloud/menpo/",
|
||||
"https://mirrors.aliyun.com/anaconda/cloud/pytorch/",
|
||||
],
|
||||
}
|
||||
CONDA_CONFIG_PATH = Path.home() / ".condarc"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Qt 配置
|
||||
# ============================================================================
|
||||
|
||||
QT_LIBS: list[str] = [
|
||||
"build-essential",
|
||||
"libgl1",
|
||||
"libegl1",
|
||||
"libglib2.0-0",
|
||||
"libfontconfig1",
|
||||
"libfreetype6",
|
||||
"libxkbcommon0",
|
||||
"libdbus-1-3",
|
||||
"libxcb-xinerama0",
|
||||
"libxcb-icccm4",
|
||||
"libxcb-image0",
|
||||
"libxcb-keysyms1",
|
||||
"libxcb-randr0",
|
||||
"libxcb-render-util0",
|
||||
"libxcb-shape0",
|
||||
"libxcb-xfixes0",
|
||||
"libxcb-cursor0",
|
||||
]
|
||||
|
||||
CHINESE_FONTS: list[str] = [
|
||||
"fonts-noto-cjk",
|
||||
"fonts-wqy-microhei",
|
||||
"fonts-wqy-zenhei",
|
||||
"fonts-noto-color-emoji",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# Rust 配置
|
||||
# ============================================================================
|
||||
RustMirrorType = Literal["tsinghua", "ustc", "aliyun"]
|
||||
RustVersionType = Literal["stable", "nightly", "beta"]
|
||||
DEFAULT_RUST_VERSION: RustVersionType = "stable"
|
||||
DEFAULT_MIRROR: RustMirrorType = "tsinghua"
|
||||
|
||||
RUSTUP_MIRRORS: dict[RustMirrorType, dict[str, str]] = {
|
||||
"tsinghua": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.tuna.tsinghua.edu.cn/rustup",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.tuna.tsinghua.edu.cn/rustup/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.tuna.tsinghua.edu.cn/crates.io-index/",
|
||||
},
|
||||
"aliyun": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.aliyun.com/rustup",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.aliyun.com/rustup/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.aliyun.com/crates.io-index/",
|
||||
},
|
||||
"ustc": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.ustc.edu.cn/rust-static",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.ustc.edu.cn/rust-static/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.ustc.edu.cn/crates.io-index/",
|
||||
},
|
||||
}
|
||||
RUSTUP_DOWNLOAD_URL_LINUX = "https://mirrors.aliyun.com/repo/rust/rustup-init.sh"
|
||||
RUSTUP_DOWNLOAD_URL_WINDOWS = "https://static.rust-lang.org/rustup/dist/x86_64-pc-windows-msvc/rustup-init.exe"
|
||||
RUST_CONFIG_PATH = Path.home() / ".cargo" / "config.toml"
|
||||
RUST_SCCACHE_DIR: Path = Path.home() / ".cargo" / "sccache"
|
||||
RUST_SCCACHE_CACHE_SIZE: str = "20G"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""主函数."""
|
||||
parser = argparse.ArgumentParser(description="环境开发工具")
|
||||
parser.add_argument(
|
||||
"--python-mirror",
|
||||
nargs="?",
|
||||
type=str,
|
||||
default="tsinghua",
|
||||
choices=get_args(PyMirrorType),
|
||||
help="Python 镜像源",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--conda-mirror",
|
||||
nargs="?",
|
||||
type=str,
|
||||
default="tsinghua",
|
||||
choices=get_args(CondaMirrorType),
|
||||
help="Conda 镜镜像源",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rust-mirror",
|
||||
nargs="?",
|
||||
type=str,
|
||||
default=DEFAULT_MIRROR,
|
||||
choices=get_args(RustMirrorType),
|
||||
help="Rust 镜像源",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rust-version",
|
||||
nargs="?",
|
||||
type=str,
|
||||
default=DEFAULT_RUST_VERSION,
|
||||
choices=get_args(RustVersionType),
|
||||
help=f"Rust 版本, 推荐: {get_args(RustVersionType)}",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
python_mirror = args.python_mirror
|
||||
conda_mirror_urls = CONDA_MIRROR_URLS[args.conda_mirror]
|
||||
rust_mirror = args.rust_mirror
|
||||
rust_version = args.rust_version
|
||||
|
||||
# 确保配置文件目录存在
|
||||
PIP_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
CONDA_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
RUST_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
RUST_SCCACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 使用 conditions 自动控制任务执行
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
# 系统镜像配置(仅 Linux 且未配置国内镜像)
|
||||
px.TaskSpec(
|
||||
"download_mirror",
|
||||
cmd=DOWNLOAD_MIRROR_SCRIPT,
|
||||
conditions=(
|
||||
BuiltinConditions.IS_LINUX(),
|
||||
BuiltinConditions.NOT(
|
||||
BuiltinConditions.OR(
|
||||
*[
|
||||
BuiltinConditions.FILE_CONTENT_EXISTS(f, m)
|
||||
for f in [
|
||||
"/etc/apt/sources.list",
|
||||
"/etc/apt/sources.list.d/ubuntu.sources",
|
||||
]
|
||||
for m in get_args(PyMirrorType)
|
||||
],
|
||||
)
|
||||
),
|
||||
),
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
"install_mirror",
|
||||
cmd=INSTALL_MIRROR_SCRIPT,
|
||||
depends_on=("download_mirror",),
|
||||
verbose=True,
|
||||
),
|
||||
# 安装 Qt 依赖(仅 Linux)
|
||||
px.TaskSpec(
|
||||
"install_qt_libs",
|
||||
cmd=["sudo", "apt", "install", "-y", *QT_LIBS],
|
||||
conditions=(BuiltinConditions.IS_LINUX(),),
|
||||
depends_on=("install_mirror",),
|
||||
allow_upstream_skip=True,
|
||||
verbose=True,
|
||||
),
|
||||
# 安装中文字体(仅 Linux)
|
||||
px.TaskSpec(
|
||||
"install_fonts",
|
||||
cmd=["sudo", "apt", "install", "-y", *CHINESE_FONTS],
|
||||
conditions=(BuiltinConditions.IS_LINUX(),),
|
||||
depends_on=("install_mirror",),
|
||||
allow_upstream_skip=True,
|
||||
verbose=True,
|
||||
),
|
||||
# 安装 Docker
|
||||
px.TaskSpec(
|
||||
"install_docker",
|
||||
cmd=["sudo", "apt", "install", "-y", "docker-compose-v2"],
|
||||
conditions=(BuiltinConditions.IS_LINUX(),),
|
||||
depends_on=("install_mirror",),
|
||||
allow_upstream_skip=True,
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
"add_docker_group",
|
||||
cmd=["sudo", "usermod", "-aG", "docker", getpass.getuser()],
|
||||
conditions=(BuiltinConditions.IS_LINUX(),),
|
||||
depends_on=("install_docker",),
|
||||
allow_upstream_skip=True,
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
"refresh_docker_group",
|
||||
cmd=["newgrp", "docker"],
|
||||
conditions=(BuiltinConditions.IS_LINUX(),),
|
||||
depends_on=("add_docker_group",),
|
||||
allow_upstream_skip=True,
|
||||
verbose=True,
|
||||
),
|
||||
# 设置 Python 环境变量
|
||||
*setenv_group(
|
||||
{
|
||||
"PIP_INDEX_URL": PIP_INDEX_URLS[python_mirror],
|
||||
"PIP_TRUSTED_HOSTS": PIP_TRUSTED_HOSTS[python_mirror],
|
||||
"UV_INDEX_URL": UV_INDEX_URLS[python_mirror],
|
||||
"UV_PYTHON_INSTALL_MIRROR": UV_PYTHON_INSTALL_MIRROR,
|
||||
"UV_HTTP_TIMEOUT": "600",
|
||||
"UV_LINK_MODE": "copy",
|
||||
}
|
||||
),
|
||||
# 写入 Python 配置(仅当未配置)
|
||||
write_file(
|
||||
str(PIP_CONFIG_PATH),
|
||||
f"[global]\nindex-url = {PIP_INDEX_URLS[python_mirror]}\ntrusted-host = {PIP_TRUSTED_HOSTS[python_mirror]}",
|
||||
),
|
||||
# 写入 Conda 配置(仅当未配置)
|
||||
write_file(
|
||||
str(CONDA_CONFIG_PATH),
|
||||
"show_channel_urls: true\nchannels:\n - " + "\n - ".join(conda_mirror_urls) + "\n - defaults",
|
||||
),
|
||||
# 设置 Rust 镜像源
|
||||
*setenv_group(
|
||||
{
|
||||
"RUSTUP_DIST_SERVER": RUSTUP_MIRRORS[rust_mirror]["RUSTUP_DIST_SERVER"],
|
||||
"RUSTUP_UPDATE_ROOT": RUSTUP_MIRRORS[rust_mirror]["RUSTUP_UPDATE_ROOT"],
|
||||
"RUST_SCCACHE_DIR": str(RUST_SCCACHE_DIR),
|
||||
"RUST_SCCACHE_CACHE_SIZE": RUST_SCCACHE_CACHE_SIZE,
|
||||
}
|
||||
),
|
||||
# 写入 Rust 配置(仅当未配置)
|
||||
write_file(
|
||||
str(RUST_CONFIG_PATH),
|
||||
f"""
|
||||
[source.crates-io]
|
||||
replace-with = '{rust_mirror}'
|
||||
|
||||
[source.{rust_mirror}]
|
||||
registry = "sparse+{RUSTUP_MIRRORS[rust_mirror]["TOML_REGISTRY"]}"
|
||||
|
||||
[registries.{rust_mirror}]
|
||||
index = "sparse+{RUSTUP_MIRRORS[rust_mirror]["TOML_REGISTRY"]}"
|
||||
""",
|
||||
),
|
||||
# 下载 Rustup 安装脚本
|
||||
px.TaskSpec(
|
||||
"download_rustup",
|
||||
cmd=["curl", "-fsSL", RUSTUP_DOWNLOAD_URL_LINUX, "-o", "rustup-init.sh"],
|
||||
conditions=(
|
||||
BuiltinConditions.IS_LINUX(),
|
||||
BuiltinConditions.NOT(BuiltinConditions.HAS_INSTALLED("rustup")),
|
||||
),
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
"download_rustup_win",
|
||||
cmd=[
|
||||
"powershell",
|
||||
"-Command",
|
||||
"Invoke-WebRequest",
|
||||
"-Uri",
|
||||
RUSTUP_DOWNLOAD_URL_WINDOWS,
|
||||
"-OutFile",
|
||||
"rustup-init.exe",
|
||||
],
|
||||
conditions=(
|
||||
BuiltinConditions.IS_WINDOWS(),
|
||||
BuiltinConditions.NOT(BuiltinConditions.HAS_INSTALLED("rustup")),
|
||||
),
|
||||
verbose=True,
|
||||
),
|
||||
# 安装 Rust 工具链
|
||||
px.TaskSpec(
|
||||
"install_rust",
|
||||
cmd=["rustup", "toolchain", "install", rust_version],
|
||||
conditions=(BuiltinConditions.HAS_INSTALLED("rustup"),),
|
||||
depends_on=("setenv_rustup_dist_server",),
|
||||
allow_upstream_skip=True,
|
||||
verbose=True,
|
||||
),
|
||||
]
|
||||
)
|
||||
px.run(graph, strategy="thread", verbose=True)
|
||||
@@ -88,6 +88,8 @@ class EmailDatabase:
|
||||
|
||||
def insert_email(self, email_data: dict[str, Any]) -> bool:
|
||||
"""插入邮件数据."""
|
||||
assert self.conn, "数据库连接未初始化"
|
||||
|
||||
try:
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
@@ -123,6 +125,8 @@ class EmailDatabase:
|
||||
self, keyword: str = "", field: str = "all", limit: int = 100, offset: int = 0
|
||||
) -> list[dict[str, Any]]:
|
||||
"""搜索邮件."""
|
||||
assert self.conn, "数据库连接未初始化"
|
||||
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
|
||||
@@ -154,6 +158,8 @@ class EmailDatabase:
|
||||
|
||||
def get_grouped_emails(self) -> dict[str, list[dict[str, Any]]]:
|
||||
"""获取按主题分组的邮件."""
|
||||
assert self.conn, "数据库连接未初始化"
|
||||
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute(f"SELECT * FROM {TABLE_NAME} ORDER BY subject, date_parsed DESC")
|
||||
@@ -183,6 +189,8 @@ class EmailDatabase:
|
||||
|
||||
def get_email_count(self) -> int:
|
||||
"""获取邮件总数."""
|
||||
assert self.conn, "数据库连接未初始化"
|
||||
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute(f"SELECT COUNT(*) FROM {TABLE_NAME}")
|
||||
@@ -190,6 +198,8 @@ class EmailDatabase:
|
||||
|
||||
def clear_all(self) -> None:
|
||||
"""清空所有邮件数据."""
|
||||
assert self.conn, "数据库连接未初始化"
|
||||
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute(f"DELETE FROM {TABLE_NAME}")
|
||||
@@ -230,7 +240,7 @@ def _parse_email_date(date_str: str) -> str:
|
||||
try:
|
||||
dt = parsedate_to_datetime(date_str)
|
||||
return dt.isoformat()
|
||||
except Exception:
|
||||
except (ValueError, TypeError, OverflowError):
|
||||
return date_str
|
||||
|
||||
|
||||
@@ -267,11 +277,11 @@ def _extract_email_body_part(part: Any) -> str:
|
||||
decoded_text = payload.decode(charset, errors="replace")
|
||||
except (UnicodeDecodeError, LookupError) as decode_error:
|
||||
# 如果指定编码失败,尝试常见编码
|
||||
logger.warning(f"字符编码 {charset} 解码失败: {decode_error}")
|
||||
logger.warning("字符编码 %s 解码失败: %s", charset, decode_error)
|
||||
for fallback_charset in ["utf-8", "gbk", "gb2312", "latin-1"]:
|
||||
try:
|
||||
decoded_text = payload.decode(fallback_charset, errors="replace")
|
||||
logger.info(f"成功使用备用编码 {fallback_charset} 解码")
|
||||
logger.info("成功使用备用编码 %s 解码", fallback_charset)
|
||||
break
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
continue
|
||||
@@ -283,15 +293,15 @@ def _extract_email_body_part(part: Any) -> str:
|
||||
# 限制长度并返回
|
||||
result = decoded_text[:MAX_BODY_LENGTH]
|
||||
if len(decoded_text) > MAX_BODY_LENGTH:
|
||||
logger.debug(f"正文内容过长,截取前{MAX_BODY_LENGTH}字符")
|
||||
logger.debug("正文内容过长,截取前%d字符", MAX_BODY_LENGTH)
|
||||
|
||||
return result
|
||||
|
||||
except AttributeError as attr_error:
|
||||
logger.error(f"邮件部分对象属性错误: {attr_error}")
|
||||
logger.error("邮件部分对象属性错误: %s", attr_error)
|
||||
return ""
|
||||
except Exception as unexpected_error:
|
||||
logger.error(f"提取邮件正文时发生未知错误: {unexpected_error}")
|
||||
logger.error("提取邮件正文时发生未知错误: %s", unexpected_error)
|
||||
return ""
|
||||
|
||||
|
||||
@@ -578,6 +588,10 @@ class EmlManagerHandler(BaseHTTPRequestHandler):
|
||||
self._send_json_response({"error": "缺少邮件ID"}, 400)
|
||||
return
|
||||
|
||||
if not self.db.conn:
|
||||
self._send_json_response({"error": "数据库连接未初始化"}, 500)
|
||||
return
|
||||
|
||||
with self.db._lock:
|
||||
cursor = self.db.conn.cursor()
|
||||
cursor.execute(f"SELECT * FROM {TABLE_NAME} WHERE id = ?", (int(email_id),))
|
||||
@@ -630,6 +644,10 @@ class EmlManagerHandler(BaseHTTPRequestHandler):
|
||||
if not eml_files:
|
||||
return
|
||||
|
||||
if not self.db.conn:
|
||||
self._send_json_response({"error": "数据库连接未初始化"}, 500)
|
||||
return
|
||||
|
||||
# 先批量查询所有已存在的文件
|
||||
with self.db._lock:
|
||||
cursor = self.db.conn.cursor()
|
||||
@@ -1268,6 +1286,10 @@ def main() -> None:
|
||||
if eml_files:
|
||||
print(f"发现 {len(eml_files)} 个 EML 文件,开始导入...")
|
||||
|
||||
if not EmlManagerHandler.db.conn:
|
||||
print("数据库连接未初始化,无法导入邮件")
|
||||
return
|
||||
|
||||
# 先批量查询所有已存在的文件
|
||||
with EmlManagerHandler.db._lock:
|
||||
cursor = EmlManagerHandler.db.conn.cursor()
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
import pyflowx as px
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""主函数."""
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"envlinux", cmd=["sudo", "curl", "-sSL", "https://linuxmirrors.cn/main.sh", "|", "bash"], verbose=True
|
||||
)
|
||||
]
|
||||
)
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,122 +0,0 @@
|
||||
"""Python 环境配置工具.
|
||||
|
||||
用于设置 pip 镜像源, 支持清华和阿里云等国内镜像源,
|
||||
同时配置 UV 和 Conda 的镜像源.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
PIP_INDEX_URLS: dict[str, str] = {
|
||||
"tsinghua": "https://pypi.tuna.tsinghua.edu.cn/simple",
|
||||
"aliyun": "https://mirrors.aliyun.com/pypi/simple/",
|
||||
}
|
||||
|
||||
PIP_TRUSTED_HOSTS: dict[str, str] = {
|
||||
"tsinghua": "pypi.tuna.tsinghua.edu.cn",
|
||||
"aliyun": "mirrors.aliyun.com",
|
||||
}
|
||||
|
||||
UV_INDEX_URL: str = "https://mirrors.aliyun.com/pypi/simple/"
|
||||
UV_PYTHON_INSTALL_MIRROR: str = "https://registry.npmmirror.com/-/binary/python-build-standalone"
|
||||
|
||||
CONDA_MIRROR_URLS: dict[str, list[str]] = {
|
||||
"tsinghua": [
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/",
|
||||
],
|
||||
"aliyun": [
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/main/",
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/free/",
|
||||
"https://mirrors.aliyun.com/anaconda/cloud/conda-forge/",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def set_pip_mirror(mirror: str = "tsinghua", token: str | None = None) -> None:
|
||||
"""设置 pip 镜像源.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mirror : str
|
||||
镜像源名称: tsinghua, aliyun
|
||||
token : str | None
|
||||
PyPI token for publishing
|
||||
"""
|
||||
index_url = PIP_INDEX_URLS.get(mirror, PIP_INDEX_URLS["tsinghua"])
|
||||
trusted_host = PIP_TRUSTED_HOSTS.get(mirror, "")
|
||||
|
||||
# 设置环境变量
|
||||
os.environ["PIP_INDEX_URL"] = index_url
|
||||
os.environ["UV_INDEX_URL"] = UV_INDEX_URL
|
||||
os.environ["UV_DEFAULT_INDEX"] = UV_INDEX_URL
|
||||
os.environ["UV_PYTHON_INSTALL_MIRROR"] = UV_PYTHON_INSTALL_MIRROR
|
||||
|
||||
# 写入 pip 配置文件
|
||||
pip_dir = Path.home() / "pip"
|
||||
pip_dir.mkdir(exist_ok=True)
|
||||
pip_conf = pip_dir / ("pip.ini" if Constants.IS_WINDOWS else "pip.conf")
|
||||
pip_conf.write_text(f"[global]\nindex-url = {index_url}\n[install]\ntrusted-host = {trusted_host}\n")
|
||||
|
||||
# 写入 conda 配置文件
|
||||
condarc = Path.home() / ".condarc"
|
||||
conda_urls = CONDA_MIRROR_URLS.get(mirror, CONDA_MIRROR_URLS["tsinghua"])
|
||||
condarc.write_text(
|
||||
"show_channel_urls: true\nchannels:\n" + "\n".join(f" - {url}" for url in conda_urls) + "\n - defaults\n"
|
||||
)
|
||||
|
||||
# 写入 pypirc 配置文件 (如果有 token)
|
||||
if token:
|
||||
pypirc = Path.home() / ".pypirc"
|
||||
pypirc.write_text(
|
||||
f"[pypi]\nrepository: https://upload.pypi.org/legacy/\nusername: __token__\npassword: {token}\n"
|
||||
)
|
||||
|
||||
print(f"已设置 pip 镜像源: {mirror} ({index_url})")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Python 环境配置工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="EnvPy - Python 环境配置工具",
|
||||
usage="envpy <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 设置镜像源命令
|
||||
mirror_parser = subparsers.add_parser("mirror", help="设置 pip 镜像源")
|
||||
mirror_parser.add_argument("name", choices=["tsinghua", "aliyun"], help="镜像源名称")
|
||||
mirror_parser.add_argument("--token", type=str, help="PyPI token for publishing")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "mirror":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("set_pip_mirror", fn=set_pip_mirror, args=(args.name,), kwargs={"token": args.token})]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,57 +0,0 @@
|
||||
"""PyQt 环境配置工具.
|
||||
|
||||
用于设置 PyQt 相关环境变量, 安装依赖环境.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
QT_LIBS: list[str] = [
|
||||
"build-essential",
|
||||
"libgl1",
|
||||
"libegl1",
|
||||
"libglib2.0-0",
|
||||
"libfontconfig1",
|
||||
"libfreetype6",
|
||||
"libxkbcommon0",
|
||||
"libdbus-1-3",
|
||||
"libxcb-xinerama0",
|
||||
"libxcb-icccm4",
|
||||
"libxcb-image0",
|
||||
"libxcb-keysyms1",
|
||||
"libxcb-randr0",
|
||||
"libxcb-render-util0",
|
||||
"libxcb-shape0",
|
||||
"libxcb-xfixes0",
|
||||
"libxcb-cursor0",
|
||||
]
|
||||
|
||||
CHINESE_FONTS: list[str] = [
|
||||
"fonts-noto-cjk",
|
||||
"fonts-wqy-microhei",
|
||||
"fonts-wqy-zenhei",
|
||||
"fonts-noto-color-emoji",
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""PyQt 环境配置工具主函数."""
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"envqt_install",
|
||||
cmd=["sudo", "apt", "install", "-y", *QT_LIBS],
|
||||
conditions=(lambda: Constants.IS_LINUX,),
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
"envqt_fonts",
|
||||
cmd=["sudo", "apt", "install", "-y", *CHINESE_FONTS],
|
||||
conditions=(lambda: Constants.IS_LINUX,),
|
||||
verbose=True,
|
||||
),
|
||||
],
|
||||
)
|
||||
px.run(graph, strategy="thread", verbose=True)
|
||||
@@ -1,150 +0,0 @@
|
||||
"""Rust 环境配置工具.
|
||||
|
||||
配置 Rustup 和 Cargo 的国内镜像源,
|
||||
加速 Rust 工具链和依赖包的下载.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
RUSTUP_MIRRORS: dict[str, dict[str, str]] = {
|
||||
"aliyun": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.aliyun.com/rustup",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.aliyun.com/rustup/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.aliyun.com/crates.io-index/",
|
||||
},
|
||||
"ustc": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.ustc.edu.cn/rust-static",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.ustc.edu.cn/rust-static/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.ustc.edu.cn/crates.io-index/",
|
||||
},
|
||||
"tsinghua": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.tuna.tsinghua.edu.cn/rustup",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.tuna.tsinghua.edu.cn/rustup/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.tuna.tsinghua.edu.cn/crates.io-index/",
|
||||
},
|
||||
}
|
||||
|
||||
UsableRustVersion = Literal["stable", "nightly", "beta"]
|
||||
UsableMirror = Literal["aliyun", "ustc", "tsinghua"]
|
||||
|
||||
DEFAULT_RUST_VERSION: str = "stable"
|
||||
DEFAULT_MIRROR: UsableMirror = "tsinghua"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def set_rust_mirror(mirror: UsableMirror = DEFAULT_MIRROR) -> None:
|
||||
"""设置 Rust 镜像源.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mirror : str
|
||||
镜像源名称: aliyun, ustc, tsinghua
|
||||
"""
|
||||
mirror_dict = RUSTUP_MIRRORS.get(mirror, RUSTUP_MIRRORS[DEFAULT_MIRROR])
|
||||
server = mirror_dict["RUSTUP_DIST_SERVER"]
|
||||
update_root = mirror_dict["RUSTUP_UPDATE_ROOT"]
|
||||
toml_registry = mirror_dict["TOML_REGISTRY"]
|
||||
|
||||
# 设置环境变量
|
||||
os.environ["RUSTUP_DIST_SERVER"] = server
|
||||
os.environ["RUSTUP_UPDATE_ROOT"] = update_root
|
||||
|
||||
# 写入 cargo 配置
|
||||
cargo_dir = Path.home() / ".cargo"
|
||||
cargo_dir.mkdir(exist_ok=True)
|
||||
cargo_config = cargo_dir / "config.toml"
|
||||
cargo_config.write_text(
|
||||
f"""[source.crates-io]
|
||||
replace-with = '{mirror}'
|
||||
|
||||
[source.{mirror}]
|
||||
registry = "sparse+{toml_registry}"
|
||||
|
||||
[registries.{mirror}]
|
||||
index = "sparse+{toml_registry}"
|
||||
"""
|
||||
)
|
||||
|
||||
print(f"已设置 Rust 镜像源: {mirror}")
|
||||
|
||||
|
||||
def install_rust(version: UsableRustVersion = DEFAULT_RUST_VERSION) -> None:
|
||||
"""安装 Rust 工具链.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
version : str
|
||||
Rust 版本: stable, nightly, beta
|
||||
"""
|
||||
try:
|
||||
subprocess.run(["rustup", "toolchain", "install", version], check=True)
|
||||
print(f"已安装 Rust {version}")
|
||||
except FileNotFoundError:
|
||||
print("未找到 rustup,请先安装 Rust: https://rustup.rs")
|
||||
raise
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Rust 环境配置工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="EnvRs - Rust 环境配置工具",
|
||||
usage="envrs <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 设置镜像源命令
|
||||
mirror_parser = subparsers.add_parser("mirror", help="设置 Rust 镜像源")
|
||||
mirror_parser.add_argument(
|
||||
"name",
|
||||
nargs="?",
|
||||
default=DEFAULT_MIRROR,
|
||||
choices=get_args(UsableMirror),
|
||||
help=f"镜像源名称 ({get_args(UsableMirror)})",
|
||||
)
|
||||
|
||||
# 安装 Rust 命令
|
||||
install_parser = subparsers.add_parser("install", help="安装 Rust 工具链")
|
||||
install_parser.add_argument(
|
||||
"version",
|
||||
nargs="?",
|
||||
default=DEFAULT_RUST_VERSION,
|
||||
choices=get_args(UsableRustVersion),
|
||||
help=f"Rust 版本 ({get_args(UsableRustVersion)})",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "mirror":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("set_rust_mirror", fn=set_rust_mirror, args=(args.name,), verbose=True)]
|
||||
)
|
||||
elif args.command == "install":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("install_rust", cmd=["rustup", "toolchain", "install", args.version], verbose=True)]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread", verbose=True)
|
||||
@@ -1,141 +0,0 @@
|
||||
"""文件日期处理工具.
|
||||
|
||||
自动检测文件名的日期前缀,
|
||||
并根据文件的实际创建或修改时间重命名文件.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
DATE_PATTERN = re.compile(r"(20|19)\d{2}[-_#.~]?((0[1-9])|(1[012]))[-_#.~]?((0[1-9])|([12]\d)|(3[01]))[-_#.~]?")
|
||||
SEP = "_"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def get_file_timestamp(filepath: Path) -> str:
|
||||
"""获取文件时间戳."""
|
||||
modified_time = filepath.stat().st_mtime
|
||||
created_time = filepath.stat().st_ctime
|
||||
return time.strftime("%Y%m%d", time.localtime(max((modified_time, created_time))))
|
||||
|
||||
|
||||
def remove_date_prefix(filepath: Path) -> Path:
|
||||
"""移除文件日期前缀."""
|
||||
stem = filepath.stem
|
||||
new_stem = DATE_PATTERN.sub("", stem)
|
||||
if new_stem != stem:
|
||||
new_path = filepath.with_name(new_stem + filepath.suffix)
|
||||
filepath.rename(new_path)
|
||||
return new_path
|
||||
return filepath
|
||||
|
||||
|
||||
def add_date_prefix(filepath: Path) -> Path:
|
||||
"""添加文件日期前缀."""
|
||||
timestamp = get_file_timestamp(filepath)
|
||||
stem = filepath.stem
|
||||
new_stem = f"{timestamp}{SEP}{stem}"
|
||||
new_path = filepath.with_name(new_stem + filepath.suffix)
|
||||
if new_path != filepath:
|
||||
filepath.rename(new_path)
|
||||
return new_path
|
||||
return filepath
|
||||
|
||||
|
||||
def process_file_date(filepath: Path, clear: bool = False) -> None:
|
||||
"""处理单个文件的日期前缀.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filepath : Path
|
||||
文件路径
|
||||
clear : bool
|
||||
是否清除日期前缀
|
||||
"""
|
||||
if clear:
|
||||
remove_date_prefix(filepath)
|
||||
else:
|
||||
# 先移除旧日期前缀,再添加新日期前缀
|
||||
new_path = remove_date_prefix(filepath)
|
||||
add_date_prefix(new_path)
|
||||
|
||||
|
||||
def process_files_date(targets: list[Path], clear: bool = False) -> None:
|
||||
"""批量处理文件日期前缀.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
targets : list[Path]
|
||||
文件路径列表
|
||||
clear : bool
|
||||
是否清除日期前缀
|
||||
"""
|
||||
for target in targets:
|
||||
if target.exists() and not target.name.startswith("."):
|
||||
process_file_date(target, clear)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""文件日期处理工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="FileDate - 文件日期处理工具",
|
||||
usage="filedate <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 添加日期前缀命令
|
||||
add_parser = subparsers.add_parser("add", help="添加日期前缀")
|
||||
add_parser.add_argument("files", nargs="+", help="文件路径")
|
||||
|
||||
# 清除日期前缀命令
|
||||
clear_parser = subparsers.add_parser("clear", help="清除日期前缀")
|
||||
clear_parser.add_argument("files", nargs="+", help="文件路径")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "add":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"process_files_date",
|
||||
fn=process_files_date,
|
||||
args=([Path(f) for f in args.files],),
|
||||
kwargs={"clear": False},
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "clear":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"process_files_date",
|
||||
fn=process_files_date,
|
||||
args=([Path(f) for f in args.files],),
|
||||
kwargs={"clear": True},
|
||||
)
|
||||
]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,140 +0,0 @@
|
||||
"""文件等级重命名工具.
|
||||
|
||||
根据文件等级配置自动重命名文件,
|
||||
支持多种等级标识和括号格式.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
LEVELS: dict[str, str] = {
|
||||
"0": "",
|
||||
"1": "PUB,NOR",
|
||||
"2": "INT",
|
||||
"3": "CON",
|
||||
"4": "CLA",
|
||||
}
|
||||
|
||||
BRACKETS: tuple[str, str] = (" ([_(【-", " )]_)】")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def remove_marks(stem: str, marks: list[str]) -> str:
|
||||
"""从文件名主干中移除所有标记."""
|
||||
left_brackets, right_brackets = BRACKETS
|
||||
for mark in marks:
|
||||
pos = 0
|
||||
while True:
|
||||
pos = stem.find(mark, pos)
|
||||
if pos == -1:
|
||||
break
|
||||
b, e = pos - 1, pos + len(mark)
|
||||
if b >= 0 and e < len(stem) and stem[b] in left_brackets and stem[e] in right_brackets:
|
||||
stem = stem[:b] + stem[e + 1 :]
|
||||
else:
|
||||
pos = e
|
||||
return stem
|
||||
|
||||
|
||||
def process_file_level(filepath: Path, level: int = 0) -> None:
|
||||
"""处理单个文件的等级标记.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filepath : Path
|
||||
文件路径
|
||||
level : int
|
||||
文件等级 (0-4), 0 用于清除等级
|
||||
"""
|
||||
if not (0 <= level < len(LEVELS)):
|
||||
print(f"无效的等级 {level}, 必须在 0 和 {len(LEVELS) - 1} 之间")
|
||||
return
|
||||
|
||||
if not filepath.exists():
|
||||
print(f"文件不存在: {filepath}")
|
||||
return
|
||||
|
||||
filestem = filepath.stem
|
||||
original_stem = filestem
|
||||
|
||||
# 移除所有等级标记
|
||||
for level_names in LEVELS.values():
|
||||
if level_names:
|
||||
filestem = remove_marks(filestem, level_names.split(","))
|
||||
|
||||
# 移除数字标记
|
||||
for digit in map(str, range(1, 10)):
|
||||
filestem = remove_marks(filestem, [digit])
|
||||
|
||||
# 添加等级标记
|
||||
if level > 0:
|
||||
levelstr = LEVELS.get(str(level), "").split(",")[0]
|
||||
if levelstr:
|
||||
filestem = f"{filestem}({levelstr})"
|
||||
|
||||
# 重命名文件
|
||||
if filestem != original_stem:
|
||||
new_path = filepath.with_name(filestem + filepath.suffix)
|
||||
filepath.rename(new_path)
|
||||
print(f"重命名: {filepath} -> {new_path}")
|
||||
|
||||
|
||||
def process_files_level(targets: list[Path], level: int = 0) -> None:
|
||||
"""批量处理文件等级标记.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
targets : list[Path]
|
||||
文件路径列表
|
||||
level : int
|
||||
文件等级 (0-4)
|
||||
"""
|
||||
for target in targets:
|
||||
process_file_level(target, level)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""文件等级重命名工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="FileLevel - 文件等级重命名工具",
|
||||
usage="filelevel <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 设置等级命令
|
||||
level_parser = subparsers.add_parser("set", help="设置文件等级")
|
||||
level_parser.add_argument("files", nargs="+", help="文件路径")
|
||||
level_parser.add_argument("--level", type=int, choices=[0, 1, 2, 3, 4], required=True, help="文件等级 (0-4)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "set":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"process_files_level", fn=process_files_level, args=([Path(f) for f in args.files], args.level)
|
||||
)
|
||||
]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,94 +0,0 @@
|
||||
"""文件夹备份工具.
|
||||
|
||||
备份文件和文件夹为 zip 文件,
|
||||
自动删除超过最大数量的旧备份文件.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def remove_dump(src: Path, dst: Path, max_zip: int) -> None:
|
||||
"""递归删除旧的备份 zip 文件."""
|
||||
zip_paths = [filepath for filepath in dst.rglob("*.zip") if src.stem in str(filepath)]
|
||||
zip_files = sorted(zip_paths, key=lambda fn: str(fn)[-19:-4])
|
||||
if len(zip_files) > max_zip:
|
||||
zip_files[0].unlink()
|
||||
remove_dump(src, dst, max_zip)
|
||||
|
||||
|
||||
def zip_target(src: Path, dst: Path, max_zip: int) -> None:
|
||||
"""将单个文件或文件夹压缩为 zip 文件."""
|
||||
files = [str(_) for _ in src.rglob("*")]
|
||||
timestamp = time.strftime("_%Y%m%d_%H%M%S")
|
||||
target_path = dst / (src.stem + timestamp + ".zip")
|
||||
|
||||
with zipfile.ZipFile(target_path, "w") as zip_file:
|
||||
for file in files:
|
||||
zip_file.write(file, arcname=file.replace(str(src.parent), ""))
|
||||
|
||||
remove_dump(src, dst, max_zip)
|
||||
print(f"备份完成: {target_path}")
|
||||
|
||||
|
||||
def backup_folder(src: str, dst: str, max_zip: int = 5) -> None:
|
||||
"""备份文件夹.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
src : str
|
||||
源文件夹路径
|
||||
dst : str
|
||||
目标文件夹路径
|
||||
max_zip : int
|
||||
最大备份数量
|
||||
"""
|
||||
src_path = Path(src)
|
||||
dst_path = Path(dst)
|
||||
|
||||
if not src_path.exists():
|
||||
print(f"源文件夹不存在: {src_path}")
|
||||
return
|
||||
|
||||
if not dst_path.exists():
|
||||
dst_path.mkdir(parents=True, exist_ok=True)
|
||||
print(f"创建目标文件夹: {dst_path}")
|
||||
|
||||
zip_target(src_path, dst_path, max_zip)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# TaskSpec 定义
|
||||
# ============================================================================
|
||||
|
||||
folderback_default: px.TaskSpec = px.TaskSpec(
|
||||
"folderback_default",
|
||||
fn=lambda: backup_folder(".", "./backup", 5),
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""文件夹备份工具主函数."""
|
||||
runner = px.CliRunner(
|
||||
strategy="thread",
|
||||
description="FolderBack - 文件夹备份工具",
|
||||
graphs={
|
||||
# 备份当前目录到 ./backup
|
||||
"b": px.Graph.from_specs([folderback_default]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
@@ -1,82 +0,0 @@
|
||||
"""文件夹压缩工具.
|
||||
|
||||
压缩目录下的所有文件/文件夹为 zip 文件,
|
||||
默认压缩当前目录下的所有子文件夹.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
IGNORE_DIRS: list[str] = [".git", ".idea", ".vscode", "__pycache__"]
|
||||
IGNORE_FILES: list[str] = [".gitignore"]
|
||||
IGNORE: list[str] = [*IGNORE_DIRS, *IGNORE_FILES]
|
||||
IGNORE_EXT: list[str] = [".zip", ".rar", ".7z", ".tar", ".gz"]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def archive_folder(folder: Path) -> None:
|
||||
"""压缩单个文件夹."""
|
||||
shutil.make_archive(
|
||||
str(folder.with_name(folder.name)),
|
||||
format="zip",
|
||||
base_dir=folder,
|
||||
)
|
||||
print(f"压缩完成: {folder.name}.zip")
|
||||
|
||||
|
||||
def zip_folders(cwd: str = ".") -> None:
|
||||
"""压缩目录下的所有文件夹.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
cwd : str
|
||||
工作目录
|
||||
"""
|
||||
cwd_path = Path(cwd)
|
||||
if not cwd_path.exists():
|
||||
print(f"目录不存在: {cwd_path}")
|
||||
return
|
||||
|
||||
dirs: list[Path] = [
|
||||
e for e in cwd_path.iterdir() if e.is_dir() and e.name not in IGNORE_DIRS and e.suffix not in IGNORE_EXT
|
||||
]
|
||||
|
||||
for dir_path in dirs:
|
||||
archive_folder(dir_path)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# TaskSpec 定义
|
||||
# ============================================================================
|
||||
|
||||
folderzip_default: px.TaskSpec = px.TaskSpec("folderzip_default", fn=lambda: zip_folders("."))
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""文件夹压缩工具主函数."""
|
||||
runner = px.CliRunner(
|
||||
strategy="thread",
|
||||
description="FolderZip - 文件夹压缩工具",
|
||||
graphs={
|
||||
# 压缩当前目录下的所有文件夹
|
||||
"z": px.Graph.from_specs([folderzip_default]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
@@ -1,109 +0,0 @@
|
||||
"""Git 工具模块.
|
||||
|
||||
提供 Git 仓库管理的常用操作封装,
|
||||
支持初始化、提交、清理、推送等功能.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
EXCLUDE_DIRS = [
|
||||
# 编辑器相关目录
|
||||
".vscode",
|
||||
".idea",
|
||||
".editorconfig",
|
||||
".trae",
|
||||
".qoder",
|
||||
# 项目相关目录
|
||||
".venv",
|
||||
".git",
|
||||
".tox",
|
||||
".pytest_cache",
|
||||
"node_modules",
|
||||
".ruff_cache",
|
||||
]
|
||||
EXCLUDE_CMDS = [arg for d in EXCLUDE_DIRS for arg in ["-e", d]]
|
||||
|
||||
|
||||
def init_sub_dirs() -> None:
|
||||
"""初始化子目录的Git仓库."""
|
||||
sub_dirs = [subdir for subdir in Path.cwd().iterdir() if subdir.is_dir()]
|
||||
for subdir in sub_dirs:
|
||||
px.run(
|
||||
px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"init",
|
||||
cmd=["git", "init"],
|
||||
conditions=[not_has_git_repo],
|
||||
cwd=str(subdir),
|
||||
),
|
||||
px.TaskSpec("add", cmd=["git", "add", "."], depends_on=["init"], cwd=str(subdir)),
|
||||
px.TaskSpec(
|
||||
"commit", cmd=["git", "commit", "-m", "init commit"], depends_on=["add"], cwd=str(subdir)
|
||||
),
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
isub: px.TaskSpec = px.TaskSpec("isub", fn=init_sub_dirs)
|
||||
push: px.TaskSpec = px.TaskSpec("push", cmd=["git", "push"])
|
||||
pull: px.TaskSpec = px.TaskSpec("pull", cmd=["git", "pull"])
|
||||
kill_tgit: px.TaskSpec = px.TaskSpec("task_kill", cmd=["taskkill", "/f", "/t", "/im", "tgitcache.exe"])
|
||||
|
||||
|
||||
def not_has_git_repo() -> bool:
|
||||
"""检查当前目录没有Git仓库."""
|
||||
return not Path.cwd().exists() or not (Path.cwd() / ".git").is_dir()
|
||||
|
||||
|
||||
def has_files() -> bool:
|
||||
"""检查当前目录是否有文件."""
|
||||
return bool(list(Path.cwd().glob("*")))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Git工具主函数."""
|
||||
runner = px.CliRunner(
|
||||
strategy="thread",
|
||||
description="Gittool - Git 执行工具.",
|
||||
graphs={
|
||||
# 添加并提交
|
||||
"a": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("add", cmd=["git", "add", "."], conditions=[has_files]),
|
||||
px.TaskSpec("commit", cmd=["git", "commit", "-m", "chore: update"], depends_on=["add"]),
|
||||
]
|
||||
),
|
||||
# 清理
|
||||
"c": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("clean", cmd=["git", "clean", "-xfd", *EXCLUDE_CMDS]),
|
||||
px.TaskSpec("status", cmd=["git", "status", "--porcelain"], depends_on=["clean"]),
|
||||
]
|
||||
),
|
||||
# 初始化、添加并提交
|
||||
"i": px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("init", cmd=["git", "init"], conditions=[not_has_git_repo]),
|
||||
px.TaskSpec("add", cmd=["git", "add", "."], depends_on=["init"], conditions=[has_files]),
|
||||
px.TaskSpec(
|
||||
"commit", cmd=["git", "commit", "-m", "init commit"], depends_on=["add"], conditions=[has_files]
|
||||
),
|
||||
]
|
||||
),
|
||||
# 初始化子目录
|
||||
"isub": px.Graph.from_specs([isub]),
|
||||
# 推送
|
||||
"p": px.Graph.from_specs([push]),
|
||||
# 拉取
|
||||
"pl": px.Graph.from_specs([pull]),
|
||||
# 重启TGit缓存
|
||||
"r": px.Graph.from_specs([kill_tgit]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
@@ -1,86 +0,0 @@
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
HFDownloadType = Literal["model", "dataset", "space"]
|
||||
|
||||
|
||||
def setenvs():
|
||||
"""设置 HuggingFace mirror 环境变量."""
|
||||
os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Download a model from HuggingFace.")
|
||||
parser.add_argument("dataset_name", type=str, help="HuggingFace dataset name.")
|
||||
parser.add_argument(
|
||||
"--type",
|
||||
type=str,
|
||||
nargs="?",
|
||||
default="dataset",
|
||||
choices=get_args(HFDownloadType),
|
||||
help="HuggingFace dataset type.",
|
||||
)
|
||||
parser.add_argument("--use-hfd", action="store_true", help="Use HFD tool to download dataset.")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.dataset_name:
|
||||
parser.error("dataset_name is required")
|
||||
|
||||
dataset_name = args.dataset_name
|
||||
|
||||
# 创建下载目录
|
||||
download_dir = Path.cwd() / dataset_name
|
||||
download_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if args.use_hfd:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(name="setenvs", fn=setenvs, verbose=True),
|
||||
px.TaskSpec(
|
||||
name="download_hfd",
|
||||
cmd=["wget", "https://hf-mirror.com/hfd/hfd.sh"],
|
||||
depends_on=["setenvs"],
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
name="chmod_hfd",
|
||||
cmd=["chmod", "a+x", "hfd.sh"],
|
||||
depends_on=["download_hfd"],
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
name="run_hfd",
|
||||
cmd=["./hfd.sh", dataset_name, args.type],
|
||||
depends_on=["chmod_hfd"],
|
||||
verbose=True,
|
||||
),
|
||||
]
|
||||
)
|
||||
else:
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(name="setenvs", fn=setenvs, verbose=True),
|
||||
px.TaskSpec(
|
||||
name="download",
|
||||
cmd=[
|
||||
"uvx",
|
||||
"hf",
|
||||
"download",
|
||||
"--repo-type",
|
||||
args.type,
|
||||
"--force-download",
|
||||
dataset_name,
|
||||
"--local-dir",
|
||||
str(Path.cwd() / dataset_name),
|
||||
],
|
||||
depends_on=["setenvs"],
|
||||
verbose=True,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
px.run(graph, strategy="thread", verbose=True)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Download from ModelScopeHub."""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
DownloadType = Literal["model", "dataset", "space"]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Download a model from ModelScopeHub.")
|
||||
parser.add_argument("name", help="Target name.")
|
||||
parser.add_argument("--type", "-t", nargs="?", default="model", choices=get_args(DownloadType), help="Target type.")
|
||||
parser.add_argument("--dir", default=None, help="Download directory.")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.name:
|
||||
parser.error("name is required")
|
||||
|
||||
download_dir: Path = Path(args.dir) if args.dir else Path.home() / ".models" / args.name.split("/")[-1]
|
||||
download_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
name="download",
|
||||
cmd=[
|
||||
"uvx",
|
||||
"modelscope",
|
||||
"download",
|
||||
f"--{args.type}",
|
||||
args.name,
|
||||
"--local_dir",
|
||||
str(download_dir),
|
||||
],
|
||||
verbose=True,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
px.run(graph, strategy="thread", verbose=True)
|
||||
@@ -0,0 +1,65 @@
|
||||
"""使用 SGLang 运行本地模型."""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import BuiltinConditions, Constants
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="启动 SGLang 服务")
|
||||
parser.add_argument("--model", default="~/.models/Qwen2.5-Coder-32B-Instruct-AWQ", help="模型路径")
|
||||
parser.add_argument("--port", type=int, default=8000, help="服务端口")
|
||||
parser.add_argument("--ctx-len", type=int, default=28672, help="最大上下文长度")
|
||||
parser.add_argument("--mem", type=float, default=0.75, help="显存占比 (0-1)")
|
||||
parser.add_argument("--host", default="0.0.0.0", help="主机地址")
|
||||
parser.add_argument("--log-level", default="info", help="日志级别")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.model:
|
||||
parser.error("model is required")
|
||||
|
||||
model_dir = Path(args.model).expanduser()
|
||||
if not model_dir.exists():
|
||||
parser.error(f"Model directory {model_dir} does not exist.")
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
name="download",
|
||||
cmd=[
|
||||
"uv",
|
||||
"install",
|
||||
"sglang[all]",
|
||||
],
|
||||
conditions=(BuiltinConditions.NOT(BuiltinConditions.HAS_INSTALLED("sglang")),),
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
name="run",
|
||||
cmd=[
|
||||
"python" if Constants.IS_WINDOWS else "python3",
|
||||
"-m",
|
||||
"sglang.launch_server",
|
||||
"--model-path",
|
||||
str(model_dir),
|
||||
"--host",
|
||||
str(args.host),
|
||||
"--port",
|
||||
"8000",
|
||||
"--mem-fraction-static",
|
||||
str(args.mem),
|
||||
"--context-length",
|
||||
"32768",
|
||||
"--tool-call-parser",
|
||||
"qwen",
|
||||
"--log-level",
|
||||
str(args.log_level),
|
||||
],
|
||||
verbose=True,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
px.run(graph, strategy="sequential", verbose=True)
|
||||
@@ -1,174 +0,0 @@
|
||||
"""LS-DYNA 计算工具.
|
||||
|
||||
用于管理 LS-DYNA 仿真计算任务,
|
||||
支持启动、监控和管理计算进程.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
LS_DYNA_COMMANDS: dict[str, list[str]] = {
|
||||
"windows": ["ls-dyna_mpp", "i=input.k", "ncpu=4"],
|
||||
"linux": ["ls-dyna_mpp", "i=input.k", "ncpu=8"],
|
||||
"macos": ["ls-dyna_mpp", "i=input.k", "ncpu=4"],
|
||||
}
|
||||
|
||||
DEFAULT_INPUT_FILE: str = "input.k"
|
||||
DEFAULT_NCPU: int = 4
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def get_ls_dyna_command(input_file: str, ncpu: int) -> list[str]:
|
||||
"""获取 LS-DYNA 命令.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_file : str
|
||||
输入文件路径
|
||||
ncpu : int
|
||||
CPU 核心数
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[str]
|
||||
LS-DYNA 命令列表
|
||||
"""
|
||||
if Constants.IS_WINDOWS or Constants.IS_MACOS:
|
||||
return ["ls-dyna_mpp", f"i={input_file}", f"ncpu={ncpu}"]
|
||||
else:
|
||||
return ["ls-dyna_mpp", f"i={input_file}", f"ncpu={ncpu}"]
|
||||
|
||||
|
||||
def run_ls_dyna(input_file: str, ncpu: int = DEFAULT_NCPU) -> None:
|
||||
"""运行 LS-DYNA 计算.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_file : str
|
||||
输入文件路径
|
||||
ncpu : int
|
||||
CPU 核心数
|
||||
"""
|
||||
input_path = Path(input_file)
|
||||
if not input_path.exists():
|
||||
print(f"输入文件不存在: {input_path}")
|
||||
return
|
||||
|
||||
cmd = get_ls_dyna_command(input_file, ncpu)
|
||||
try:
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"LS-DYNA 计算完成: {input_file}")
|
||||
except FileNotFoundError:
|
||||
print("未找到 ls-dyna_mpp 命令")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"LS-DYNA 计算失败: {e}")
|
||||
|
||||
|
||||
def run_ls_dyna_mpi(input_file: str, ncpu: int = DEFAULT_NCPU) -> None:
|
||||
"""运行 LS-DYNA MPI 计算.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_file : str
|
||||
输入文件路径
|
||||
ncpu : int
|
||||
CPU 核心数
|
||||
"""
|
||||
input_path = Path(input_file)
|
||||
if not input_path.exists():
|
||||
print(f"输入文件不存在: {input_path}")
|
||||
return
|
||||
|
||||
cmd = ["mpirun", "-np", str(ncpu), "ls-dyna_mpp", f"i={input_file}"]
|
||||
try:
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"LS-DYNA MPI 计算完成: {input_file}")
|
||||
except FileNotFoundError:
|
||||
print("未找到 mpirun 或 ls-dyna_mpp 命令")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"LS-DYNA MPI 计算失败: {e}")
|
||||
|
||||
|
||||
def check_ls_dyna_status() -> None:
|
||||
"""检查 LS-DYNA 进程状态."""
|
||||
try:
|
||||
if Constants.IS_WINDOWS:
|
||||
result = subprocess.run(
|
||||
["tasklist", "/fi", "imagename eq ls-dyna_mpp.exe"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
print(result.stdout)
|
||||
else:
|
||||
result = subprocess.run(
|
||||
["pgrep", "-f", "ls-dyna"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.stdout.strip():
|
||||
print(f"运行中的 LS-DYNA 进程 PID: {result.stdout.strip()}")
|
||||
else:
|
||||
print("没有运行中的 LS-DYNA 进程")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"检查进程状态失败: {e}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""LS-DYNA 计算工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="LSCalc - LS-DYNA 计算工具",
|
||||
usage="lscalc <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 运行计算命令
|
||||
run_parser = subparsers.add_parser("run", help="运行 LS-DYNA 计算")
|
||||
run_parser.add_argument("input_file", help="输入文件路径")
|
||||
run_parser.add_argument("--ncpu", type=int, default=DEFAULT_NCPU, help="CPU 核心数")
|
||||
|
||||
# 运行 MPI 计算命令
|
||||
mpi_parser = subparsers.add_parser("mpi", help="运行 LS-DYNA MPI 计算")
|
||||
mpi_parser.add_argument("input_file", help="输入文件路径")
|
||||
mpi_parser.add_argument("--ncpu", type=int, default=DEFAULT_NCPU, help="CPU 核心数")
|
||||
|
||||
# 检查进程状态命令
|
||||
subparsers.add_parser("status", help="检查 LS-DYNA 进程状态")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "run":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("run_ls_dyna", fn=run_ls_dyna, args=(args.input_file,), kwargs={"ncpu": args.ncpu})]
|
||||
)
|
||||
elif args.command == "mpi":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("run_ls_dyna_mpi", fn=run_ls_dyna_mpi, args=(args.input_file,), kwargs={"ncpu": args.ncpu})]
|
||||
)
|
||||
elif args.command == "status":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("check_ls_dyna_status", fn=check_ls_dyna_status)])
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,349 +0,0 @@
|
||||
"""Python 打包工具模块.
|
||||
|
||||
提供 Python 项目打包的常用功能封装,
|
||||
支持源码打包、依赖打包、嵌入式 Python 安装等功能.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
import subprocess
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
DEFAULT_BUILD_DIR = ".pypack"
|
||||
DEFAULT_DIST_DIR = "dist"
|
||||
DEFAULT_LIB_DIR = "libs"
|
||||
DEFAULT_CACHE_DIR = ".cache/pypack"
|
||||
|
||||
IGNORE_PATTERNS = [
|
||||
"__pycache__",
|
||||
"*.pyc",
|
||||
"*.pyo",
|
||||
".git",
|
||||
".venv",
|
||||
".idea",
|
||||
".vscode",
|
||||
"*.egg-info",
|
||||
"dist",
|
||||
"build",
|
||||
".pytest_cache",
|
||||
".tox",
|
||||
".mypy_cache",
|
||||
]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def pack_source(project_dir: Path, output_dir: Path) -> None:
|
||||
"""打包项目源码.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
project_dir : Path
|
||||
项目目录
|
||||
output_dir : Path
|
||||
输出目录
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 检测项目名称
|
||||
pyproject_file = project_dir / "pyproject.toml"
|
||||
project_name = project_dir.name
|
||||
|
||||
if pyproject_file.exists():
|
||||
try:
|
||||
import tomllib
|
||||
|
||||
content = pyproject_file.read_text(encoding="utf-8")
|
||||
data = tomllib.loads(content)
|
||||
project_name = data.get("project", {}).get("name", project_name)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# 打包源码
|
||||
source_dir = output_dir / "src" / project_name
|
||||
source_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 复制文件
|
||||
src_subdir = project_dir / "src"
|
||||
if src_subdir.exists():
|
||||
shutil.copytree(
|
||||
src_subdir,
|
||||
source_dir / "src",
|
||||
ignore=shutil.ignore_patterns(*IGNORE_PATTERNS),
|
||||
dirs_exist_ok=True,
|
||||
)
|
||||
else:
|
||||
for item in project_dir.iterdir():
|
||||
if item.name in IGNORE_PATTERNS or item.name.startswith("."):
|
||||
continue
|
||||
dst_item = source_dir / item.name
|
||||
if item.is_dir():
|
||||
shutil.copytree(
|
||||
item,
|
||||
dst_item,
|
||||
ignore=shutil.ignore_patterns(*IGNORE_PATTERNS),
|
||||
dirs_exist_ok=True,
|
||||
)
|
||||
else:
|
||||
shutil.copy2(item, dst_item)
|
||||
|
||||
print(f"源码打包完成: {source_dir}")
|
||||
|
||||
|
||||
def pack_dependencies(lib_dir: Path, dependencies: list[str]) -> None:
|
||||
"""打包项目依赖.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lib_dir : Path
|
||||
依赖库目录
|
||||
dependencies : list[str]
|
||||
依赖列表
|
||||
"""
|
||||
lib_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if not dependencies:
|
||||
print("没有依赖需要打包")
|
||||
return
|
||||
|
||||
# 使用 pip 安装依赖到目标目录
|
||||
cmd = [
|
||||
"pip",
|
||||
"install",
|
||||
"--target",
|
||||
str(lib_dir),
|
||||
"--no-compile",
|
||||
"--no-warn-script-location",
|
||||
]
|
||||
cmd.extend(dependencies)
|
||||
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"依赖打包完成: {lib_dir}")
|
||||
|
||||
|
||||
def pack_wheel(project_dir: Path, output_dir: Path) -> None:
|
||||
"""打包项目为 wheel 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
project_dir : Path
|
||||
项目目录
|
||||
output_dir : Path
|
||||
输出目录
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 使用 pip wheel 打包
|
||||
cmd = [
|
||||
"pip",
|
||||
"wheel",
|
||||
"--no-deps",
|
||||
"--wheel-dir",
|
||||
str(output_dir),
|
||||
str(project_dir),
|
||||
]
|
||||
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"Wheel 打包完成: {output_dir}")
|
||||
|
||||
|
||||
def install_embed_python(version: str, output_dir: Path) -> None:
|
||||
"""安装嵌入式 Python.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
version : str
|
||||
Python 版本 (如: 3.10, 3.11)
|
||||
output_dir : Path
|
||||
输出目录
|
||||
"""
|
||||
import platform
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 构建下载 URL
|
||||
arch = platform.machine().lower()
|
||||
if arch in ["x86_64", "amd64"]:
|
||||
arch = "amd64"
|
||||
elif arch in ["arm64", "aarch64"]:
|
||||
arch = "arm64"
|
||||
|
||||
# 解析完整版本号
|
||||
version_map = {
|
||||
"3.8": "3.8.10",
|
||||
"3.9": "3.9.13",
|
||||
"3.10": "3.10.11",
|
||||
"3.11": "3.11.9",
|
||||
"3.12": "3.12.4",
|
||||
}
|
||||
full_version = version_map.get(version, f"{version}.0")
|
||||
|
||||
# Windows 嵌入式 Python 下载 URL
|
||||
url = f"https://www.python.org/ftp/python/{full_version}/python-{full_version}-embed-{arch}.zip"
|
||||
|
||||
# 下载并解压
|
||||
cache_file = Path(DEFAULT_CACHE_DIR) / f"python-{full_version}-embed-{arch}.zip"
|
||||
cache_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if not cache_file.exists():
|
||||
print(f"正在下载嵌入式 Python {full_version}...")
|
||||
import urllib.request
|
||||
|
||||
urllib.request.urlretrieve(url, cache_file)
|
||||
print(f"下载完成: {cache_file}")
|
||||
|
||||
# 解压
|
||||
with zipfile.ZipFile(cache_file, "r") as zf:
|
||||
zf.extractall(output_dir)
|
||||
|
||||
print(f"嵌入式 Python 安装完成: {output_dir}")
|
||||
|
||||
|
||||
def create_zip_package(source_dir: Path, output_file: Path) -> None:
|
||||
"""创建 ZIP 打包文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
source_dir : Path
|
||||
源目录
|
||||
output_file : Path
|
||||
输出文件
|
||||
"""
|
||||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with zipfile.ZipFile(output_file, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
for file in source_dir.rglob("*"):
|
||||
if file.is_file():
|
||||
arcname = file.relative_to(source_dir)
|
||||
zf.write(file, arcname)
|
||||
|
||||
print(f"ZIP 打包完成: {output_file}")
|
||||
|
||||
|
||||
def clean_build_dir(build_dir: Path) -> None:
|
||||
"""清理构建目录.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
build_dir : Path
|
||||
构建目录
|
||||
"""
|
||||
if build_dir.exists():
|
||||
shutil.rmtree(build_dir)
|
||||
print(f"清理完成: {build_dir}")
|
||||
else:
|
||||
print(f"目录不存在: {build_dir}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Python 打包工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="PackTool - Python 打包工具",
|
||||
usage="packtool <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 源码打包命令
|
||||
src_parser = subparsers.add_parser("src", help="打包项目源码")
|
||||
src_parser.add_argument("--project-dir", type=str, default=".", help="项目目录")
|
||||
src_parser.add_argument("--output-dir", type=str, default=DEFAULT_BUILD_DIR, help="输出目录")
|
||||
|
||||
# 依赖打包命令
|
||||
deps_parser = subparsers.add_parser("deps", help="打包项目依赖")
|
||||
deps_parser.add_argument("--lib-dir", type=str, default=DEFAULT_LIB_DIR, help="依赖库目录")
|
||||
deps_parser.add_argument("dependencies", nargs="*", help="依赖列表")
|
||||
|
||||
# Wheel 打包命令
|
||||
wheel_parser = subparsers.add_parser("wheel", help="打包项目为 wheel 文件")
|
||||
wheel_parser.add_argument("--project-dir", type=str, default=".", help="项目目录")
|
||||
wheel_parser.add_argument("--output-dir", type=str, default=DEFAULT_DIST_DIR, help="输出目录")
|
||||
|
||||
# 嵌入式 Python 安装命令
|
||||
embed_parser = subparsers.add_parser("embed", help="安装嵌入式 Python")
|
||||
embed_parser.add_argument("--version", type=str, default="3.10", help="Python 版本")
|
||||
embed_parser.add_argument("--output-dir", type=str, default="python", help="输出目录")
|
||||
|
||||
# ZIP 打包命令
|
||||
zip_parser = subparsers.add_parser("zip", help="创建 ZIP 打包文件")
|
||||
zip_parser.add_argument("--source-dir", type=str, default=".", help="源目录")
|
||||
zip_parser.add_argument("--output-file", type=str, default="package.zip", help="输出文件")
|
||||
|
||||
# 清理命令
|
||||
subparsers.add_parser("clean", help="清理构建目录")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "src":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pack_source",
|
||||
fn=pack_source,
|
||||
args=(Path(args.project_dir), Path(args.output_dir)),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "deps":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pack_deps",
|
||||
fn=pack_dependencies,
|
||||
args=(Path(args.lib_dir), args.dependencies),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "wheel":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pack_wheel",
|
||||
fn=pack_wheel,
|
||||
args=(Path(args.project_dir), Path(args.output_dir)),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "embed":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"install_embed",
|
||||
fn=install_embed_python,
|
||||
args=(args.version, Path(args.output_dir)),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "zip":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"create_zip",
|
||||
fn=create_zip_package,
|
||||
args=(Path(args.source_dir), Path(args.output_file)),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "clean":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("clean_build", fn=clean_build_dir, args=(Path(DEFAULT_BUILD_DIR),))])
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,524 +0,0 @@
|
||||
"""PDF 工具模块.
|
||||
|
||||
提供 PDF 文件操作的常用功能封装,
|
||||
支持合并、拆分、压缩、加密、水印、OCR等功能.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
try:
|
||||
import fitz # PyMuPDF
|
||||
|
||||
HAS_PYMUPDF = True
|
||||
except ImportError:
|
||||
HAS_PYMUPDF = False
|
||||
|
||||
try:
|
||||
import pypdf
|
||||
|
||||
HAS_PYPDF = True
|
||||
except ImportError:
|
||||
HAS_PYPDF = False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
PDF_SUFFIX = ".pdf"
|
||||
DEFAULT_QUALITY = 75
|
||||
DEFAULT_PASSWORD = ""
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def pdf_merge(input_paths: list[Path], output_path: Path) -> None:
|
||||
"""合并多个 PDF 文件."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
writer = pypdf.PdfWriter()
|
||||
for input_path in input_paths:
|
||||
if input_path.exists():
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"合并完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_split(input_path: Path, output_dir: Path) -> None:
|
||||
"""拆分 PDF 文件为单页."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for i, page in enumerate(reader.pages):
|
||||
writer = pypdf.PdfWriter()
|
||||
writer.add_page(page)
|
||||
output_file = output_dir / f"{input_path.stem}_page_{i + 1}.pdf"
|
||||
with open(output_file, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"拆分完成: {output_dir}")
|
||||
|
||||
|
||||
def pdf_compress(input_path: Path, output_path: Path) -> None:
|
||||
"""压缩 PDF 文件."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path), garbage=4, deflate=True, clean=True)
|
||||
doc.close()
|
||||
|
||||
original_size = input_path.stat().st_size
|
||||
new_size = output_path.stat().st_size
|
||||
ratio = (1 - new_size / original_size) * 100
|
||||
print(f"压缩完成: {output_path} (缩小 {ratio:.1f}%)")
|
||||
|
||||
|
||||
def pdf_encrypt(input_path: Path, output_path: Path, password: str) -> None:
|
||||
"""加密 PDF 文件."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
writer = pypdf.PdfWriter()
|
||||
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
writer.encrypt(user_password=password, owner_password=password, use_128bit=True)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"加密完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_decrypt(input_path: Path, output_path: Path, password: str) -> None:
|
||||
"""解密 PDF 文件."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
if reader.is_encrypted:
|
||||
reader.decrypt(password)
|
||||
|
||||
writer = pypdf.PdfWriter()
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"解密完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_extract_text(input_path: Path, output_path: Path) -> None:
|
||||
"""提取 PDF 文本."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
text = ""
|
||||
for page in doc:
|
||||
text += page.get_text() + "\n\n"
|
||||
doc.close()
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(text, encoding="utf-8")
|
||||
print(f"文本提取完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_extract_images(input_path: Path, output_dir: Path) -> None:
|
||||
"""提取 PDF 图片."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
image_count = 0
|
||||
for page_num, page in enumerate(doc):
|
||||
images = page.get_images(full=True)
|
||||
for img_idx, img in enumerate(images):
|
||||
xref = img[0]
|
||||
base_image = doc.extract_image(xref)
|
||||
image_data = base_image["image"]
|
||||
image_ext = base_image["ext"]
|
||||
image_path = output_dir / f"page_{page_num + 1}_img_{img_idx + 1}.{image_ext}"
|
||||
image_path.write_bytes(image_data)
|
||||
image_count += 1
|
||||
|
||||
doc.close()
|
||||
print(f"图片提取完成: {output_dir} (共 {image_count} 张)")
|
||||
|
||||
|
||||
def pdf_add_watermark(input_path: Path, output_path: Path, text: str = "CONFIDENTIAL") -> None:
|
||||
"""添加 PDF 水印."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
for page in doc:
|
||||
rect = page.rect
|
||||
text_width = fitz.get_text_length(text, fontsize=48)
|
||||
x = (rect.width - text_width) / 2
|
||||
y = rect.height / 2
|
||||
page.insert_text((x, y), text, fontsize=48, rotate=45, color=(0, 0, 0))
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"水印添加完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_rotate(input_path: Path, output_path: Path, rotation: int = 90) -> None:
|
||||
"""旋转 PDF 页面."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
for page in doc:
|
||||
page.set_rotation(rotation)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"旋转完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_crop(input_path: Path, output_path: Path, margins: tuple[int, int, int, int]) -> None:
|
||||
"""裁剪 PDF 页面."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
left, top, right, bottom = margins
|
||||
|
||||
for page in doc:
|
||||
rect = page.rect
|
||||
new_rect = fitz.Rect(
|
||||
rect.x0 + left,
|
||||
rect.y0 + top,
|
||||
rect.x1 - right,
|
||||
rect.y1 - bottom,
|
||||
)
|
||||
page.set_cropbox(new_rect)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"裁剪完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_info(input_path: Path) -> None:
|
||||
"""显示 PDF 信息."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
print(f"文件: {input_path}")
|
||||
print(f"页数: {doc.page_count}")
|
||||
print(f"标题: {doc.metadata.get('title', 'N/A')}")
|
||||
print(f"作者: {doc.metadata.get('author', 'N/A')}")
|
||||
print(f"创建日期: {doc.metadata.get('creationDate', 'N/A')}")
|
||||
print(f"修改日期: {doc.metadata.get('modDate', 'N/A')}")
|
||||
print(f"文件大小: {input_path.stat().st_size / 1024:.1f} KB")
|
||||
doc.close()
|
||||
|
||||
|
||||
def pdf_ocr(input_path: Path, output_path: Path, lang: str = "chi_sim+eng") -> None:
|
||||
"""PDF OCR 识别."""
|
||||
try:
|
||||
import pytesseract
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
print("未安装 OCR 相关库,请安装: pip install pytesseract pillow")
|
||||
return
|
||||
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
new_doc = fitz.open()
|
||||
|
||||
for page in doc:
|
||||
pix = page.get_pixmap()
|
||||
img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
|
||||
ocr_text = pytesseract.image_to_string(img, lang=lang)
|
||||
|
||||
new_page = new_doc.new_page(width=page.rect.width, height=page.rect.height)
|
||||
new_page.insert_image(new_page.rect, pixmap=pix)
|
||||
text_rect = fitz.Rect(0, 0, page.rect.width, page.rect.height)
|
||||
new_page.insert_textbox(text_rect, ocr_text)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
new_doc.save(str(output_path))
|
||||
new_doc.close()
|
||||
doc.close()
|
||||
print(f"OCR 识别完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_reorder(input_path: Path, output_path: Path, order: list[int]) -> None:
|
||||
"""重排 PDF 页面顺序."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
writer = pypdf.PdfWriter()
|
||||
|
||||
for page_num in order:
|
||||
if 0 <= page_num < len(reader.pages):
|
||||
writer.add_page(reader.pages[page_num])
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"重排完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_to_images(input_path: Path, output_dir: Path, dpi: int = 300) -> None:
|
||||
"""PDF 转图片."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for page_num, page in enumerate(doc):
|
||||
pix = page.get_pixmap(dpi=dpi)
|
||||
image_path = output_dir / f"{input_path.stem}_page_{page_num + 1}.png"
|
||||
pix.save(str(image_path))
|
||||
|
||||
doc.close()
|
||||
print(f"转换完成: {output_dir}")
|
||||
|
||||
|
||||
def pdf_repair(input_path: Path, output_path: Path) -> None:
|
||||
"""修复 PDF 文件."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path), garbage=4, deflate=True, clean=True)
|
||||
doc.close()
|
||||
print(f"修复完成: {output_path}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None: # noqa: PLR0912
|
||||
"""PDF 工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="PDFTool - PDF 文件工具集",
|
||||
usage="pdftool <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 合并 PDF 命令
|
||||
merge_parser = subparsers.add_parser("m", help="合并 PDF 文件")
|
||||
merge_parser.add_argument("inputs", nargs="+", help="输入 PDF 文件路径")
|
||||
merge_parser.add_argument("--output", type=str, default="merged.pdf", help="输出文件路径")
|
||||
|
||||
# 拆分 PDF 命令
|
||||
split_parser = subparsers.add_parser("s", help="拆分 PDF 文件为单页")
|
||||
split_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
split_parser.add_argument("--output-dir", type=str, default="split", help="输出目录")
|
||||
|
||||
# 压缩 PDF 命令
|
||||
compress_parser = subparsers.add_parser("c", help="压缩 PDF 文件")
|
||||
compress_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
compress_parser.add_argument("--output", type=str, default="compressed.pdf", help="输出文件路径")
|
||||
|
||||
# 加密 PDF 命令
|
||||
encrypt_parser = subparsers.add_parser("e", help="加密 PDF 文件")
|
||||
encrypt_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
encrypt_parser.add_argument("--output", type=str, default="encrypted.pdf", help="输出文件路径")
|
||||
encrypt_parser.add_argument("--password", type=str, required=True, help="密码")
|
||||
|
||||
# 解密 PDF 命令
|
||||
decrypt_parser = subparsers.add_parser("d", help="解密 PDF 文件")
|
||||
decrypt_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
decrypt_parser.add_argument("--output", type=str, default="decrypted.pdf", help="输出文件路径")
|
||||
decrypt_parser.add_argument("--password", type=str, required=True, help="密码")
|
||||
|
||||
# 提取文本命令
|
||||
extract_text_parser = subparsers.add_parser("xt", help="提取 PDF 文本")
|
||||
extract_text_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
extract_text_parser.add_argument("--output", type=str, default="output.txt", help="输出文件路径")
|
||||
|
||||
# 提取图片命令
|
||||
extract_images_parser = subparsers.add_parser("xi", help="提取 PDF 图片")
|
||||
extract_images_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
extract_images_parser.add_argument("--output-dir", type=str, default="images", help="输出目录")
|
||||
|
||||
# 添加水印命令
|
||||
watermark_parser = subparsers.add_parser("w", help="添加 PDF 水印")
|
||||
watermark_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
watermark_parser.add_argument("--output", type=str, default="watermarked.pdf", help="输出文件路径")
|
||||
watermark_parser.add_argument("--text", type=str, default="CONFIDENTIAL", help="水印文本")
|
||||
|
||||
# 旋转 PDF 命令
|
||||
rotate_parser = subparsers.add_parser("r", help="旋转 PDF 页面")
|
||||
rotate_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
rotate_parser.add_argument("--output", type=str, default="rotated.pdf", help="输出文件路径")
|
||||
rotate_parser.add_argument("--rotation", type=int, default=90, help="旋转角度 (90, 180, 270)")
|
||||
|
||||
# 裁剪 PDF 命令
|
||||
crop_parser = subparsers.add_parser("crop", help="裁剪 PDF 页面")
|
||||
crop_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
crop_parser.add_argument("--output", type=str, default="cropped.pdf", help="输出文件路径")
|
||||
crop_parser.add_argument("--left", type=int, default=10, help="左边裁剪")
|
||||
crop_parser.add_argument("--top", type=int, default=10, help="顶部裁剪")
|
||||
crop_parser.add_argument("--right", type=int, default=10, help="右边裁剪")
|
||||
crop_parser.add_argument("--bottom", type=int, default=10, help="底部裁剪")
|
||||
|
||||
# 显示信息命令
|
||||
info_parser = subparsers.add_parser("i", help="显示 PDF 信息")
|
||||
info_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
|
||||
# OCR 识别命令
|
||||
ocr_parser = subparsers.add_parser("ocr", help="PDF OCR 识别")
|
||||
ocr_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
ocr_parser.add_argument("--output", type=str, default="ocr.pdf", help="输出文件路径")
|
||||
ocr_parser.add_argument("--lang", type=str, default="chi_sim+eng", help="OCR 语言")
|
||||
|
||||
# 转换图片命令
|
||||
to_images_parser = subparsers.add_parser("img", help="PDF 转图片")
|
||||
to_images_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
to_images_parser.add_argument("--output-dir", type=str, default="images", help="输出目录")
|
||||
to_images_parser.add_argument("--dpi", type=int, default=300, help="图片 DPI")
|
||||
|
||||
# 修复 PDF 命令
|
||||
repair_parser = subparsers.add_parser("repair", help="修复 PDF 文件")
|
||||
repair_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
repair_parser.add_argument("--output", type=str, default="repaired.pdf", help="输出文件路径")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "m":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_merge", fn=pdf_merge, args=([Path(p) for p in args.inputs], Path(args.output)))]
|
||||
)
|
||||
elif args.command == "s":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_split", fn=pdf_split, args=(Path(args.input), Path(args.output_dir)))]
|
||||
)
|
||||
elif args.command == "c":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_compress", fn=pdf_compress, args=(Path(args.input), Path(args.output)))]
|
||||
)
|
||||
elif args.command == "e":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_encrypt", fn=pdf_encrypt, args=(Path(args.input), Path(args.output), args.password))]
|
||||
)
|
||||
elif args.command == "d":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_decrypt", fn=pdf_decrypt, args=(Path(args.input), Path(args.output), args.password))]
|
||||
)
|
||||
elif args.command == "xt":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_extract_text", fn=pdf_extract_text, args=(Path(args.input), Path(args.output)))]
|
||||
)
|
||||
elif args.command == "xi":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_extract_images", fn=pdf_extract_images, args=(Path(args.input), Path(args.output_dir)))]
|
||||
)
|
||||
elif args.command == "w":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pdf_watermark",
|
||||
fn=pdf_add_watermark,
|
||||
args=(Path(args.input), Path(args.output)),
|
||||
kwargs={"text": args.text},
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "r":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pdf_rotate",
|
||||
fn=pdf_rotate,
|
||||
args=(Path(args.input), Path(args.output)),
|
||||
kwargs={"rotation": args.rotation},
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "crop":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pdf_crop",
|
||||
fn=pdf_crop,
|
||||
args=(Path(args.input), Path(args.output)),
|
||||
kwargs={"margins": (args.left, args.top, args.right, args.bottom)},
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "i":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("pdf_info", fn=pdf_info, args=(Path(args.input),))])
|
||||
elif args.command == "ocr":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_ocr", fn=pdf_ocr, args=(Path(args.input), Path(args.output)), kwargs={"lang": args.lang})]
|
||||
)
|
||||
elif args.command == "img":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pdf_to_images",
|
||||
fn=pdf_to_images,
|
||||
args=(Path(args.input), Path(args.output_dir)),
|
||||
kwargs={"dpi": args.dpi},
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "repair":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pdf_repair", fn=pdf_repair, args=(Path(args.input), Path(args.output)))]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,186 @@
|
||||
"""PyFlowX 统一 CLI 入口.
|
||||
|
||||
通过 ``pf <tool> [command] [options]`` 调用所有工具,
|
||||
工具定义在 ``configs/`` 目录下的 YAML 文件中.
|
||||
|
||||
用法
|
||||
----
|
||||
pf # 列出所有可用工具
|
||||
pf filedate # 查看 filedate 工具帮助
|
||||
pf filedate add a.txt # 调用 filedate 的 add 子命令
|
||||
pf pymake b # 调用 pymake 的 b 别名
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
|
||||
class PfApp:
|
||||
"""pf 统一入口应用.
|
||||
|
||||
路由 ``pf <tool> [command]`` 到 YAML 配置工具或传统 Python 工具.
|
||||
"""
|
||||
|
||||
_CONFIGS_DIR = Path(__file__).parent.parent / "configs"
|
||||
|
||||
# 工具名到 YAML 配置文件的映射 (支持短别名)
|
||||
_TOOL_ALIASES: dict[str, str] = {
|
||||
"autofmt": "autofmt",
|
||||
"af": "autofmt",
|
||||
"bump": "bumpversion",
|
||||
"bumpversion": "bumpversion",
|
||||
"bv": "bumpversion",
|
||||
"filedate": "filedate",
|
||||
"fd": "filedate",
|
||||
"filelevel": "filelevel",
|
||||
"fl": "filelevel",
|
||||
"folderback": "folderback",
|
||||
"foldback": "folderback",
|
||||
"fb": "folderback",
|
||||
"folderzip": "folderzip",
|
||||
"foldzip": "folderzip",
|
||||
"fz": "folderzip",
|
||||
"git": "gittool",
|
||||
"gitt": "gittool",
|
||||
"gittool": "gittool",
|
||||
"gt": "gittool",
|
||||
"ls": "lscalc",
|
||||
"lscalc": "lscalc",
|
||||
"pack": "packtool",
|
||||
"packtool": "packtool",
|
||||
"pk": "packtool",
|
||||
"pdf": "pdftool",
|
||||
"pdftool": "pdftool",
|
||||
"pt": "pdftool",
|
||||
"pip": "piptool",
|
||||
"pymake": "pymake",
|
||||
"piptool": "piptool",
|
||||
"pp": "piptool",
|
||||
"reseticon": "reseticoncache",
|
||||
"reseticoncache": "reseticoncache",
|
||||
"ric": "reseticoncache",
|
||||
"screenshot": "screenshot",
|
||||
"scrcap": "screenshot",
|
||||
"ss": "screenshot",
|
||||
"ssh": "sshcopyid",
|
||||
"sshcopy": "sshcopyid",
|
||||
"sshcopyid": "sshcopyid",
|
||||
"sc": "sshcopyid",
|
||||
}
|
||||
|
||||
# 传统工具: 有自己的 main() 函数 (无法 YAML 化的复杂逻辑)
|
||||
_LEGACY_TOOLS: dict[str, str] = {
|
||||
"emlman": "pyflowx.cli.emlmanager:main",
|
||||
"profiler": "pyflowx.cli.profiler:main",
|
||||
"pxp": "pyflowx.cli.profiler:main",
|
||||
"yamlrun": "pyflowx.cli.yamlrun:main",
|
||||
}
|
||||
|
||||
def __init__(self, argv: Sequence[str] | None = None) -> None:
|
||||
self._argv = list(argv) if argv is not None else sys.argv[1:]
|
||||
|
||||
def run(self) -> int:
|
||||
"""主入口, 返回退出码."""
|
||||
if not self._argv:
|
||||
self._list_tools()
|
||||
return 0
|
||||
|
||||
tool_name = self._argv[0]
|
||||
rest_argv = self._argv[1:]
|
||||
|
||||
resolved = self._resolve_tool(tool_name)
|
||||
if resolved is None:
|
||||
print(f"错误: 未知工具 '{tool_name}'", file=sys.stderr)
|
||||
print("运行 'pf' 查看可用工具列表", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
tool_type, target = resolved
|
||||
if tool_type == "legacy":
|
||||
return self._run_legacy(target, rest_argv)
|
||||
return self._run_yaml(target, rest_argv)
|
||||
|
||||
def _list_tools(self) -> None:
|
||||
"""列出所有可用工具."""
|
||||
print("PyFlowX 工具列表:")
|
||||
print()
|
||||
print("YAML 配置工具:")
|
||||
yaml_tools = sorted(set(self._TOOL_ALIASES.values()))
|
||||
for tool in yaml_tools:
|
||||
print(f" pf {tool:<15} - {self._tool_description(tool)}")
|
||||
print()
|
||||
print("传统工具:")
|
||||
for tool in sorted(self._LEGACY_TOOLS):
|
||||
print(f" pf {tool:<15}")
|
||||
print()
|
||||
print("示例:")
|
||||
print(" pf filedate add a.txt")
|
||||
print(" pf pymake b")
|
||||
|
||||
def _tool_description(self, tool_name: str) -> str:
|
||||
"""获取工具描述 (从 YAML cli.description)."""
|
||||
config_path = self._CONFIGS_DIR / f"{tool_name}.yaml"
|
||||
if not config_path.exists():
|
||||
return ""
|
||||
try:
|
||||
import yaml
|
||||
|
||||
data = yaml.safe_load(config_path.read_text(encoding="utf-8"))
|
||||
if isinstance(data, dict) and isinstance(data.get("cli"), dict):
|
||||
return str(data["cli"].get("description", ""))
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
def _resolve_tool(self, name: str) -> tuple[str, str] | None:
|
||||
"""解析工具名, 返回 (类型, 目标).
|
||||
|
||||
类型: "yaml" 或 "legacy"
|
||||
目标: YAML 文件名 (不含 .yaml) 或 legacy 模块路径
|
||||
"""
|
||||
if name in self._TOOL_ALIASES:
|
||||
return ("yaml", self._TOOL_ALIASES[name])
|
||||
if name in self._LEGACY_TOOLS:
|
||||
return ("legacy", self._LEGACY_TOOLS[name])
|
||||
return None
|
||||
|
||||
def _run_legacy(self, module_path: str, argv: list[str]) -> int:
|
||||
"""运行传统工具的 main() 函数."""
|
||||
module_name, func_name = module_path.split(":", 1)
|
||||
module = importlib.import_module(module_name)
|
||||
func = getattr(module, func_name)
|
||||
|
||||
original_argv = sys.argv
|
||||
sys.argv = [f"pf {module_name.split('.')[-1]}", *argv]
|
||||
try:
|
||||
func()
|
||||
return 0
|
||||
except SystemExit as e:
|
||||
return int(e.code) if e.code is not None else 0
|
||||
finally:
|
||||
sys.argv = original_argv
|
||||
|
||||
def _run_yaml(self, target: str, argv: list[str]) -> int:
|
||||
"""运行 YAML 配置工具."""
|
||||
config_path = self._CONFIGS_DIR / f"{target}.yaml"
|
||||
if not config_path.exists():
|
||||
print(f"错误: 未找到配置文件 '{config_path}'", file=sys.stderr)
|
||||
print("运行 'pf' 查看可用工具列表", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"运行配置文件 '{config_path}'")
|
||||
return px.run_cli(config_path, argv)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""pf 统一入口主函数."""
|
||||
sys.exit(PfApp().run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,201 +0,0 @@
|
||||
"""pip 包管理工具模块.
|
||||
|
||||
提供 pip 包管理操作的封装,
|
||||
支持安装、卸载、下载等功能.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import fnmatch
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
PACKAGE_DIR = "packages"
|
||||
REQUIREMENTS_FILE = "requirements.txt"
|
||||
|
||||
# 受保护的包名集合
|
||||
_PROTECTED_PACKAGES: frozenset[str] = frozenset(
|
||||
{
|
||||
"pyflowx",
|
||||
"bitool",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _get_installed_packages() -> list[str]:
|
||||
"""获取当前环境中所有已安装的包名."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pip", "list", "--format=freeze"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
packages: list[str] = []
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
if line and "==" in line:
|
||||
pkg_name = line.split("==")[0].strip()
|
||||
packages.append(pkg_name)
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
return []
|
||||
return packages
|
||||
|
||||
|
||||
def _expand_wildcard_packages(pattern: str) -> list[str]:
|
||||
"""展开通配符模式为实际的包名列表."""
|
||||
if not any(char in pattern for char in ["*", "?", "[", "]"]):
|
||||
return [pattern]
|
||||
|
||||
installed_packages = _get_installed_packages()
|
||||
matched = [pkg for pkg in installed_packages if fnmatch.fnmatchcase(pkg.lower(), pattern.lower())]
|
||||
return matched
|
||||
|
||||
|
||||
def _filter_protected_packages(packages: list[str]) -> list[str]:
|
||||
"""过滤掉受保护的包名."""
|
||||
safe = [p for p in packages if p.lower() not in {p.lower() for p in _PROTECTED_PACKAGES}]
|
||||
filtered = [p for p in packages if p.lower() in {p.lower() for p in _PROTECTED_PACKAGES}]
|
||||
if filtered:
|
||||
print(f"跳过受保护的包: {', '.join(filtered)}")
|
||||
return safe
|
||||
|
||||
|
||||
def pip_uninstall(pkg_names: list[str]) -> None:
|
||||
"""卸载包."""
|
||||
packages_to_uninstall: list[str] = []
|
||||
for pattern in pkg_names:
|
||||
packages_to_uninstall.extend(_expand_wildcard_packages(pattern))
|
||||
|
||||
packages_to_uninstall = _filter_protected_packages(packages_to_uninstall)
|
||||
|
||||
if not packages_to_uninstall:
|
||||
return
|
||||
|
||||
subprocess.run(["pip", "uninstall", "-y", *packages_to_uninstall], check=True)
|
||||
|
||||
|
||||
def pip_reinstall(pkg_names: list[str], offline: bool = False) -> None:
|
||||
"""重新安装包."""
|
||||
safe_pkgs = _filter_protected_packages(pkg_names)
|
||||
if not safe_pkgs:
|
||||
print("所有指定的包均为受保护包, 跳过重装")
|
||||
return
|
||||
|
||||
subprocess.run(["pip", "uninstall", "-y", *safe_pkgs], check=True)
|
||||
|
||||
options = ["--no-index", "--find-links", "."] if offline else []
|
||||
subprocess.run(["pip", "install", *options, *safe_pkgs], check=True)
|
||||
|
||||
|
||||
def pip_download(pkg_names: list[str], offline: bool = False) -> None:
|
||||
"""下载包."""
|
||||
options = ["--no-index", "--find-links", "."] if offline else []
|
||||
subprocess.run(
|
||||
["pip", "download", *pkg_names, *options, "-d", PACKAGE_DIR],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def pip_freeze() -> None:
|
||||
"""冻结依赖."""
|
||||
result = subprocess.run(
|
||||
["pip", "freeze", "--exclude-editable"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
Path(REQUIREMENTS_FILE).write_text(result.stdout)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""pip 工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="PipTool - pip 包管理工具",
|
||||
usage="piptool <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 安装命令
|
||||
install_parser = subparsers.add_parser("i", help="安装包")
|
||||
install_parser.add_argument("packages", nargs="+", help="要安装的包名")
|
||||
|
||||
# 卸载命令
|
||||
uninstall_parser = subparsers.add_parser("u", help="卸载包")
|
||||
uninstall_parser.add_argument("packages", nargs="+", help="要卸载的包名 (支持通配符)")
|
||||
|
||||
# 重装命令
|
||||
reinstall_parser = subparsers.add_parser("r", help="重新安装包")
|
||||
reinstall_parser.add_argument("packages", nargs="+", help="要重装的包名")
|
||||
reinstall_parser.add_argument("--offline", action="store_true", help="使用离线模式")
|
||||
|
||||
# 下载命令
|
||||
download_parser = subparsers.add_parser("d", help="下载包")
|
||||
download_parser.add_argument("packages", nargs="+", help="要下载的包名")
|
||||
download_parser.add_argument("--offline", action="store_true", help="使用离线模式")
|
||||
|
||||
# 升级 pip 命令
|
||||
subparsers.add_parser("up", help="升级 pip")
|
||||
|
||||
# 冻结依赖命令
|
||||
subparsers.add_parser("f", help="冻结依赖到 requirements.txt")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "i":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("pip_install", cmd=["pip", "install", *args.packages], verbose=True)])
|
||||
elif args.command == "u":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pip_uninstall", fn=pip_uninstall, args=(args.packages,), verbose=True)]
|
||||
)
|
||||
elif args.command == "r":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pip_reinstall",
|
||||
fn=pip_reinstall,
|
||||
args=(args.packages,),
|
||||
kwargs={"offline": args.offline},
|
||||
verbose=True,
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "d":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pip_download",
|
||||
fn=pip_download,
|
||||
args=(args.packages,),
|
||||
kwargs={"offline": args.offline},
|
||||
verbose=True,
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "up":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("pip_upgrade", cmd=["python", "-m", "pip", "install", "--upgrade", "pip"], verbose=True)]
|
||||
)
|
||||
elif args.command == "f":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("pip_freeze", fn=pip_freeze, verbose=True)])
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,272 @@
|
||||
"""pxp —— PyFlowX 性能分析器.
|
||||
|
||||
分析包含 ``px`` 调用的 Python 脚本,生成工作流执行性能剖面报告。
|
||||
|
||||
工作原理
|
||||
--------
|
||||
1. 注入 hook:monkey-patch ``pyflowx.run`` / ``pyflowx.executors.run`` /
|
||||
``pyflowx.runner.run``,捕获最后一次执行的 ``Graph`` 与 ``RunReport``。
|
||||
2. 执行目标脚本:用 ``runpy.run_path`` 以 ``__main__`` 身份执行,
|
||||
捕获 ``SystemExit``(脚本可能调 ``sys.exit``)。
|
||||
3. 生成报告:从捕获的 report + graph 构建 :class:`ProfileReport`,
|
||||
默认输出 HTML 并自动打开浏览器。
|
||||
|
||||
使用方式
|
||||
--------
|
||||
# 分析 pymake.py,生成 HTML 报告并打开浏览器
|
||||
pxp pymake.py
|
||||
|
||||
# 传递参数给被分析脚本(用 -- 分隔)
|
||||
pxp pymake.py -- t
|
||||
|
||||
# 指定输出文件
|
||||
pxp pymake.py -o report.html
|
||||
|
||||
# 不打开浏览器
|
||||
pxp pymake.py --no-browser
|
||||
|
||||
# 输出纯文本报告
|
||||
pxp pymake.py -E text
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["main"]
|
||||
|
||||
import argparse
|
||||
import runpy
|
||||
import sys
|
||||
import webbrowser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .. import executors as _executors
|
||||
from .. import runner as _runner
|
||||
from ..profiling import ProfileReport
|
||||
from ..report import RunReport
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
"""构建参数解析器。"""
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="pxp",
|
||||
description="PyFlowX 性能分析器:分析包含 px 调用的脚本,生成性能剖面报告。",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=(
|
||||
"示例:\n"
|
||||
" pxp pymake.py # 分析并打开 HTML 报告\n"
|
||||
" pxp pymake.py -- t # 传递参数 t 给脚本\n"
|
||||
" pxp pymake.py -E text # 输出纯文本报告\n"
|
||||
" pxp pymake.py -o out.html # 指定输出文件\n"
|
||||
),
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"--export",
|
||||
"-E",
|
||||
choices=["html", "text"],
|
||||
default="html",
|
||||
help="导出格式(默认: html)",
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"--no-browser",
|
||||
action="store_true",
|
||||
help="不自动打开浏览器(仅 HTML 格式有效)",
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
help="输出文件路径(默认: <script>_profile.html)",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def _capture_px_run() -> dict[str, Any]:
|
||||
"""注入 hook 捕获 px.run() 调用。
|
||||
|
||||
返回一个字典,``run()`` 执行后填充 ``graph`` 与 ``report``。
|
||||
同时返回还原函数用于 finally 块。
|
||||
|
||||
Note
|
||||
-----
|
||||
需同时 patch 三处引用:
|
||||
* ``pyflowx.executors.run`` —— 实际实现
|
||||
* ``pyflowx.runner.run`` —— ``CliRunner`` 直接 import 的引用
|
||||
* ``pyflowx.run`` —— 顶层包导出的引用(用户脚本常用 ``px.run()``)
|
||||
|
||||
另外 patch ``RunReport.__init__`` 以捕获 ``run()`` 内部创建的 report 实例。
|
||||
这对于 ``run()`` 抛出 ``TaskFailedError`` 的场景至关重要:此时 ``run()``
|
||||
不会正常返回 report,但 report 对象已在内部创建并填充了已执行任务的结果。
|
||||
通过 ``capture_enabled`` 标志确保只在 ``patched_run`` 调用期间捕获。
|
||||
"""
|
||||
captured: dict[str, Any] = {}
|
||||
original_exec_run = _executors.run
|
||||
original_runner_run = _runner.run
|
||||
# 惰性获取顶层 pyflowx.run 引用(避免循环导入)
|
||||
import pyflowx as px_mod
|
||||
|
||||
original_px_run = px_mod.run
|
||||
original_report_init = RunReport.__init__
|
||||
capture_enabled = [False]
|
||||
|
||||
def patched_report_init(self: RunReport, *args: Any, **kwargs: Any) -> None:
|
||||
original_report_init(self, *args, **kwargs)
|
||||
if capture_enabled[0]:
|
||||
captured["report"] = self
|
||||
|
||||
RunReport.__init__ = patched_report_init # type: ignore[assignment]
|
||||
|
||||
def patched_run(graph: Any, *args: Any, **kwargs: Any) -> RunReport:
|
||||
captured["graph"] = graph
|
||||
capture_enabled[0] = True
|
||||
try:
|
||||
report = original_exec_run(graph, *args, **kwargs)
|
||||
# 正常返回时确保 captured["report"] 是返回的 report
|
||||
captured["report"] = report
|
||||
return report
|
||||
finally:
|
||||
capture_enabled[0] = False
|
||||
|
||||
# patch 所有引用 run 的入口
|
||||
_executors.run = patched_run # type: ignore[assignment]
|
||||
_runner.run = patched_run # type: ignore[assignment]
|
||||
px_mod.run = patched_run # type: ignore[assignment]
|
||||
|
||||
def _restore() -> None:
|
||||
_executors.run = original_exec_run # type: ignore[assignment]
|
||||
_runner.run = original_runner_run # type: ignore[assignment]
|
||||
px_mod.run = original_px_run # type: ignore[assignment]
|
||||
RunReport.__init__ = original_report_init # type: ignore[assignment]
|
||||
|
||||
captured["_restore"] = _restore
|
||||
return captured
|
||||
|
||||
|
||||
def _run_target_script(script: Path, script_args: list[str]) -> dict[str, Any]:
|
||||
"""执行目标脚本。
|
||||
|
||||
将脚本所在目录加入 ``sys.path``,设置 ``sys.argv``,然后用
|
||||
``runpy.run_path`` 以 ``__main__`` 身份执行。捕获 ``SystemExit``。
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[str, Any]
|
||||
脚本模块的全局变量字典(含 ``main`` 等定义)。
|
||||
"""
|
||||
sys.argv = [str(script), *script_args]
|
||||
script_dir = str(script.parent.resolve())
|
||||
if script_dir not in sys.path:
|
||||
sys.path.insert(0, script_dir)
|
||||
return runpy.run_path(str(script), run_name="__main__")
|
||||
|
||||
|
||||
def _try_call_main(module_globals: dict[str, Any]) -> None:
|
||||
"""若模块定义了 ``main`` 可调用对象,调用它。
|
||||
|
||||
用于脚本无 ``if __name__ == "__main__"`` 块的场景(如通过 entry points
|
||||
注册的 CLI 工具脚本)。``main`` 通常调用 ``CliRunner.run_cli()``,
|
||||
后者读取 ``sys.argv[1:]`` 执行对应命令。
|
||||
"""
|
||||
main_fn = module_globals.get("main")
|
||||
if callable(main_fn):
|
||||
main_fn()
|
||||
|
||||
|
||||
def _output_report(
|
||||
profile: ProfileReport,
|
||||
export: str,
|
||||
output: str | None,
|
||||
script_stem: str,
|
||||
no_browser: bool,
|
||||
) -> None:
|
||||
"""输出性能报告。"""
|
||||
if export == "text":
|
||||
print(profile.describe())
|
||||
return
|
||||
|
||||
# HTML 格式
|
||||
html = profile.to_html()
|
||||
if output:
|
||||
out_path = Path(output)
|
||||
else:
|
||||
out_path = Path.cwd() / f"{script_stem}_profile.html"
|
||||
out_path.write_text(html, encoding="utf-8")
|
||||
print(f"HTML 报告已生成: {out_path}")
|
||||
|
||||
if not no_browser:
|
||||
try:
|
||||
webbrowser.open(f"file://{out_path.resolve()}")
|
||||
except Exception as e:
|
||||
print(f"警告:无法打开浏览器: {e}", file=sys.stderr)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""pxp CLI 入口。"""
|
||||
parser = _build_parser()
|
||||
pxp_args, remaining = parser.parse_known_args()
|
||||
|
||||
if not remaining:
|
||||
parser.print_help()
|
||||
sys.exit(2)
|
||||
|
||||
script_str = remaining[0]
|
||||
script_args = remaining[1:]
|
||||
script_path = Path(script_str).resolve()
|
||||
|
||||
if not script_path.is_file():
|
||||
print(f"错误:脚本不存在: {script_path}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# 注入 hook
|
||||
captured = _capture_px_run()
|
||||
|
||||
# 执行目标脚本
|
||||
print(f"正在分析: {script_path}")
|
||||
if script_args:
|
||||
print(f"脚本参数: {script_args}")
|
||||
print("-" * 60)
|
||||
|
||||
module_globals: dict[str, Any] = {}
|
||||
try:
|
||||
module_globals = _run_target_script(script_path, script_args)
|
||||
except SystemExit:
|
||||
# 脚本调用了 sys.exit,正常情况
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"警告:脚本执行抛出异常: {e}", file=sys.stderr)
|
||||
|
||||
# 若脚本执行未捕获到 run(),尝试调用模块的 main() 函数
|
||||
# (适用于无 ``if __name__ == "__main__"`` 块的 CLI 脚本)
|
||||
if captured.get("report") is None and module_globals:
|
||||
try:
|
||||
_try_call_main(module_globals)
|
||||
except SystemExit:
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"警告:调用 main() 抛出异常: {e}", file=sys.stderr)
|
||||
|
||||
# 还原 hook
|
||||
restore = captured.pop("_restore", None)
|
||||
if restore is not None:
|
||||
restore()
|
||||
|
||||
# 检查是否捕获到 run() 调用
|
||||
report = captured.get("report")
|
||||
graph = captured.get("graph")
|
||||
if report is None or graph is None:
|
||||
print("错误:未捕获到 px.run() 调用,无法生成性能报告", file=sys.stderr)
|
||||
print("请确保脚本通过 px.run() 或 CliRunner 执行任务流图。", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# 生成报告
|
||||
profile = ProfileReport.from_report(report, graph)
|
||||
_output_report(
|
||||
profile,
|
||||
export=pxp_args.export,
|
||||
output=pxp_args.output,
|
||||
script_stem=script_path.stem,
|
||||
no_browser=pxp_args.no_browser,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,125 +0,0 @@
|
||||
"""Python 构建工具模块.
|
||||
|
||||
完全替代传统的 Makefile,
|
||||
提供更好的跨平台兼容性和 Python 生态集成.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
def maturin_build_cmd() -> list[str]:
|
||||
"""获取 maturin 构建命令(根据平台自动添加参数).
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[str]
|
||||
完整的 maturin 构建命令列表.
|
||||
"""
|
||||
command = ["maturin", "build", "-r"].copy()
|
||||
if Constants.IS_WINDOWS:
|
||||
command.extend(["--target", "x86_64-win7-windows-msvc", "-Zbuild-std", "-i", "python3.8"])
|
||||
return command
|
||||
|
||||
|
||||
uv_build: px.TaskSpec = px.TaskSpec("uv_build", cmd=["uv", "build"])
|
||||
maturin_build: px.TaskSpec = px.TaskSpec("maturin_build", cmd=maturin_build_cmd())
|
||||
uv_sync: px.TaskSpec = px.TaskSpec("uv_sync", cmd=["uv", "sync"])
|
||||
git_clean: px.TaskSpec = px.TaskSpec("git_clean", cmd=["gitt", "c"])
|
||||
test: px.TaskSpec = px.TaskSpec(
|
||||
"test", cmd=["pytest", "-m", "not slow", "-n", "8", "--dist", "loadfile", "--color=yes", "--durations=10"]
|
||||
)
|
||||
test_fast: px.TaskSpec = px.TaskSpec(
|
||||
"test_fast", cmd=["pytest", "-m", "not slow", "--dist", "loadfile", "--color=yes", "--durations=10"]
|
||||
)
|
||||
test_coverage: px.TaskSpec = px.TaskSpec(
|
||||
"test_coverage",
|
||||
cmd=["pytest", "--cov", "-n", "8", "--dist", "loadfile", "--tb=short", "-v", "--color=yes", "--durations=10"],
|
||||
)
|
||||
ruff_lint: px.TaskSpec = px.TaskSpec("lint", cmd=["ruff", "check", "--fix", "--unsafe-fixes"])
|
||||
typecheck: px.TaskSpec = px.TaskSpec("pyrefly_check", cmd=["pyrefly", "check", "."])
|
||||
git_add_all: px.TaskSpec = px.TaskSpec("git_add_all", cmd=["git", "add", "-A"])
|
||||
bump: px.TaskSpec = px.TaskSpec("bumpversion", cmd=["bumpversion"])
|
||||
doc: px.TaskSpec = px.TaskSpec("doc", cmd=["sphinx-build", "-b", "html", "docs", "docs/_build"])
|
||||
git_push: px.TaskSpec = px.TaskSpec("git_push", cmd=["git", "push"])
|
||||
git_push_tags: px.TaskSpec = px.TaskSpec("git_push_tags", cmd=["git", "push", "--tags"])
|
||||
hatch_publish: px.TaskSpec = px.TaskSpec("publish_python", cmd=["hatch", "publish"])
|
||||
twine_publish: px.TaskSpec = px.TaskSpec("twine_publish", cmd=["twine", "upload", "--disable-progress-bar"])
|
||||
tox: px.TaskSpec = px.TaskSpec("tox", cmd=["tox", "-p", "auto"])
|
||||
|
||||
|
||||
def main():
|
||||
"""pymake 构建工具.
|
||||
|
||||
🔨 构建命令:
|
||||
pymake b - 构建 Python 主包 (uv build)
|
||||
pymake bc - 构建 Rust 核心模块 (maturin build)
|
||||
pymake ba - 构建所有包 (先 Python 后 Rust)
|
||||
|
||||
📦 安装命令 (开发模式):
|
||||
pymake sync - 安装依赖包 (uv sync)
|
||||
|
||||
🧹 清理命令:
|
||||
pymake c - 清理所有构建产物 (gitt c)
|
||||
|
||||
🛠️ 开发工具:
|
||||
pymake t - 运行测试 (pytest)
|
||||
pymake tc - 运行测试并生成覆盖率报告
|
||||
pymake tf - 运行快速测试 (pytest -m not slow)
|
||||
pymake lint - 代码格式化与检查 (ruff)
|
||||
pymake type - 类型检查 (mypy, ty)
|
||||
pymake doc - 构建文档 (sphinx)
|
||||
|
||||
🔬 多版本测试:
|
||||
pymake tox - 多版本 Python 测试 (tox -p auto)
|
||||
|
||||
📦 发布命令:
|
||||
pymake pb - 发布到 PyPI (twine + hatch)
|
||||
|
||||
� 版本管理:
|
||||
pymake bump - 自动升级版本号并提交修改 (清理 + 检查 + 格式化 + git add + bumpversion)
|
||||
|
||||
�💡 常用工作流:
|
||||
1. 日常开发: pymake lint && pymake t
|
||||
2. 构建发布包: pymake ba
|
||||
3. 多版本兼容性测试: pymake tox
|
||||
4. 发布到 PyPI: pymake pb
|
||||
|
||||
📝 示例:
|
||||
pymake ba # 构建所有包
|
||||
pymake sync # 安装依赖
|
||||
pymake t # 运行测试
|
||||
pymake tox # 多版本兼容性测试
|
||||
pymake lint # 格式化代码
|
||||
pymake type # 类型检查
|
||||
"""
|
||||
runner = px.CliRunner(
|
||||
strategy="sequential",
|
||||
description="PyMake - Python 构建工具",
|
||||
graphs={
|
||||
# 构建命令
|
||||
"b": px.Graph.from_specs([uv_build]),
|
||||
"bc": px.Graph.from_specs([maturin_build]),
|
||||
"ba": px.Graph.from_specs(["b", "bc"]),
|
||||
# 安装命令
|
||||
"sync": px.Graph.from_specs([uv_sync]),
|
||||
# 清理命令
|
||||
"c": px.Graph.from_specs([git_clean]),
|
||||
# 开发工具
|
||||
"bump": px.Graph.from_specs(["c", "tc", git_add_all, bump]),
|
||||
"bumpmi": px.Graph.from_specs([px.TaskSpec("bumpversion_minor", cmd=["bumpversion", "minor"])]),
|
||||
"cov": px.Graph.from_specs([git_clean, test_coverage]),
|
||||
"doc": px.Graph.from_specs([doc]),
|
||||
"lint": px.Graph.from_specs([ruff_lint]),
|
||||
"pb": px.Graph.from_specs([twine_publish, hatch_publish]),
|
||||
"t": px.Graph.from_specs([test]),
|
||||
"tf": px.Graph.from_specs([test_fast]),
|
||||
"tc": px.Graph.from_specs([typecheck, "lint"]),
|
||||
"tox": px.Graph.from_specs([tox]),
|
||||
# 发布命令
|
||||
"p": px.Graph.from_specs([git_clean, git_push, git_push_tags]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
@@ -1,163 +0,0 @@
|
||||
"""截图工具.
|
||||
|
||||
跨平台截图工具, 支持全屏截图和区域截图.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def get_screenshot_path(filename: str | None = None) -> Path:
|
||||
"""获取截图保存路径.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filename : str | None
|
||||
文件名, 如果为 None 则自动生成
|
||||
|
||||
Returns
|
||||
-------
|
||||
Path
|
||||
截图保存路径
|
||||
"""
|
||||
if filename is None:
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
filename = f"screenshot_{timestamp}.png"
|
||||
|
||||
screenshots_dir = Path.home() / "Pictures" / "screenshots"
|
||||
screenshots_dir.mkdir(parents=True, exist_ok=True)
|
||||
return screenshots_dir / filename
|
||||
|
||||
|
||||
def take_screenshot_full(filename: str | None = None) -> None:
|
||||
"""全屏截图.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filename : str | None
|
||||
文件名
|
||||
"""
|
||||
output_path = get_screenshot_path(filename)
|
||||
|
||||
if Constants.IS_WINDOWS:
|
||||
# Windows: 使用 PowerShell 截图
|
||||
ps_script = f"""
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
$screen = [System.Windows.Forms.Screen]::PrimaryScreen
|
||||
$bounds = $screen.Bounds
|
||||
$bitmap = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height
|
||||
$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
|
||||
$graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)
|
||||
$bitmap.Save('{output_path.as_posix()}')
|
||||
$graphics.Dispose()
|
||||
$bitmap.Dispose()
|
||||
"""
|
||||
subprocess.run(["powershell", "-Command", ps_script], check=True)
|
||||
elif Constants.IS_MACOS:
|
||||
# macOS: 使用 screencapture
|
||||
subprocess.run(["screencapture", "-x", str(output_path)], check=True)
|
||||
else:
|
||||
# Linux: 使用 gnome-screenshot 或 scrot
|
||||
try:
|
||||
subprocess.run(["gnome-screenshot", "-f", str(output_path)], check=True)
|
||||
except FileNotFoundError:
|
||||
subprocess.run(["scrot", str(output_path)], check=True)
|
||||
|
||||
print(f"截图已保存: {output_path}")
|
||||
|
||||
|
||||
def take_screenshot_area(filename: str | None = None) -> None:
|
||||
"""区域截图.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filename : str | None
|
||||
文件名
|
||||
"""
|
||||
output_path = get_screenshot_path(filename)
|
||||
|
||||
if Constants.IS_WINDOWS:
|
||||
# Windows: 使用 PowerShell 截图 (需要用户选择区域)
|
||||
ps_script = f"""
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
$form = New-Object System.Windows.Forms.Form
|
||||
$form.WindowState = 'Maximized'
|
||||
$form.FormBorderStyle = 'None'
|
||||
$form.BackColor = [System.Drawing.Color]::FromArgb(1, 0, 0)
|
||||
$form.Opacity = 0.5
|
||||
$form.TopMost = $true
|
||||
$form.Show()
|
||||
Start-Sleep -Milliseconds 100
|
||||
$screen = [System.Windows.Forms.Screen]::PrimaryScreen
|
||||
$bounds = $screen.Bounds
|
||||
$bitmap = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height
|
||||
$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
|
||||
$graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)
|
||||
$form.Close()
|
||||
$bitmap.Save('{output_path.as_posix()}')
|
||||
$graphics.Dispose()
|
||||
$bitmap.Dispose()
|
||||
"""
|
||||
subprocess.run(["powershell", "-Command", ps_script], check=True)
|
||||
elif Constants.IS_MACOS:
|
||||
# macOS: 使用 screencapture 交互模式
|
||||
subprocess.run(["screencapture", "-i", str(output_path)], check=True)
|
||||
else:
|
||||
# Linux: 使用 gnome-screenshot 交互模式
|
||||
try:
|
||||
subprocess.run(["gnome-screenshot", "-a", "-f", str(output_path)], check=True)
|
||||
except FileNotFoundError:
|
||||
subprocess.run(["scrot", "-s", str(output_path)], check=True)
|
||||
|
||||
print(f"截图已保存: {output_path}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""截图工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Screenshot - 截图工具",
|
||||
usage="screenshot <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 全屏截图命令
|
||||
full_parser = subparsers.add_parser("full", help="全屏截图")
|
||||
full_parser.add_argument("--filename", type=str, help="文件名")
|
||||
|
||||
# 区域截图命令
|
||||
area_parser = subparsers.add_parser("area", help="区域截图")
|
||||
area_parser.add_argument("--filename", type=str, help="文件名")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "full":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("screenshot_full", fn=take_screenshot_full, kwargs={"filename": args.filename})]
|
||||
)
|
||||
elif args.command == "area":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("screenshot_area", fn=take_screenshot_area, kwargs={"filename": args.filename})]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,122 +0,0 @@
|
||||
"""SSH 密钥部署工具.
|
||||
|
||||
类似 ssh-copy-id, 自动将 SSH 公钥部署到远程服务器,
|
||||
支持密码认证和密钥认证两种方式.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def ssh_copy_id(
|
||||
hostname: str,
|
||||
username: str,
|
||||
password: str,
|
||||
port: int = 22,
|
||||
keypath: str = "~/.ssh/id_rsa.pub",
|
||||
timeout: int = 30,
|
||||
) -> None:
|
||||
"""将 SSH 公钥部署到远程服务器.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
hostname : str
|
||||
远程服务器主机名或 IP 地址
|
||||
username : str
|
||||
远程服务器用户名
|
||||
password : str
|
||||
远程服务器密码
|
||||
port : int
|
||||
SSH 端口, 默认 22
|
||||
keypath : str
|
||||
公钥文件路径, 默认 ~/.ssh/id_rsa.pub
|
||||
timeout : int
|
||||
SSH 操作超时秒数, 默认 30
|
||||
"""
|
||||
# 读取公钥
|
||||
pub_key_path = Path(keypath).expanduser()
|
||||
if not pub_key_path.exists():
|
||||
print(f"公钥文件不存在: {pub_key_path}")
|
||||
sys.exit(1)
|
||||
|
||||
pub_key = pub_key_path.read_text().strip()
|
||||
|
||||
# 构建部署脚本
|
||||
script = f"""mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||
cd ~/.ssh && touch authorized_keys && chmod 600 authorized_keys
|
||||
grep -qF '{pub_key.split()[1]}' authorized_keys 2>/dev/null || echo '{pub_key}' >> authorized_keys"""
|
||||
|
||||
# 使用 sshpass 执行
|
||||
try:
|
||||
subprocess.run(
|
||||
[
|
||||
"sshpass",
|
||||
"-p",
|
||||
password,
|
||||
"ssh",
|
||||
"-p",
|
||||
str(port),
|
||||
"-o",
|
||||
"StrictHostKeyChecking=no",
|
||||
"-o",
|
||||
"UserKnownHostsFile=/dev/null",
|
||||
"-o",
|
||||
f"ConnectTimeout={timeout}",
|
||||
f"{username}@{hostname}",
|
||||
script,
|
||||
],
|
||||
check=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
print(f"SSH 密钥已部署到 {username}@{hostname}:{port}")
|
||||
except FileNotFoundError:
|
||||
print(f"未找到 sshpass 工具,请手动执行: ssh-copy-id -p {port} {username}@{hostname}")
|
||||
sys.exit(1)
|
||||
except subprocess.TimeoutExpired:
|
||||
print("SSH 连接超时")
|
||||
sys.exit(1)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"SSH 执行失败: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""SSH 密钥部署工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="SSHCopyID - SSH 密钥部署工具",
|
||||
usage="sshcopyid <hostname> <username> <password> [--port PORT] [--keypath KEYPATH]",
|
||||
)
|
||||
parser.add_argument("hostname", type=str, help="远程服务器主机名或 IP 地址")
|
||||
parser.add_argument("username", type=str, help="远程服务器用户名")
|
||||
parser.add_argument("password", type=str, help="远程服务器密码")
|
||||
parser.add_argument("--port", type=int, default=22, help="SSH 端口 (默认: 22)")
|
||||
parser.add_argument("--keypath", type=str, default="~/.ssh/id_rsa.pub", help="公钥文件路径")
|
||||
parser.add_argument("--timeout", type=int, default=30, help="SSH 操作超时秒数 (默认: 30)")
|
||||
args = parser.parse_args()
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"ssh_deploy",
|
||||
fn=ssh_copy_id,
|
||||
args=(args.hostname, args.username, args.password),
|
||||
kwargs={"port": args.port, "keypath": args.keypath, "timeout": args.timeout},
|
||||
)
|
||||
]
|
||||
)
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,15 @@
|
||||
"""清屏工具.
|
||||
|
||||
跨平台清屏工具, 支持终端和控制台清屏.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.tasks.system import clr
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""清屏工具主函数."""
|
||||
graph = px.Graph.from_specs([clr()])
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -35,6 +35,6 @@ def main() -> None:
|
||||
[
|
||||
px.TaskSpec(f"kill_{proc_name}", cmd=[*cmd, f"{proc_name}*"], verbose=True)
|
||||
for proc_name in args.process_names
|
||||
]
|
||||
],
|
||||
)
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,21 @@
|
||||
"""命令查找工具.
|
||||
|
||||
跨平台查找可执行命令路径, 类似 Unix 的 which 命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.tasks.system import which
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""命令查找工具主函数."""
|
||||
parser = argparse.ArgumentParser(description="Which - 命令查找工具")
|
||||
parser.add_argument("commands", nargs="+", help="要查找的命令名称, 如: python ls ps gcc...")
|
||||
args = parser.parse_args()
|
||||
|
||||
graph = px.Graph.from_specs([which(cmd) for cmd in args.commands])
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,51 +0,0 @@
|
||||
"""命令查找工具.
|
||||
|
||||
跨平台查找可执行命令路径, 类似 Unix 的 which 命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
|
||||
def which_command(command: str) -> Path | None:
|
||||
"""查找命令路径.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
command : str
|
||||
命令名称
|
||||
|
||||
Returns
|
||||
-------
|
||||
Path | None
|
||||
命令路径, 如果未找到则返回 None
|
||||
"""
|
||||
cmd_path = shutil.which(command)
|
||||
if cmd_path:
|
||||
print(f"匹配路径: - {cmd_path}")
|
||||
return Path(cmd_path)
|
||||
else:
|
||||
print(f"{command}: 未找到")
|
||||
return None
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""命令查找工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Which - 命令查找工具",
|
||||
usage="which <command> [command ...]",
|
||||
)
|
||||
parser.add_argument(
|
||||
"commands",
|
||||
type=str,
|
||||
nargs="+",
|
||||
help="要查找的命令名称 (如: python pip node npm git uv rustc cargo)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
graph = px.Graph.from_specs([px.TaskSpec(f"which_{cmd}", fn=which_command, args=(cmd,)) for cmd in args.commands])
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,109 @@
|
||||
"""YAML 任务编排执行工具.
|
||||
|
||||
从 YAML 文件加载 GitHub Actions 风格的任务图并执行.
|
||||
支持串并行编排、矩阵扇出、条件执行等 CI/CD 核心概念.
|
||||
|
||||
用法
|
||||
----
|
||||
yamlrun pipeline.yaml # 执行 YAML 任务图
|
||||
yamlrun pipeline.yaml --strategy thread # 指定执行策略
|
||||
yamlrun pipeline.yaml --dry-run # 仅打印任务分层, 不执行
|
||||
yamlrun pipeline.yaml --list # 列出所有任务名
|
||||
yamlrun pipeline.yaml --quiet # 静默模式
|
||||
|
||||
示例 YAML
|
||||
----------
|
||||
::
|
||||
|
||||
strategy: thread
|
||||
jobs:
|
||||
setup:
|
||||
cmd: ["git", "clone", "https://github.com/foo/bar"]
|
||||
build:
|
||||
needs: [setup]
|
||||
cmd: ["python", "-m", "build"]
|
||||
test:
|
||||
needs: [build]
|
||||
cmd: ["pytest"]
|
||||
strategy:
|
||||
matrix:
|
||||
python: ["3.8", "3.9"]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.executors import Strategy
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""YAML 任务编排执行工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="YamlRun - 从 YAML 文件加载并执行任务图",
|
||||
usage="yamlrun <file.yaml> [--strategy STRATEGY] [--dry-run] [--list] [--quiet]",
|
||||
)
|
||||
parser.add_argument("file", type=str, help="YAML 任务图文件路径")
|
||||
parser.add_argument(
|
||||
"--strategy",
|
||||
type=str,
|
||||
default=None,
|
||||
help="执行策略: sequential/thread/async/dependency (默认: YAML 中指定的策略或 dependency)",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true", help="仅打印任务分层, 不执行")
|
||||
parser.add_argument("--list", action="store_true", help="列出所有任务名后退出")
|
||||
parser.add_argument("--quiet", action="store_true", help="静默模式, 不打印详细输出")
|
||||
args = parser.parse_args()
|
||||
|
||||
file_path = Path(args.file)
|
||||
if not file_path.exists():
|
||||
print(f"错误: 文件不存在: {file_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
graph = px.Graph.from_yaml(file_path)
|
||||
except px.YamlLoadError as e:
|
||||
print(f"错误: YAML 加载失败: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if args.list:
|
||||
print("任务列表:")
|
||||
for name in graph.names:
|
||||
spec = graph.spec(name)
|
||||
deps = ", ".join(spec.depends_on) if spec.depends_on else "(无依赖)"
|
||||
print(f" - {name} (依赖: {deps})")
|
||||
sys.exit(0)
|
||||
|
||||
layers = graph.layers()
|
||||
print(f"任务分层 ({len(layers)} 层):")
|
||||
for i, layer in enumerate(layers):
|
||||
print(f" 层 {i + 1}: {layer}")
|
||||
|
||||
if args.dry_run:
|
||||
print("\n[dry-run] 跳过执行")
|
||||
sys.exit(0)
|
||||
|
||||
strategy = args.strategy or graph.defaults.strategy or "dependency"
|
||||
print(f"\n执行策略: {strategy}")
|
||||
print(f"任务总数: {len(graph.names)}")
|
||||
print("-" * 40)
|
||||
|
||||
report = px.run(graph, strategy=cast(Strategy, strategy), verbose=not args.quiet)
|
||||
|
||||
print("-" * 40)
|
||||
succeeded = report.succeeded_tasks()
|
||||
failed = report.failed_tasks()
|
||||
skipped = report.skipped_tasks()
|
||||
print(f"完成: {len(succeeded)} 成功 / {len(failed)} 失败 / {len(skipped)} 跳过 (共 {len(graph.names)})")
|
||||
|
||||
if failed:
|
||||
print(f"失败任务: {failed}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,100 @@
|
||||
"""命令执行器:把 :class:`~pyflowx.task.TaskSpec` 的 ``cmd`` 字段(list /
|
||||
shell 字符串 / 可调用对象)转换为统一执行入口。
|
||||
|
||||
历史背景:原 ``task.py`` 的模块文档声明其为"纯数据结构",但 ``_run_command``
|
||||
属于命令执行逻辑,违反单一职责。此处将其抽离,``TaskSpec`` 仅持有配置,
|
||||
执行逻辑集中于本模块,便于独立测试与维护。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from typing import Any, List, Union, cast
|
||||
|
||||
from .task import TaskSpec
|
||||
|
||||
__all__ = ["run_command"]
|
||||
|
||||
|
||||
def run_command(spec: TaskSpec[Any]) -> Any: # noqa: PLR0912
|
||||
"""执行 ``spec.cmd`` 指定的命令(list / shell 字符串 / 可调用对象)。
|
||||
|
||||
与原 ``TaskSpec._run_command`` 行为一致:
|
||||
|
||||
- 可调用对象:直接调用,异常包装为 :class:`RuntimeError`。
|
||||
- list / str:通过 :func:`subprocess.run` 执行,非零返回码抛
|
||||
:class:`RuntimeError`(``verbose=False`` 时附 stderr)。
|
||||
- ``verbose=True`` 时打印执行信息与返回码到 stdout。
|
||||
- ``cwd`` / ``env`` 通过 subprocess 参数隔离(进程级状态仅在 fn 任务路径
|
||||
使用,cmd 路径不依赖 ``os.chdir`` / ``os.environ``)。
|
||||
"""
|
||||
cmd = spec.cmd
|
||||
verbose = spec.verbose
|
||||
cwd = spec.cwd
|
||||
timeout = spec.timeout
|
||||
env_override = spec.env
|
||||
|
||||
# 可调用对象:直接调用,返回其结果。
|
||||
if callable(cmd) and not isinstance(cmd, (list, str)):
|
||||
name = getattr(cmd, "__name__", "callable")
|
||||
if verbose:
|
||||
print(f"[verbose] 执行可调用命令: {name}", flush=True)
|
||||
if cwd is not None:
|
||||
print(f"[verbose] 工作目录: {cwd}", flush=True)
|
||||
try:
|
||||
return cmd()
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"可调用命令执行异常: {name}: {e}") from e
|
||||
|
||||
is_list = isinstance(cmd, list)
|
||||
if is_list:
|
||||
cmd_str = " ".join(arg for arg in cmd) # type: ignore[union-attr]
|
||||
verb = "执行命令"
|
||||
label = "命令"
|
||||
else:
|
||||
cmd_str = cast(str, cmd)
|
||||
verb = "执行 Shell"
|
||||
label = "Shell 命令"
|
||||
|
||||
if verbose:
|
||||
print(f"[verbose] {verb}: {cmd_str}", flush=True)
|
||||
if cwd is not None:
|
||||
print(f"[verbose] 工作目录: {cwd}", flush=True)
|
||||
|
||||
# 合并环境变量
|
||||
run_env: dict[str, str] | None = None
|
||||
if env_override:
|
||||
run_env = dict(os.environ)
|
||||
run_env.update(env_override)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cast(Union[str, List[str]], cmd),
|
||||
shell=not is_list,
|
||||
cwd=cwd,
|
||||
env=run_env,
|
||||
timeout=timeout,
|
||||
capture_output=not verbose,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError(f"{label}未找到: {cmd_str}") from None
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError(f"{label}执行超时: {cmd_str} ({timeout}s)") from None
|
||||
except OSError as e:
|
||||
raise RuntimeError(f"{label}执行异常: {cmd_str}: {e}") from e
|
||||
|
||||
if verbose:
|
||||
print(f"[verbose] 返回码: {result.returncode}", flush=True)
|
||||
|
||||
if result.returncode == 0:
|
||||
if not verbose and result.stdout:
|
||||
print(result.stdout, end="", flush=True)
|
||||
return None
|
||||
|
||||
err_msg = f"{label}执行失败: `{cmd_str}`, 返回码: {result.returncode}"
|
||||
if not verbose and result.stderr.strip():
|
||||
err_msg += f"\n{result.stderr.strip()}"
|
||||
raise RuntimeError(err_msg)
|
||||
@@ -0,0 +1,115 @@
|
||||
"""图组合:将带字符串引用的多个图展开为纯 :class:`~pyflowx.graph.Graph`。
|
||||
|
||||
历史背景:原 ``graph.py`` 同时承载 DAG 构建/校验/分层与多图组合逻辑,
|
||||
职责过载。组合逻辑(:class:`GraphComposer` / :func:`compose`)与单图 DAG
|
||||
模型正交,此处抽离为独立模块,便于按需导入与独立演进。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from typing import Any
|
||||
|
||||
from .graph import Graph
|
||||
from .task import TaskSpec
|
||||
|
||||
__all__ = ["GraphComposer", "compose"]
|
||||
|
||||
|
||||
class GraphComposer:
|
||||
"""将带字符串引用的图展开为纯 :class:`TaskSpec` 图。
|
||||
|
||||
引用格式:
|
||||
* ``"command_name"`` —— 引用整个命令图。
|
||||
* ``"command_name.task_name"`` —— 引用特定任务。
|
||||
|
||||
引用按顺序展开,后续引用的任务依赖前面引用的最后一个任务;
|
||||
原始 ``TaskSpec`` 之间也按出现顺序串行依赖。
|
||||
"""
|
||||
|
||||
def __init__(self, graphs: dict[str, Graph]) -> None:
|
||||
self.graphs = graphs
|
||||
|
||||
def resolve_all(self) -> dict[str, Graph]:
|
||||
"""解析所有图的字符串引用,返回展开后的新图映射。"""
|
||||
resolved: dict[str, Graph] = {}
|
||||
for cmd_name, graph in self.graphs.items():
|
||||
resolved[cmd_name] = self.expand_refs(graph, cmd_name)
|
||||
return resolved
|
||||
|
||||
def expand_refs(self, graph: Graph, current_cmd: str) -> Graph:
|
||||
"""展开图中的字符串引用。若无 ``_pending_refs``,原样返回。"""
|
||||
pending_refs = graph._pending_refs
|
||||
if not pending_refs:
|
||||
return graph
|
||||
|
||||
all_specs: list[TaskSpec[Any]] = []
|
||||
previous_ref_last_task: str | None = None
|
||||
|
||||
for ref in pending_refs:
|
||||
expanded_specs = self.parse_ref(ref, current_cmd)
|
||||
if previous_ref_last_task and expanded_specs:
|
||||
for i, task in enumerate(expanded_specs):
|
||||
if i == 0 or not task.depends_on:
|
||||
expanded_specs[i] = replace(task, depends_on=tuple({*task.depends_on, previous_ref_last_task}))
|
||||
if expanded_specs:
|
||||
previous_ref_last_task = expanded_specs[-1].name
|
||||
all_specs.extend(expanded_specs)
|
||||
|
||||
original_specs = list(graph.all_specs().values())
|
||||
if original_specs:
|
||||
if previous_ref_last_task:
|
||||
first = original_specs[0]
|
||||
all_specs.append(replace(first, depends_on=tuple({*first.depends_on, previous_ref_last_task})))
|
||||
else:
|
||||
all_specs.append(original_specs[0])
|
||||
for i in range(1, len(original_specs)):
|
||||
current_task = original_specs[i]
|
||||
previous_task_name = original_specs[i - 1].name
|
||||
all_specs.append(
|
||||
replace(current_task, depends_on=tuple({*current_task.depends_on, previous_task_name}))
|
||||
)
|
||||
|
||||
return Graph.from_specs(all_specs, defaults=graph.defaults)
|
||||
|
||||
def parse_ref(self, ref: str, current_cmd: str) -> list[TaskSpec[Any]]:
|
||||
"""解析单个字符串引用,返回对应的 TaskSpec 列表。"""
|
||||
if ref == current_cmd:
|
||||
raise ValueError(f"循环引用: 命令 '{current_cmd}' 引用了自己")
|
||||
|
||||
if "." in ref:
|
||||
cmd_name, task_name = ref.split(".", 1)
|
||||
if cmd_name not in self.graphs:
|
||||
raise ValueError(f"引用的命令 '{cmd_name}' 不存在")
|
||||
ref_graph = self.graphs[cmd_name]
|
||||
if task_name not in ref_graph.all_specs():
|
||||
raise ValueError(f"任务 '{task_name}' 不存在于命令 '{cmd_name}' 中")
|
||||
return [ref_graph.all_specs()[task_name]]
|
||||
else:
|
||||
cmd_name = ref
|
||||
if cmd_name not in self.graphs:
|
||||
raise ValueError(f"引用的命令 '{cmd_name}' 不存在")
|
||||
ref_graph = self.graphs[cmd_name]
|
||||
ref_graph = self.expand_refs(ref_graph, cmd_name)
|
||||
return list(ref_graph.all_specs().values())
|
||||
|
||||
|
||||
def compose(
|
||||
graphs: dict[str, Graph],
|
||||
) -> dict[str, Graph]:
|
||||
"""编程式解析多图的字符串引用,返回展开后的新图映射。
|
||||
|
||||
与 :class:`GraphComposer` 等价,但作为独立函数暴露,供不使用
|
||||
:class:`~pyflowx.runner.CliRunner` 的编程式用户调用。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> graphs = {
|
||||
... "build": px.Graph.from_specs([px.TaskSpec("b", cmd=["echo", "b"])]),
|
||||
... "all": px.Graph.from_specs(["build", px.TaskSpec("t", cmd=["echo", "t"])]),
|
||||
... }
|
||||
>>> resolved = px.compose(graphs)
|
||||
>>> "b" in resolved["all"].all_specs()
|
||||
True
|
||||
"""
|
||||
return GraphComposer(graphs).resolve_all()
|
||||
+190
-163
@@ -1,18 +1,29 @@
|
||||
"""条件判断模块.
|
||||
|
||||
提供平台条件、应用安装条件等预定义条件判断函数,
|
||||
用于 TaskSpec 的条件执行功能.
|
||||
所有条件均为 ``Callable[[Context], bool]``,接收依赖上下文映射(可能为空)。
|
||||
这使得条件可基于上游任务的运行时返回值做决策,实现动态分支。
|
||||
|
||||
内置条件分两类:
|
||||
1. *静态条件* —— 不依赖上下文(平台/环境变量/安装检查),通过 ``_static``
|
||||
包装忽略传入的 context,便于作为模块级常量使用。
|
||||
2. *上下文条件* —— 基于上游结果判断,如 :meth:`BuiltinConditions.DEP_EQUALS`。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
# 条件判断函数类型
|
||||
Condition = Callable[[], bool]
|
||||
from .task import Condition, Context
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = ["BuiltinConditions", "Condition", "Constants"]
|
||||
|
||||
|
||||
class Constants:
|
||||
@@ -24,200 +35,216 @@ class Constants:
|
||||
IS_POSIX: bool = sys.platform != "win32"
|
||||
|
||||
|
||||
def _static(predicate: Callable[[], bool], name: str) -> Condition:
|
||||
"""将无参谓词包装为忽略上下文的 :class:`Condition`。"""
|
||||
|
||||
def _cond(_ctx: Context) -> bool:
|
||||
return predicate()
|
||||
|
||||
_cond.__name__ = name
|
||||
return _cond
|
||||
|
||||
|
||||
def _cond_name(cond: Condition) -> str:
|
||||
"""获取条件的可读名称。"""
|
||||
return getattr(cond, "__name__", repr(cond))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 模块级静态条件常量
|
||||
# ---------------------------------------------------------------------- #
|
||||
IS_WINDOWS: Condition = _static(lambda: Constants.IS_WINDOWS, "IS_WINDOWS")
|
||||
IS_LINUX: Condition = _static(lambda: Constants.IS_LINUX, "IS_LINUX")
|
||||
IS_MACOS: Condition = _static(lambda: Constants.IS_MACOS, "IS_MACOS")
|
||||
IS_POSIX: Condition = _static(lambda: Constants.IS_POSIX, "IS_POSIX")
|
||||
|
||||
|
||||
class BuiltinConditions:
|
||||
"""内置条件判断函数集合."""
|
||||
"""内置条件判断函数集合.
|
||||
|
||||
静态条件工厂返回忽略上下文的 :class:`Condition`;上下文条件工厂返回
|
||||
会读取依赖结果的 :class:`Condition`。
|
||||
"""
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 静态条件
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
def IS_WINDOWS() -> Condition:
|
||||
"""检查是否为 Windows 平台."""
|
||||
return IS_WINDOWS
|
||||
|
||||
@staticmethod
|
||||
def IS_WINDOWS() -> bool:
|
||||
"""是否为 Windows 平台."""
|
||||
return Constants.IS_WINDOWS
|
||||
def IS_LINUX() -> Condition:
|
||||
"""检查是否为 Linux 平台."""
|
||||
return IS_LINUX
|
||||
|
||||
@staticmethod
|
||||
def IS_LINUX() -> bool:
|
||||
bool = Constants.IS_LINUX
|
||||
return bool
|
||||
def IS_MACOS() -> Condition:
|
||||
"""检查是否为 macOS 平台."""
|
||||
return IS_MACOS
|
||||
|
||||
@staticmethod
|
||||
def IS_MACOS() -> bool:
|
||||
"""是否为 macOS 平台."""
|
||||
return Constants.IS_MACOS
|
||||
def IS_POSIX() -> Condition:
|
||||
"""检查是否为 POSIX 平台."""
|
||||
return IS_POSIX
|
||||
|
||||
@staticmethod
|
||||
def IS_POSIX() -> bool:
|
||||
"""是否为 POSIX 系统 (Linux/macOS)."""
|
||||
return Constants.IS_POSIX
|
||||
|
||||
@staticmethod
|
||||
def PYTHON_VERSION(major: int, minor: int | None = None) -> bool:
|
||||
"""检查 Python 版本是否匹配.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
major : int
|
||||
主版本号.
|
||||
minor : int | None
|
||||
次版本号, 若为 None 则仅检查主版本.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
版本是否匹配.
|
||||
"""
|
||||
def PYTHON_VERSION(major: int, minor: int | None = None) -> Condition:
|
||||
"""检查 Python 版本是否匹配."""
|
||||
if minor is None:
|
||||
return sys.version_info.major == major
|
||||
return sys.version_info.major == major and sys.version_info.minor == minor
|
||||
return _static(lambda: sys.version_info.major == major, f"PYTHON_VERSION({major})")
|
||||
return _static(
|
||||
lambda: sys.version_info.major == major and sys.version_info.minor == minor,
|
||||
f"PYTHON_VERSION({major},{minor})",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def PYTHON_VERSION_AT_LEAST(major: int, minor: int = 0) -> bool:
|
||||
"""检查 Python 版本是否 >= 指定版本.
|
||||
def PYTHON_VERSION_AT_LEAST(major: int, minor: int = 0) -> Condition:
|
||||
"""检查 Python 版本是否 >= 指定版本."""
|
||||
return _static(lambda: sys.version_info >= (major, minor), f"PYTHON_VERSION_AT_LEAST({major},{minor})")
|
||||
|
||||
Parameters
|
||||
----------
|
||||
major : int
|
||||
主版本号.
|
||||
minor : int
|
||||
次版本号.
|
||||
@staticmethod
|
||||
def IS_RUNNING(app_name: str) -> Condition:
|
||||
"""检查指定应用是否正在运行."""
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
当前版本是否 >= 指定版本.
|
||||
"""
|
||||
return sys.version_info >= (major, minor)
|
||||
def _check() -> bool:
|
||||
if Constants.IS_WINDOWS:
|
||||
result = subprocess.run(
|
||||
["tasklist", "/nh", "/fi", f"imagename eq {app_name}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return app_name.lower() in result.stdout.lower()
|
||||
else:
|
||||
result = subprocess.run(["pgrep", "-x", app_name], capture_output=True, check=False)
|
||||
return result.returncode == 0
|
||||
|
||||
return _static(_check, f"IS_RUNNING({app_name!r})")
|
||||
|
||||
@staticmethod
|
||||
def HAS_INSTALLED(app_name: str) -> Condition:
|
||||
"""检查指定应用是否已安装.
|
||||
"""检查指定应用是否已安装."""
|
||||
return _static(lambda: shutil.which(app_name) is not None, f"HAS_INSTALLED({app_name!r})")
|
||||
|
||||
Parameters
|
||||
----------
|
||||
app_name : str
|
||||
应用名称 (如 "git", "python", "pytest").
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
条件判断函数.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return shutil.which(app_name) is not None
|
||||
|
||||
_check.__name__ = f"HAS_INSTALLED({app_name!r})"
|
||||
return _check
|
||||
@staticmethod
|
||||
def DIR_EXISTS(path: Path) -> Condition:
|
||||
"""路径是否存在."""
|
||||
return _static(path.exists, f"DIR_EXISTS({path!r})")
|
||||
|
||||
@staticmethod
|
||||
def ENV_VAR_EXISTS(var_name: str) -> Condition:
|
||||
"""检查环境变量是否存在.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
var_name : str
|
||||
环境变量名.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
条件判断函数.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return var_name in os.environ
|
||||
|
||||
_check.__name__ = f"ENV_VAR_EXISTS({var_name!r})"
|
||||
return _check
|
||||
"""检查环境变量是否存在."""
|
||||
return _static(lambda: var_name in os.environ, f"ENV_VAR_EXISTS({var_name!r})")
|
||||
|
||||
@staticmethod
|
||||
def ENV_VAR_EQUALS(var_name: str, value: str) -> Condition:
|
||||
"""检查环境变量是否等于指定值.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
var_name : str
|
||||
环境变量名.
|
||||
value : str
|
||||
期望的值.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
条件判断函数.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return os.environ.get(var_name) == value
|
||||
|
||||
_check.__name__ = f"ENV_VAR_EQUALS({var_name!r}, {value!r})"
|
||||
return _check
|
||||
"""检查环境变量是否等于指定值."""
|
||||
return _static(
|
||||
lambda: os.environ.get(var_name) == value,
|
||||
f"ENV_VAR_EQUALS({var_name!r},{value!r})",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def NOT(condition: Condition) -> Condition:
|
||||
"""对条件取反.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
condition : Condition
|
||||
原始条件.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
取反后的条件.
|
||||
"""
|
||||
def FILE_CONTENT_EXISTS(path: Path | str, content: str) -> Condition:
|
||||
"""检查文件是否包含指定内容."""
|
||||
|
||||
def _check() -> bool:
|
||||
return not condition()
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
return False
|
||||
try:
|
||||
return content in p.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return False
|
||||
|
||||
_check.__name__ = f"NOT({getattr(condition, '__name__', repr(condition))})"
|
||||
return _check
|
||||
return _static(_check, f"FILE_CONTENT_EXISTS({path!r},{content!r})")
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 上下文条件:基于上游依赖结果
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
def DEP_EQUALS(dep_name: str, value: Any) -> Condition:
|
||||
"""上游任务 ``dep_name`` 的返回值等于 ``value`` 时为真。
|
||||
|
||||
若依赖未在上下文中(被跳过或未执行),返回 ``False``。
|
||||
"""
|
||||
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return dep_name in ctx and ctx[dep_name] == value
|
||||
|
||||
_cond.__name__ = f"DEP_EQUALS({dep_name!r},{value!r})"
|
||||
return _cond
|
||||
|
||||
@staticmethod
|
||||
def DEP_MATCHES(dep_name: str, predicate: Callable[[Any], bool]) -> Condition:
|
||||
"""上游任务 ``dep_name`` 的返回值满足 ``predicate`` 时为真。
|
||||
|
||||
依赖不存在时返回 ``False``。
|
||||
"""
|
||||
|
||||
def _cond(ctx: Context) -> bool:
|
||||
if dep_name not in ctx:
|
||||
return False
|
||||
try:
|
||||
return predicate(ctx[dep_name])
|
||||
except Exception as exc:
|
||||
logger.warning("DEP_MATCHES predicate %r raised: %r", dep_name, exc)
|
||||
return False
|
||||
|
||||
_cond.__name__ = f"DEP_MATCHES({dep_name!r},{getattr(predicate, '__name__', 'pred')})"
|
||||
return _cond
|
||||
|
||||
@staticmethod
|
||||
def DEP_PRESENT(dep_name: str) -> Condition:
|
||||
"""上游任务 ``dep_name`` 存在于上下文(即已成功执行)时为真。"""
|
||||
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return dep_name in ctx and ctx[dep_name] is not None
|
||||
|
||||
_cond.__name__ = f"DEP_PRESENT({dep_name!r})"
|
||||
return _cond
|
||||
|
||||
@staticmethod
|
||||
def DEP_TRUTHY(dep_name: str) -> Condition:
|
||||
"""上游任务 ``dep_name`` 的返回值为真值时为真。"""
|
||||
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return bool(ctx.get(dep_name))
|
||||
|
||||
_cond.__name__ = f"DEP_TRUTHY({dep_name!r})"
|
||||
return _cond
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 逻辑组合
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
def NOT(condition: Condition) -> Condition:
|
||||
"""对条件取反."""
|
||||
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return not condition(ctx)
|
||||
|
||||
_cond.__name__ = f"NOT({_cond_name(condition)})"
|
||||
return _cond
|
||||
|
||||
@staticmethod
|
||||
def AND(*conditions: Condition) -> Condition:
|
||||
"""多个条件的逻辑与.
|
||||
"""多个条件的逻辑与."""
|
||||
|
||||
Parameters
|
||||
----------
|
||||
*conditions : Condition
|
||||
条件列表.
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return all(c(ctx) for c in conditions)
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
组合条件.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return all(c() for c in conditions)
|
||||
|
||||
names = [getattr(c, "__name__", repr(c)) for c in conditions]
|
||||
_check.__name__ = f"AND({', '.join(names)})"
|
||||
return _check
|
||||
_cond.__name__ = f"AND({', '.join(_cond_name(c) for c in conditions)})"
|
||||
return _cond
|
||||
|
||||
@staticmethod
|
||||
def OR(*conditions: Condition) -> Condition:
|
||||
"""多个条件的逻辑或.
|
||||
"""多个条件的逻辑或."""
|
||||
|
||||
Parameters
|
||||
----------
|
||||
*conditions : Condition
|
||||
条件列表.
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return any(c(ctx) for c in conditions)
|
||||
|
||||
Returns
|
||||
-------
|
||||
Condition
|
||||
组合条件.
|
||||
"""
|
||||
|
||||
def _check() -> bool:
|
||||
return any(c() for c in conditions)
|
||||
|
||||
names = [getattr(c, "__name__", repr(c)) for c in conditions]
|
||||
_check.__name__ = f"OR({', '.join(names)})"
|
||||
return _check
|
||||
|
||||
|
||||
# 导出常用条件
|
||||
IS_WINDOWS: Callable[[], bool] = BuiltinConditions.IS_WINDOWS
|
||||
IS_LINUX: Callable[[], bool] = BuiltinConditions.IS_LINUX
|
||||
IS_MACOS: Callable[[], bool] = BuiltinConditions.IS_MACOS
|
||||
IS_POSIX: Callable[[], bool] = BuiltinConditions.IS_POSIX
|
||||
_cond.__name__ = f"OR({', '.join(_cond_name(c) for c in conditions)})"
|
||||
return _cond
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
# autofmt - 自动格式化工具
|
||||
# 用法:
|
||||
# pf autofmt fmt --target .
|
||||
# pf autofmt lint --target .
|
||||
# pf autofmt lint --target . --fix
|
||||
# pf autofmt doc --root-dir .
|
||||
# pf autofmt sync --root-dir .
|
||||
strategy: thread
|
||||
variables:
|
||||
TARGET: "."
|
||||
ROOT_DIR: "."
|
||||
FIX: false
|
||||
cli:
|
||||
description: "AutoFmt - 自动格式化工具"
|
||||
usage: "pf autofmt <command> [options]"
|
||||
subcommands:
|
||||
fmt:
|
||||
help: "格式化代码"
|
||||
options:
|
||||
- name: TARGET
|
||||
flag: "--target"
|
||||
type: str
|
||||
default: "."
|
||||
help: "目标路径 (默认: .)"
|
||||
lint:
|
||||
help: "代码检查"
|
||||
options:
|
||||
- name: TARGET
|
||||
flag: "--target"
|
||||
type: str
|
||||
default: "."
|
||||
help: "目标路径 (默认: .)"
|
||||
- name: FIX
|
||||
flag: "--fix"
|
||||
action: "store_true"
|
||||
help: "自动修复问题"
|
||||
doc:
|
||||
help: "自动添加文档字符串"
|
||||
options:
|
||||
- name: ROOT_DIR
|
||||
flag: "--root-dir"
|
||||
type: str
|
||||
default: "."
|
||||
help: "根目录 (默认: .)"
|
||||
sync:
|
||||
help: "同步 pyproject 配置"
|
||||
options:
|
||||
- name: ROOT_DIR
|
||||
flag: "--root-dir"
|
||||
type: str
|
||||
default: "."
|
||||
help: "根目录 (默认: .)"
|
||||
jobs:
|
||||
fmt:
|
||||
cmd: ["ruff", "format", "${TARGET}"]
|
||||
lint:
|
||||
cmd: ["ruff", "check", "${TARGET}"]
|
||||
lint_fix:
|
||||
cmd: ["ruff", "check", "--fix", "--unsafe-fixes", "${TARGET}"]
|
||||
doc:
|
||||
fn: auto_add_docstrings
|
||||
args: ["${ROOT_DIR}"]
|
||||
sync:
|
||||
fn: sync_pyproject_config
|
||||
args: ["${ROOT_DIR}"]
|
||||
@@ -0,0 +1,27 @@
|
||||
# bumpversion - 版本号自动管理工具
|
||||
# 用法:
|
||||
# pf bumpversion
|
||||
# pf bumpversion minor --no-tag
|
||||
strategy: sequential
|
||||
variables:
|
||||
PART: patch
|
||||
NO_TAG: false
|
||||
cli:
|
||||
description: "BumpVersion - 版本号自动管理工具"
|
||||
usage: "pf bumpversion [part] [options]"
|
||||
positional:
|
||||
- name: PART
|
||||
type: str
|
||||
default: patch
|
||||
help: "版本部分: patch, minor, major"
|
||||
options:
|
||||
- name: NO_TAG
|
||||
flag: "--no-tag"
|
||||
action: "store_true"
|
||||
help: "提交后不创建 git tag"
|
||||
jobs:
|
||||
bump:
|
||||
fn: bump_project_version
|
||||
args: ["${PART}"]
|
||||
kwargs:
|
||||
no_tag: ${NO_TAG}
|
||||
@@ -0,0 +1,36 @@
|
||||
# filedate - 文件日期处理工具
|
||||
# 用法:
|
||||
# pf filedate add file1.txt file2.txt
|
||||
# pf filedate clear file1.txt file2.txt
|
||||
strategy: thread
|
||||
variables:
|
||||
FILES: []
|
||||
cli:
|
||||
description: "FileDate - 文件日期处理工具"
|
||||
usage: "pf filedate <command> [files...]"
|
||||
subcommands:
|
||||
add:
|
||||
help: "添加日期前缀"
|
||||
positional:
|
||||
- name: FILES
|
||||
nargs: "+"
|
||||
type: path
|
||||
help: "文件路径"
|
||||
clear:
|
||||
help: "清除日期前缀"
|
||||
positional:
|
||||
- name: FILES
|
||||
nargs: "+"
|
||||
type: path
|
||||
help: "文件路径"
|
||||
jobs:
|
||||
add:
|
||||
fn: process_files_date
|
||||
args: ["${FILES}"]
|
||||
kwargs:
|
||||
clear: false
|
||||
clear:
|
||||
fn: process_files_date
|
||||
args: ["${FILES}"]
|
||||
kwargs:
|
||||
clear: true
|
||||
@@ -0,0 +1,28 @@
|
||||
# filelevel - 文件等级重命名工具
|
||||
# 用法:
|
||||
# pf filelevel set file.txt --level 2
|
||||
strategy: thread
|
||||
variables:
|
||||
FILES: []
|
||||
LEVEL: 0
|
||||
cli:
|
||||
description: "FileLevel - 文件等级重命名工具"
|
||||
usage: "pf filelevel <command> [files...] [options]"
|
||||
subcommands:
|
||||
set:
|
||||
help: "设置文件等级"
|
||||
positional:
|
||||
- name: FILES
|
||||
nargs: "+"
|
||||
type: path
|
||||
help: "文件路径"
|
||||
options:
|
||||
- name: LEVEL
|
||||
flag: "--level"
|
||||
type: int
|
||||
required: true
|
||||
help: "文件等级 (0-4)"
|
||||
jobs:
|
||||
set:
|
||||
fn: process_files_level
|
||||
args: ["${FILES}", "${LEVEL}"]
|
||||
@@ -0,0 +1,34 @@
|
||||
# folderback - 文件夹备份工具
|
||||
# 用法:
|
||||
# pf folderback
|
||||
# pf folderback --src ./project --dst ./backup --max-zip 10
|
||||
strategy: thread
|
||||
variables:
|
||||
SRC: "."
|
||||
DST: "./backup"
|
||||
MAX_ZIP: 5
|
||||
cli:
|
||||
description: "FolderBack - 文件夹备份工具"
|
||||
usage: "pf folderback [options]"
|
||||
options:
|
||||
- name: SRC
|
||||
flag: "--src"
|
||||
type: str
|
||||
default: "."
|
||||
help: "源文件夹路径 (默认: 当前目录)"
|
||||
- name: DST
|
||||
flag: "--dst"
|
||||
type: str
|
||||
default: "./backup"
|
||||
help: "目标文件夹路径 (默认: ./backup)"
|
||||
- name: MAX_ZIP
|
||||
flag: "--max-zip"
|
||||
type: int
|
||||
default: 5
|
||||
help: "最大备份数量 (默认: 5)"
|
||||
jobs:
|
||||
backup:
|
||||
fn: backup_folder
|
||||
args: ["${SRC}", "${DST}"]
|
||||
kwargs:
|
||||
max_zip: ${MAX_ZIP}
|
||||
@@ -0,0 +1,21 @@
|
||||
# folderzip - 文件夹压缩工具
|
||||
# 用法:
|
||||
# pf folderzip
|
||||
# pf folderzip --cwd ./project
|
||||
strategy: thread
|
||||
variables:
|
||||
CWD: "."
|
||||
cli:
|
||||
description: "FolderZip - 文件夹压缩工具"
|
||||
usage: "pf folderzip [options]"
|
||||
options:
|
||||
- name: CWD
|
||||
flag: "--cwd"
|
||||
type: str
|
||||
required: false
|
||||
default: "."
|
||||
help: "工作目录 (默认: 当前目录)"
|
||||
jobs:
|
||||
zip:
|
||||
fn: zip_folders
|
||||
args: ["${CWD}"]
|
||||
@@ -0,0 +1,51 @@
|
||||
# gittool - Git 执行工具
|
||||
# 用法:
|
||||
# pf gittool a
|
||||
# pf gittool c
|
||||
# pf gittool i
|
||||
# pf gittool isub
|
||||
# pf gittool p
|
||||
# pf gittool pl
|
||||
strategy: thread
|
||||
variables:
|
||||
# git clean -e 参数列表 (展开为 cmd 数组元素)
|
||||
CLEAN_EXCLUDES: ["-e", ".venv", "-e", ".tox", "-e", ".pytest_cache",
|
||||
"-e", ".ruff_cache", "-e", "node_modules",
|
||||
"-e", ".idea", "-e", ".vscode",
|
||||
"-e", ".trae", "-e", ".qoder",
|
||||
"-e", ".editorconfig", "-e", "idea.config",
|
||||
"-e", "idea_modules.xml", "-e", "vcs.xml"]
|
||||
cli:
|
||||
description: "GitTool - Git 执行工具"
|
||||
usage: "pf gittool <command>"
|
||||
subcommands:
|
||||
a:
|
||||
help: "添加并提交"
|
||||
c:
|
||||
help: "清理并查看状态"
|
||||
i:
|
||||
help: "初始化并提交"
|
||||
isub:
|
||||
help: "初始化子目录"
|
||||
p:
|
||||
help: "推送"
|
||||
pl:
|
||||
help: "拉取"
|
||||
jobs:
|
||||
a:
|
||||
fn: git_add_commit
|
||||
args: ["chore: update"]
|
||||
clean:
|
||||
cmd: ["git", "clean", "-xfd", "${CLEAN_EXCLUDES}"]
|
||||
c:
|
||||
needs: [clean]
|
||||
cmd: ["git", "status", "--porcelain"]
|
||||
i:
|
||||
fn: git_init_add_commit
|
||||
args: ["init commit"]
|
||||
isub:
|
||||
fn: init_sub_dirs
|
||||
p:
|
||||
cmd: ["git", "push"]
|
||||
pl:
|
||||
cmd: ["git", "pull"]
|
||||
@@ -0,0 +1,51 @@
|
||||
# lscalc - LS-DYNA 计算工具
|
||||
# 用法:
|
||||
# pf lscalc run input.k --ncpu 4
|
||||
# pf lscalc status
|
||||
strategy: thread
|
||||
variables:
|
||||
INPUT_FILE: input.k
|
||||
NCPU: 4
|
||||
cli:
|
||||
description: "LSCalc - LS-DYNA 计算工具"
|
||||
usage: "pf lscalc <command> [options]"
|
||||
subcommands:
|
||||
run:
|
||||
help: "运行 LS-DYNA 计算"
|
||||
positional:
|
||||
- name: INPUT_FILE
|
||||
type: str
|
||||
help: "输入文件路径"
|
||||
options:
|
||||
- name: NCPU
|
||||
flag: "--ncpu"
|
||||
type: int
|
||||
default: 4
|
||||
help: "CPU 核心数 (默认: 4)"
|
||||
mpi:
|
||||
help: "运行 LS-DYNA MPI 计算"
|
||||
positional:
|
||||
- name: INPUT_FILE
|
||||
type: str
|
||||
help: "输入文件路径"
|
||||
options:
|
||||
- name: NCPU
|
||||
flag: "--ncpu"
|
||||
type: int
|
||||
default: 4
|
||||
help: "CPU 核心数 (默认: 4)"
|
||||
status:
|
||||
help: "检查 LS-DYNA 进程状态"
|
||||
jobs:
|
||||
run:
|
||||
fn: run_ls_dyna
|
||||
args: ["${INPUT_FILE}"]
|
||||
kwargs:
|
||||
ncpu: ${NCPU}
|
||||
mpi:
|
||||
fn: run_ls_dyna_mpi
|
||||
args: ["${INPUT_FILE}"]
|
||||
kwargs:
|
||||
ncpu: ${NCPU}
|
||||
status:
|
||||
fn: check_ls_dyna_status
|
||||
@@ -0,0 +1,107 @@
|
||||
# packtool - Python 打包工具
|
||||
# 用法:
|
||||
# pf packtool src --project-dir . --output-dir .pypack
|
||||
# pf packtool deps requests numpy --lib-dir libs
|
||||
# pf packtool wheel --project-dir . --output-dir dist
|
||||
# pf packtool embed --version 3.10 --output-dir python
|
||||
# pf packtool zip --source-dir . --output-file package.zip
|
||||
# pf packtool clean
|
||||
strategy: thread
|
||||
variables:
|
||||
PROJECT_DIR: "."
|
||||
OUTPUT_DIR: ".pypack"
|
||||
LIB_DIR: "libs"
|
||||
DEPENDENCIES: []
|
||||
VERSION: "3.10"
|
||||
OUTPUT_FILE: "package.zip"
|
||||
SOURCE_DIR: "."
|
||||
cli:
|
||||
description: "PackTool - Python 打包工具"
|
||||
usage: "pf packtool <command> [options]"
|
||||
subcommands:
|
||||
src:
|
||||
help: "打包源码"
|
||||
options:
|
||||
- name: PROJECT_DIR
|
||||
flag: "--project-dir"
|
||||
type: path
|
||||
default: "."
|
||||
help: "项目目录 (默认: .)"
|
||||
- name: OUTPUT_DIR
|
||||
flag: "--output-dir"
|
||||
type: str
|
||||
default: ".pypack"
|
||||
help: "输出目录 (默认: .pypack)"
|
||||
deps:
|
||||
help: "打包依赖"
|
||||
positional:
|
||||
- name: DEPENDENCIES
|
||||
nargs: "*"
|
||||
type: str
|
||||
help: "依赖包列表"
|
||||
options:
|
||||
- name: LIB_DIR
|
||||
flag: "--lib-dir"
|
||||
type: path
|
||||
default: "libs"
|
||||
help: "依赖库目录 (默认: libs)"
|
||||
wheel:
|
||||
help: "构建 wheel"
|
||||
options:
|
||||
- name: PROJECT_DIR
|
||||
flag: "--project-dir"
|
||||
type: path
|
||||
default: "."
|
||||
help: "项目目录 (默认: .)"
|
||||
- name: OUTPUT_DIR
|
||||
flag: "--output-dir"
|
||||
type: path
|
||||
default: "dist"
|
||||
help: "输出目录 (默认: dist)"
|
||||
embed:
|
||||
help: "安装嵌入式 Python"
|
||||
options:
|
||||
- name: VERSION
|
||||
flag: "--version"
|
||||
type: str
|
||||
default: "3.10"
|
||||
help: "Python 版本 (默认: 3.10)"
|
||||
- name: OUTPUT_DIR
|
||||
flag: "--output-dir"
|
||||
type: path
|
||||
default: "python"
|
||||
help: "输出目录 (默认: python)"
|
||||
zip:
|
||||
help: "创建 zip 包"
|
||||
options:
|
||||
- name: SOURCE_DIR
|
||||
flag: "--source-dir"
|
||||
type: path
|
||||
default: "."
|
||||
help: "源目录 (默认: .)"
|
||||
- name: OUTPUT_FILE
|
||||
flag: "--output-file"
|
||||
type: path
|
||||
default: "package.zip"
|
||||
help: "输出文件 (默认: package.zip)"
|
||||
clean:
|
||||
help: "清理构建目录"
|
||||
jobs:
|
||||
src:
|
||||
fn: pack_source
|
||||
args: ["${PROJECT_DIR}", "${OUTPUT_DIR}"]
|
||||
deps:
|
||||
fn: pack_dependencies
|
||||
args: ["${LIB_DIR}", "${DEPENDENCIES}"]
|
||||
wheel:
|
||||
fn: pack_wheel
|
||||
args: ["${PROJECT_DIR}", "${OUTPUT_DIR}"]
|
||||
embed:
|
||||
fn: install_embed_python
|
||||
args: ["${VERSION}", "${OUTPUT_DIR}"]
|
||||
zip:
|
||||
fn: create_zip_package
|
||||
args: ["${SOURCE_DIR}", "${OUTPUT_FILE}"]
|
||||
clean:
|
||||
fn: clean_build_dir
|
||||
args: ["${OUTPUT_DIR}"]
|
||||
@@ -0,0 +1,303 @@
|
||||
# pdftool - PDF 文件工具集
|
||||
# 用法:
|
||||
# pf pdftool m a.pdf b.pdf --output merged.pdf
|
||||
# pf pdftool s input.pdf --output-dir split
|
||||
# pf pdftool c input.pdf --output compressed.pdf
|
||||
# pf pdftool e input.pdf --output encrypted.pdf --password 123456
|
||||
# pf pdftool d input.pdf --output decrypted.pdf --password 123456
|
||||
# pf pdftool xt input.pdf --output output.txt
|
||||
# pf pdftool xi input.pdf --output-dir images
|
||||
# pf pdftool w input.pdf --output watermarked.pdf --text CONFIDENTIAL
|
||||
# pf pdftool r input.pdf --output rotated.pdf --rotation 90
|
||||
# pf pdftool crop input.pdf --output cropped.pdf --left 10 --top 10 --right 10 --bottom 10
|
||||
# pf pdftool i input.pdf
|
||||
# pf pdftool ocr input.pdf --output ocr.pdf --lang chi_sim+eng
|
||||
# pf pdftool img input.pdf --output-dir images --dpi 300
|
||||
# pf pdftool repair input.pdf --output repaired.pdf
|
||||
strategy: thread
|
||||
variables:
|
||||
INPUT: input.pdf
|
||||
INPUTS: []
|
||||
OUTPUT: output.pdf
|
||||
OUTPUT_DIR: output
|
||||
PASSWORD: ""
|
||||
TEXT: CONFIDENTIAL
|
||||
ROTATION: 90
|
||||
MARGINS: [10, 10, 10, 10]
|
||||
DPI: 300
|
||||
LANG: chi_sim+eng
|
||||
ORDER: []
|
||||
LEFT: 10
|
||||
TOP: 10
|
||||
RIGHT: 10
|
||||
BOTTOM: 10
|
||||
cli:
|
||||
description: "PdfTool - PDF 文件工具集"
|
||||
usage: "pf pdftool <command> [options]"
|
||||
subcommands:
|
||||
m:
|
||||
help: "合并 PDF"
|
||||
positional:
|
||||
- name: INPUTS
|
||||
nargs: "+"
|
||||
type: path
|
||||
help: "输入 PDF 文件列表"
|
||||
options:
|
||||
- name: OUTPUT
|
||||
flag: "--output"
|
||||
type: path
|
||||
default: "merged.pdf"
|
||||
help: "输出文件 (默认: merged.pdf)"
|
||||
s:
|
||||
help: "拆分 PDF"
|
||||
positional:
|
||||
- name: INPUT
|
||||
type: path
|
||||
help: "输入 PDF 文件"
|
||||
options:
|
||||
- name: OUTPUT_DIR
|
||||
flag: "--output-dir"
|
||||
type: path
|
||||
default: "split"
|
||||
help: "输出目录 (默认: split)"
|
||||
c:
|
||||
help: "压缩 PDF"
|
||||
positional:
|
||||
- name: INPUT
|
||||
type: path
|
||||
help: "输入 PDF 文件"
|
||||
options:
|
||||
- name: OUTPUT
|
||||
flag: "--output"
|
||||
type: path
|
||||
default: "compressed.pdf"
|
||||
help: "输出文件 (默认: compressed.pdf)"
|
||||
e:
|
||||
help: "加密 PDF"
|
||||
positional:
|
||||
- name: INPUT
|
||||
type: path
|
||||
help: "输入 PDF 文件"
|
||||
options:
|
||||
- name: OUTPUT
|
||||
flag: "--output"
|
||||
type: path
|
||||
default: "encrypted.pdf"
|
||||
help: "输出文件 (默认: encrypted.pdf)"
|
||||
- name: PASSWORD
|
||||
flag: "--password"
|
||||
type: str
|
||||
required: true
|
||||
help: "密码 (必填)"
|
||||
d:
|
||||
help: "解密 PDF"
|
||||
positional:
|
||||
- name: INPUT
|
||||
type: path
|
||||
help: "输入 PDF 文件"
|
||||
options:
|
||||
- name: OUTPUT
|
||||
flag: "--output"
|
||||
type: path
|
||||
default: "decrypted.pdf"
|
||||
help: "输出文件 (默认: decrypted.pdf)"
|
||||
- name: PASSWORD
|
||||
flag: "--password"
|
||||
type: str
|
||||
required: true
|
||||
help: "密码 (必填)"
|
||||
xt:
|
||||
help: "提取文本"
|
||||
positional:
|
||||
- name: INPUT
|
||||
type: path
|
||||
help: "输入 PDF 文件"
|
||||
options:
|
||||
- name: OUTPUT
|
||||
flag: "--output"
|
||||
type: path
|
||||
default: "output.txt"
|
||||
help: "输出文件 (默认: output.txt)"
|
||||
xi:
|
||||
help: "提取图片"
|
||||
positional:
|
||||
- name: INPUT
|
||||
type: path
|
||||
help: "输入 PDF 文件"
|
||||
options:
|
||||
- name: OUTPUT_DIR
|
||||
flag: "--output-dir"
|
||||
type: path
|
||||
default: "images"
|
||||
help: "输出目录 (默认: images)"
|
||||
w:
|
||||
help: "添加水印"
|
||||
positional:
|
||||
- name: INPUT
|
||||
type: path
|
||||
help: "输入 PDF 文件"
|
||||
options:
|
||||
- name: OUTPUT
|
||||
flag: "--output"
|
||||
type: path
|
||||
default: "watermarked.pdf"
|
||||
help: "输出文件 (默认: watermarked.pdf)"
|
||||
- name: TEXT
|
||||
flag: "--text"
|
||||
type: str
|
||||
default: "CONFIDENTIAL"
|
||||
help: "水印文字 (默认: CONFIDENTIAL)"
|
||||
r:
|
||||
help: "旋转 PDF"
|
||||
positional:
|
||||
- name: INPUT
|
||||
type: path
|
||||
help: "输入 PDF 文件"
|
||||
options:
|
||||
- name: OUTPUT
|
||||
flag: "--output"
|
||||
type: path
|
||||
default: "rotated.pdf"
|
||||
help: "输出文件 (默认: rotated.pdf)"
|
||||
- name: ROTATION
|
||||
flag: "--rotation"
|
||||
type: int
|
||||
default: 90
|
||||
help: "旋转角度 (默认: 90)"
|
||||
crop:
|
||||
help: "裁剪 PDF"
|
||||
positional:
|
||||
- name: INPUT
|
||||
type: path
|
||||
help: "输入 PDF 文件"
|
||||
options:
|
||||
- name: OUTPUT
|
||||
flag: "--output"
|
||||
type: path
|
||||
default: "cropped.pdf"
|
||||
help: "输出文件 (默认: cropped.pdf)"
|
||||
- name: LEFT
|
||||
flag: "--left"
|
||||
type: int
|
||||
default: 10
|
||||
help: "左边距 (默认: 10)"
|
||||
- name: TOP
|
||||
flag: "--top"
|
||||
type: int
|
||||
default: 10
|
||||
help: "上边距 (默认: 10)"
|
||||
- name: RIGHT
|
||||
flag: "--right"
|
||||
type: int
|
||||
default: 10
|
||||
help: "右边距 (默认: 10)"
|
||||
- name: BOTTOM
|
||||
flag: "--bottom"
|
||||
type: int
|
||||
default: 10
|
||||
help: "下边距 (默认: 10)"
|
||||
i:
|
||||
help: "查看 PDF 信息"
|
||||
positional:
|
||||
- name: INPUT
|
||||
type: path
|
||||
help: "输入 PDF 文件"
|
||||
ocr:
|
||||
help: "PDF OCR 识别"
|
||||
positional:
|
||||
- name: INPUT
|
||||
type: path
|
||||
help: "输入 PDF 文件"
|
||||
options:
|
||||
- name: OUTPUT
|
||||
flag: "--output"
|
||||
type: path
|
||||
default: "ocr.pdf"
|
||||
help: "输出文件 (默认: ocr.pdf)"
|
||||
- name: LANG
|
||||
flag: "--lang"
|
||||
type: str
|
||||
default: "chi_sim+eng"
|
||||
help: "识别语言 (默认: chi_sim+eng)"
|
||||
img:
|
||||
help: "PDF 转图片"
|
||||
positional:
|
||||
- name: INPUT
|
||||
type: path
|
||||
help: "输入 PDF 文件"
|
||||
options:
|
||||
- name: OUTPUT_DIR
|
||||
flag: "--output-dir"
|
||||
type: path
|
||||
default: "images"
|
||||
help: "输出目录 (默认: images)"
|
||||
- name: DPI
|
||||
flag: "--dpi"
|
||||
type: int
|
||||
default: 300
|
||||
help: "DPI (默认: 300)"
|
||||
repair:
|
||||
help: "修复 PDF"
|
||||
positional:
|
||||
- name: INPUT
|
||||
type: path
|
||||
help: "输入 PDF 文件"
|
||||
options:
|
||||
- name: OUTPUT
|
||||
flag: "--output"
|
||||
type: path
|
||||
default: "repaired.pdf"
|
||||
help: "输出文件 (默认: repaired.pdf)"
|
||||
jobs:
|
||||
m:
|
||||
fn: pdf_merge
|
||||
args: ["${INPUTS}", "${OUTPUT}"]
|
||||
s:
|
||||
fn: pdf_split
|
||||
args: ["${INPUT}", "${OUTPUT_DIR}"]
|
||||
c:
|
||||
fn: pdf_compress
|
||||
args: ["${INPUT}", "${OUTPUT}"]
|
||||
e:
|
||||
fn: pdf_encrypt
|
||||
args: ["${INPUT}", "${OUTPUT}", "${PASSWORD}"]
|
||||
d:
|
||||
fn: pdf_decrypt
|
||||
args: ["${INPUT}", "${OUTPUT}", "${PASSWORD}"]
|
||||
xt:
|
||||
fn: pdf_extract_text
|
||||
args: ["${INPUT}", "${OUTPUT}"]
|
||||
xi:
|
||||
fn: pdf_extract_images
|
||||
args: ["${INPUT}", "${OUTPUT_DIR}"]
|
||||
w:
|
||||
fn: pdf_add_watermark
|
||||
args: ["${INPUT}", "${OUTPUT}"]
|
||||
kwargs:
|
||||
text: "${TEXT}"
|
||||
r:
|
||||
fn: pdf_rotate
|
||||
args: ["${INPUT}", "${OUTPUT}"]
|
||||
kwargs:
|
||||
rotation: ${ROTATION}
|
||||
crop:
|
||||
fn: pdf_crop
|
||||
args: ["${INPUT}", "${OUTPUT}"]
|
||||
kwargs:
|
||||
margins: "${MARGINS}"
|
||||
i:
|
||||
fn: pdf_info
|
||||
args: ["${INPUT}"]
|
||||
ocr:
|
||||
fn: pdf_ocr
|
||||
args: ["${INPUT}", "${OUTPUT}"]
|
||||
kwargs:
|
||||
lang: "${LANG}"
|
||||
img:
|
||||
fn: pdf_to_images
|
||||
args: ["${INPUT}", "${OUTPUT_DIR}"]
|
||||
kwargs:
|
||||
dpi: ${DPI}
|
||||
repair:
|
||||
fn: pdf_repair
|
||||
args: ["${INPUT}", "${OUTPUT}"]
|
||||
@@ -0,0 +1,78 @@
|
||||
# piptool - pip 包管理工具
|
||||
# 用法:
|
||||
# pf piptool i requests
|
||||
# pf piptool u requests
|
||||
# pf piptool r requests
|
||||
# pf piptool d requests
|
||||
# pf piptool up
|
||||
# pf piptool f
|
||||
strategy: thread
|
||||
variables:
|
||||
PACKAGES: []
|
||||
OFFLINE: false
|
||||
cli:
|
||||
description: "PipTool - pip 包管理工具"
|
||||
usage: "pf piptool <command> [packages...] [options]"
|
||||
subcommands:
|
||||
i:
|
||||
help: "安装包"
|
||||
positional:
|
||||
- name: PACKAGES
|
||||
nargs: "+"
|
||||
type: str
|
||||
help: "包名列表"
|
||||
u:
|
||||
help: "卸载包"
|
||||
positional:
|
||||
- name: PACKAGES
|
||||
nargs: "+"
|
||||
type: str
|
||||
help: "包名列表"
|
||||
r:
|
||||
help: "重装包"
|
||||
positional:
|
||||
- name: PACKAGES
|
||||
nargs: "+"
|
||||
type: str
|
||||
help: "包名列表"
|
||||
options:
|
||||
- name: OFFLINE
|
||||
flag: "--offline"
|
||||
action: "store_true"
|
||||
help: "离线模式"
|
||||
d:
|
||||
help: "下载包"
|
||||
positional:
|
||||
- name: PACKAGES
|
||||
nargs: "+"
|
||||
type: str
|
||||
help: "包名列表"
|
||||
options:
|
||||
- name: OFFLINE
|
||||
flag: "--offline"
|
||||
action: "store_true"
|
||||
help: "离线模式"
|
||||
up:
|
||||
help: "升级 pip"
|
||||
f:
|
||||
help: "导出依赖"
|
||||
jobs:
|
||||
i:
|
||||
cmd: ["pip", "install", "${PACKAGES}"]
|
||||
u:
|
||||
fn: pip_uninstall
|
||||
args: ["${PACKAGES}"]
|
||||
r:
|
||||
fn: pip_reinstall
|
||||
args: ["${PACKAGES}"]
|
||||
kwargs:
|
||||
offline: ${OFFLINE}
|
||||
d:
|
||||
fn: pip_download
|
||||
args: ["${PACKAGES}"]
|
||||
kwargs:
|
||||
offline: ${OFFLINE}
|
||||
up:
|
||||
cmd: ["python", "-m", "pip", "install", "--upgrade", "pip"]
|
||||
f:
|
||||
fn: pip_freeze
|
||||
@@ -0,0 +1,125 @@
|
||||
# pymake - 项目构建工具
|
||||
# 用法
|
||||
# pf pymake <command>
|
||||
# 命令
|
||||
# b: 构建 Python 主包 (uv build)
|
||||
# ba: 构建所有包 (Python + Rust)
|
||||
# bc: 构建 Rust 核心模块 (maturin build)
|
||||
# bump: 升级版本号 (清理 + 检查 + add + bumpversion)
|
||||
# bumpmi: 升级次版本号 (bumpversion minor)
|
||||
# c: 清理构建产物 (调用 gitt c)
|
||||
# cov: 测试并生成覆盖率
|
||||
# doc: 构建 Sphinx 文档
|
||||
# lint: 代码格式化与检查 (ruff)
|
||||
# p: 推送代码 (清理 + push + push tags)
|
||||
# pb: 发布到 PyPI (twine + hatch)
|
||||
# sync: 同步依赖 (uv sync)
|
||||
# t: 运行测试
|
||||
# tc: 类型检查 (pyrefly + ruff)
|
||||
# tf: 快速测试 (无 slow)
|
||||
# tox: 多版本测试 (tox)
|
||||
strategy: thread
|
||||
variables:
|
||||
CWD: "."
|
||||
cli:
|
||||
description: "PyMake - 项目构建工具"
|
||||
usage: "pf pymake <command>"
|
||||
options:
|
||||
- name: CWD
|
||||
flag: "--cwd"
|
||||
type: path
|
||||
required: false
|
||||
default: "."
|
||||
help: "工作目录 (默认: 当前目录)"
|
||||
subcommands:
|
||||
b: {help: "构建 Python 主包 (uv build)"}
|
||||
ba: {help: "构建所有包 (Python + Rust)"}
|
||||
bc: {help: "构建 Rust 核心模块 (maturin build)"}
|
||||
bump: {help: "升级版本号 (清理 + 检查 + add + bumpversion)"}
|
||||
bumpmi: {help: "升级次版本号 (bumpversion minor)"}
|
||||
c: {help: "清理构建产物 (调用 gitt c)"}
|
||||
cov: {help: "测试并生成覆盖率"}
|
||||
doc: {help: "构建 Sphinx 文档"}
|
||||
lint: {help: "代码格式化与检查 (ruff)"}
|
||||
p: {help: "推送代码 (清理 + push + push tags)"}
|
||||
pb: {help: "发布到 PyPI (twine + hatch)"}
|
||||
sync: {help: "同步依赖 (uv sync)"}
|
||||
t: {help: "运行测试"}
|
||||
tc: {help: "类型检查 (pyrefly + ruff)"}
|
||||
tf: {help: "快速测试 (无 slow)"}
|
||||
tox: {help: "多版本测试 (tox)"}
|
||||
jobs:
|
||||
# 单任务别名
|
||||
b:
|
||||
cmd: ["uv", "build"]
|
||||
cwd: ${CWD}
|
||||
bc:
|
||||
cmd: ["maturin", "build", "-r"]
|
||||
cwd: ${CWD}
|
||||
sync:
|
||||
cmd: ["uv", "sync"]
|
||||
cwd: ${CWD}
|
||||
c:
|
||||
cmd: ["pf", "gitt", "c"]
|
||||
cwd: ${CWD}
|
||||
t:
|
||||
cmd: ["pytest", "-m", "not slow", "-n", "8", "--dist", "loadfile", "--color=yes", "--durations=10"]
|
||||
cwd: ${CWD}
|
||||
tf:
|
||||
cmd: ["pytest", "-m", "not slow", "--dist", "loadfile", "--color=yes", "--durations=10"]
|
||||
cwd: ${CWD}
|
||||
bumpversion:
|
||||
cmd: ["bumpversion", "patch"]
|
||||
needs: [git_add_all]
|
||||
cwd: ${CWD}
|
||||
bumpmi:
|
||||
cmd: ["bumpversion", "minor"]
|
||||
cwd: ${CWD}
|
||||
doc:
|
||||
cmd: ["sphinx-build", "-b", "html", "docs", "docs/_build"]
|
||||
cwd: ${CWD}
|
||||
lint:
|
||||
cmd: ["ruff", "check", "--fix", "--unsafe-fixes"]
|
||||
cwd: ${CWD}
|
||||
tox:
|
||||
cmd: ["tox", "-p", "auto"]
|
||||
cwd: ${CWD}
|
||||
|
||||
# 内部 job (不暴露为 subcommand)
|
||||
test_coverage:
|
||||
cmd: ["pytest", "--cov", "-n", "8", "--dist", "loadfile", "--tb=short", "-v", "--color=yes", "--durations=10"]
|
||||
needs: [c]
|
||||
cwd: ${CWD}
|
||||
pyrefly_check:
|
||||
cmd: ["pyrefly", "check", "."]
|
||||
cwd: ${CWD}
|
||||
git_add_all:
|
||||
cmd: ["git", "add", "-A"]
|
||||
needs: [tc]
|
||||
cwd: ${CWD}
|
||||
git_push:
|
||||
cmd: ["git", "push"]
|
||||
cwd: ${CWD}
|
||||
git_push_tags:
|
||||
cmd: ["git", "push", "--tags"]
|
||||
cwd: ${CWD}
|
||||
twine_publish:
|
||||
cmd: ["twine", "upload", "--disable-progress-bar"]
|
||||
cwd: ${CWD}
|
||||
publish_python:
|
||||
cmd: ["hatch", "publish"]
|
||||
cwd: ${CWD}
|
||||
|
||||
# 聚合 job (方向 B: 有 needs 无 cmd/fn)
|
||||
ba:
|
||||
needs: [b, bc]
|
||||
bump:
|
||||
needs: [bumpversion]
|
||||
cov:
|
||||
needs: [test_coverage]
|
||||
tc:
|
||||
needs: [c, pyrefly_check, lint]
|
||||
p:
|
||||
needs: [c, git_push, git_push_tags]
|
||||
pb:
|
||||
needs: [twine_publish, publish_python]
|
||||
@@ -0,0 +1,13 @@
|
||||
# reseticoncache - 重置 Windows 图标缓存
|
||||
# 用法
|
||||
# pf reseticon
|
||||
# 说明
|
||||
# 杀掉 explorer → 删除 IconCache.db → 删除 iconcache* → 重启 explorer
|
||||
# 仅在 Windows 上有效, 非 Windows 平台打印提示并跳过
|
||||
strategy: sequential
|
||||
cli:
|
||||
description: "重置 Windows 图标缓存"
|
||||
usage: "pf reseticon"
|
||||
jobs:
|
||||
reset:
|
||||
fn: reset_icon_cache_run
|
||||
@@ -0,0 +1,34 @@
|
||||
# screenshot - 截图工具
|
||||
# 用法:
|
||||
# pf screenshot full
|
||||
# pf screenshot area --filename custom.png
|
||||
strategy: thread
|
||||
variables:
|
||||
FILENAME: null
|
||||
cli:
|
||||
description: "Screenshot - 截图工具"
|
||||
usage: "pf screenshot <command> [options]"
|
||||
subcommands:
|
||||
full:
|
||||
help: "全屏截图"
|
||||
options:
|
||||
- name: FILENAME
|
||||
flag: "--filename"
|
||||
type: str
|
||||
help: "文件名"
|
||||
area:
|
||||
help: "区域截图"
|
||||
options:
|
||||
- name: FILENAME
|
||||
flag: "--filename"
|
||||
type: str
|
||||
help: "文件名"
|
||||
jobs:
|
||||
full:
|
||||
fn: take_screenshot_full
|
||||
kwargs:
|
||||
filename: "${FILENAME}"
|
||||
area:
|
||||
fn: take_screenshot_area
|
||||
kwargs:
|
||||
filename: "${FILENAME}"
|
||||
@@ -0,0 +1,49 @@
|
||||
# sshcopyid - SSH 密钥部署工具
|
||||
# 用法:
|
||||
# pf sshcopyid hostname username password
|
||||
# pf sshcopyid server user pass --port 2222
|
||||
strategy: thread
|
||||
variables:
|
||||
HOSTNAME: ""
|
||||
USERNAME: ""
|
||||
PASSWORD: ""
|
||||
PORT: 22
|
||||
KEYPATH: "~/.ssh/id_rsa.pub"
|
||||
TIMEOUT: 30
|
||||
cli:
|
||||
description: "SSHCopyID - SSH 密钥部署工具"
|
||||
usage: "pf sshcopyid <hostname> <username> <password> [options]"
|
||||
positional:
|
||||
- name: HOSTNAME
|
||||
type: str
|
||||
help: "远程服务器主机名或 IP 地址"
|
||||
- name: USERNAME
|
||||
type: str
|
||||
help: "远程服务器用户名"
|
||||
- name: PASSWORD
|
||||
type: str
|
||||
help: "远程服务器密码"
|
||||
options:
|
||||
- name: PORT
|
||||
flag: "--port"
|
||||
type: int
|
||||
default: 22
|
||||
help: "SSH 端口 (默认: 22)"
|
||||
- name: KEYPATH
|
||||
flag: "--keypath"
|
||||
type: str
|
||||
default: "~/.ssh/id_rsa.pub"
|
||||
help: "公钥文件路径"
|
||||
- name: TIMEOUT
|
||||
flag: "--timeout"
|
||||
type: int
|
||||
default: 30
|
||||
help: "SSH 操作超时秒数 (默认: 30)"
|
||||
jobs:
|
||||
deploy:
|
||||
fn: ssh_copy_id
|
||||
args: ["${HOSTNAME}", "${USERNAME}", "${PASSWORD}"]
|
||||
kwargs:
|
||||
port: ${PORT}
|
||||
keypath: "${KEYPATH}"
|
||||
timeout: ${TIMEOUT}
|
||||
+35
-60
@@ -1,23 +1,22 @@
|
||||
"""上下文注入:把上游结果转换为函数参数。
|
||||
|
||||
本机制让用户可以编写普通函数,其参数名*就是*依赖声明,从而消除其他
|
||||
DAG 库中泛滥的样板包装器(如 ``def wrapper(): return fn(workflow.get_task_result('x'))``)。
|
||||
DAG 库中泛滥的样板包装器。
|
||||
|
||||
注入规则(按顺序求值)
|
||||
----------------------
|
||||
1. **标注为** :class:`Context` 的参数接收完整结果映射。适用于需要遍历
|
||||
所有输入的任务。
|
||||
2. **名称匹配某个依赖**的参数接收该依赖的结果。
|
||||
1. **标注为** :class:`Context` 的参数接收完整结果映射(含硬依赖与软依赖)。
|
||||
2. **名称匹配某个依赖**(硬或软)的参数接收该依赖的结果。
|
||||
3. ``**kwargs`` 参数以 dict 形式接收*所有*依赖结果。
|
||||
4. ``TaskSpec.args`` / ``TaskSpec.kwargs`` 为*非依赖*参数提供静态值。
|
||||
|
||||
若某参数无法解析且无默认值,则抛出 :class:`~pyflowx.errors.InjectionError`,
|
||||
并附带精确错误信息。
|
||||
若某参数无法解析且无默认值,则抛出 :class:`~pyflowx.errors.InjectionError`。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from functools import lru_cache
|
||||
from typing import Any, Mapping
|
||||
|
||||
from .errors import InjectionError
|
||||
@@ -26,22 +25,30 @@ from .task import Context, TaskSpec
|
||||
__all__ = ["Context", "_is_context_annotation", "build_call_args", "describe_injection"]
|
||||
|
||||
|
||||
def _is_context_annotation(annotation: Any) -> bool:
|
||||
"""判断参数标注是否为(或指向)``Context``。
|
||||
@lru_cache(maxsize=1024)
|
||||
def _cached_signature(fn: Any) -> inspect.Signature:
|
||||
"""缓存 ``inspect.signature`` 结果(按 fn 对象键控)。
|
||||
|
||||
处理三种形式:
|
||||
* ``Context`` 别名对象本身;
|
||||
* ``__name__``/``_name`` 为 ``Context`` 或 ``Mapping`` 的 typing 别名;
|
||||
* *字符串*标注(``from __future__ import annotations`` 会在运行时
|
||||
把所有标注变为字符串),如 ``"Context"`` 或 ``"px.Context"``。
|
||||
``fn`` 对象在 :meth:`TaskSpec.effective_fn` 缓存后稳定,签名重复内省
|
||||
属纯开销。对不可哈希的可调用对象,调用方回退到直接内省。
|
||||
"""
|
||||
return inspect.signature(fn)
|
||||
|
||||
|
||||
def _signature(fn: Any) -> inspect.Signature:
|
||||
"""获取签名,优先走缓存;``fn`` 不可哈希时回退到直接内省。"""
|
||||
try:
|
||||
return _cached_signature(fn)
|
||||
except TypeError:
|
||||
return inspect.signature(fn)
|
||||
|
||||
|
||||
def _is_context_annotation(annotation: Any) -> bool:
|
||||
"""判断参数标注是否为(或指向)``Context``。"""
|
||||
if annotation is Context:
|
||||
return True
|
||||
# `from __future__ import annotations` 产生的字符串标注。
|
||||
if isinstance(annotation, str):
|
||||
# 匹配 "Context"、"px.Context"、"pyflowx.Context" 等。
|
||||
return annotation == "Context" or annotation.endswith(".Context")
|
||||
# 按限定名匹配,支持 ``from pyflowx import Context`` 再导出。
|
||||
name = getattr(annotation, "__name__", None) or getattr(annotation, "_name", None)
|
||||
return name in ("Context", "Mapping")
|
||||
|
||||
@@ -52,39 +59,22 @@ def build_call_args(
|
||||
) -> tuple[tuple[Any, ...], dict[str, Any]]:
|
||||
"""解析用于调用 ``spec.fn`` 的 ``(args, kwargs)``。
|
||||
|
||||
参数
|
||||
----
|
||||
spec:
|
||||
任务 spec,提供 ``fn``、``depends_on``、``args``、``kwargs``。
|
||||
context:
|
||||
依赖名 -> 结果值的映射。仅保证本任务自身的 ``depends_on`` 条目
|
||||
存在;其他任务的结果被排除,以保持注入的确定性。
|
||||
|
||||
返回
|
||||
----
|
||||
(args, kwargs)
|
||||
可直接展开为 ``spec.fn(*args, **kwargs)``。
|
||||
|
||||
抛出
|
||||
----
|
||||
InjectionError
|
||||
若必需参数无法满足,或静态 ``kwargs`` 与注入依赖名冲突。
|
||||
``context`` 必须已包含所有硬依赖与软依赖的结果(软依赖被跳过时由
|
||||
执行器填入 :attr:`TaskSpec.defaults` 中的默认值)。
|
||||
"""
|
||||
# 使用 effective_fn 而不是 fn,以支持 cmd 参数
|
||||
fn = spec.effective_fn
|
||||
sig = inspect.signature(fn)
|
||||
sig = _signature(fn)
|
||||
params = sig.parameters
|
||||
|
||||
# 检测特殊参数类型。
|
||||
var_keyword = next(
|
||||
(p for p in params.values() if p.kind == inspect.Parameter.VAR_KEYWORD),
|
||||
None,
|
||||
)
|
||||
|
||||
# 与本任务相关的上下文子集。
|
||||
dep_context: dict[str, Any] = {name: context[name] for name in spec.depends_on if name in context}
|
||||
# 本任务相关的上下文子集:硬依赖 + 软依赖。
|
||||
all_deps = set(spec.depends_on) | set(spec.soft_depends_on)
|
||||
dep_context: dict[str, Any] = {name: context[name] for name in all_deps if name in context}
|
||||
|
||||
# 检测静态 kwargs 与依赖名的冲突。
|
||||
collisions = set(spec.kwargs) & set(dep_context)
|
||||
if collisions:
|
||||
raise InjectionError(
|
||||
@@ -96,8 +86,6 @@ def build_call_args(
|
||||
injected_kwargs: dict[str, Any] = {}
|
||||
leftover_dep_results: dict[str, Any] = dict(dep_context)
|
||||
|
||||
# 被 spec.args 消费的位置参数。记录哪些参数名已被位置填充,
|
||||
# 以便在基于名称的注入(依赖 / Context / 静态 kwargs)时跳过。
|
||||
positional_params: list[str] = []
|
||||
positional_kinds = (
|
||||
inspect.Parameter.POSITIONAL_ONLY,
|
||||
@@ -106,33 +94,25 @@ def build_call_args(
|
||||
for pname, param in params.items():
|
||||
if param.kind in positional_kinds:
|
||||
positional_params.append(pname)
|
||||
# 前 len(spec.args) 个位置参数由 spec.args 填充。
|
||||
args_filled: set[str] = set(positional_params[: len(spec.args)])
|
||||
|
||||
for pname, param in params.items():
|
||||
# 跳过已被位置 spec.args 填充的参数。
|
||||
if pname in args_filled:
|
||||
continue
|
||||
|
||||
# 规则 1:标注为 Context -> 完整映射。
|
||||
if _is_context_annotation(param.annotation):
|
||||
injected_kwargs[pname] = dep_context
|
||||
continue
|
||||
|
||||
# 规则 2:名称匹配某个依赖。
|
||||
if pname in dep_context:
|
||||
injected_kwargs[pname] = dep_context[pname]
|
||||
leftover_dep_results.pop(pname, None)
|
||||
continue
|
||||
|
||||
# 规则 3:在循环后通过 **kwargs 处理。
|
||||
|
||||
# 规则 4:静态 kwargs 填充其余参数。
|
||||
if pname in spec.kwargs:
|
||||
injected_kwargs[pname] = spec.kwargs[pname]
|
||||
continue
|
||||
|
||||
# 该参数无来源:必须有默认值,否则报错。
|
||||
if param.default is inspect.Parameter.empty and param.kind not in (
|
||||
inspect.Parameter.VAR_POSITIONAL,
|
||||
inspect.Parameter.VAR_KEYWORD,
|
||||
@@ -142,9 +122,7 @@ def build_call_args(
|
||||
f"parameter {pname!r} has no dependency, static value, or default.",
|
||||
)
|
||||
|
||||
# 规则 3:**kwargs 吞掉剩余依赖结果。
|
||||
if var_keyword is not None and leftover_dep_results:
|
||||
# 先合并静态 kwargs,再合并依赖结果(冲突已在上方拒绝)。
|
||||
merged = dict(spec.kwargs)
|
||||
merged.update(injected_kwargs)
|
||||
merged.update(leftover_dep_results)
|
||||
@@ -154,14 +132,9 @@ def build_call_args(
|
||||
|
||||
|
||||
def describe_injection(spec: TaskSpec[Any]) -> str:
|
||||
"""生成任务参数注入方式的人类可读描述。
|
||||
|
||||
供 ``dry_run`` 使用,在不执行的情况下展示执行计划。
|
||||
"""
|
||||
# 使用 effective_fn 而不是 fn,以支持 cmd 参数
|
||||
"""生成任务参数注入方式的人类可读描述。供 ``dry_run`` 使用。"""
|
||||
fn = spec.effective_fn
|
||||
sig = inspect.signature(fn)
|
||||
# 确定哪些位置参数由 spec.args 填充。
|
||||
sig = _signature(fn)
|
||||
positional_params = [
|
||||
p
|
||||
for p, param in sig.parameters.items()
|
||||
@@ -172,6 +145,7 @@ def describe_injection(spec: TaskSpec[Any]) -> str:
|
||||
)
|
||||
]
|
||||
args_filled = set(positional_params[: len(spec.args)])
|
||||
all_deps = set(spec.depends_on) | set(spec.soft_depends_on)
|
||||
parts = []
|
||||
for pname, param in sig.parameters.items():
|
||||
if pname in args_filled:
|
||||
@@ -179,8 +153,9 @@ def describe_injection(spec: TaskSpec[Any]) -> str:
|
||||
parts.append(f"{pname}={spec.args[idx]!r}")
|
||||
elif _is_context_annotation(param.annotation):
|
||||
parts.append(f"{pname}=<Context>")
|
||||
elif pname in spec.depends_on:
|
||||
parts.append(f"{pname}=<result:{pname}>")
|
||||
elif pname in all_deps:
|
||||
tag = "soft" if pname in spec.soft_depends_on else "dep"
|
||||
parts.append(f"{pname}=<{tag}:{pname}>")
|
||||
elif pname in spec.kwargs:
|
||||
parts.append(f"{pname}={spec.kwargs[pname]!r}")
|
||||
elif param.default is not inspect.Parameter.empty:
|
||||
|
||||
@@ -10,19 +10,21 @@ Demonstrates the core PyFlowX workflow:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# --- task functions: pure, testable, no framework coupling ------------- #
|
||||
|
||||
|
||||
def extract_customers() -> list[dict]:
|
||||
def extract_customers() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"id": "C001", "name": "Alice"},
|
||||
{"id": "C002", "name": "Bob"},
|
||||
]
|
||||
|
||||
|
||||
def extract_orders() -> list[dict]:
|
||||
def extract_orders() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"id": "O001", "customer_id": "C001", "amount": 150.0},
|
||||
{"id": "O002", "customer_id": "C002", "amount": 200.5},
|
||||
@@ -31,14 +33,14 @@ def extract_orders() -> list[dict]:
|
||||
|
||||
# Parameter names match dependency names → automatic injection.
|
||||
def transform(
|
||||
extract_customers: list[dict],
|
||||
extract_orders: list[dict],
|
||||
) -> list[dict]:
|
||||
extract_customers: list[dict[str, Any]],
|
||||
extract_orders: list[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
cmap = {c["id"]: c for c in extract_customers}
|
||||
return [{**o, "customer_name": cmap[o["customer_id"]]["name"]} for o in extract_orders if o["customer_id"] in cmap]
|
||||
|
||||
|
||||
def load(transform: list[dict]) -> int:
|
||||
def load(transform: list[dict[str, Any]]) -> int:
|
||||
print(f" loaded {len(transform)} records")
|
||||
return len(transform)
|
||||
|
||||
@@ -54,7 +56,9 @@ def main() -> None:
|
||||
depends_on=("extract_customers", "extract_orders"),
|
||||
tags=("transform",),
|
||||
),
|
||||
px.TaskSpec("load", load, depends_on=("transform",), retries=1, tags=("load",)),
|
||||
px.TaskSpec(
|
||||
"load", load, depends_on=("transform",), retry=px.RetryPolicy(max_attempts=1, delay=1.0), tags=("load",)
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
+673
-350
File diff suppressed because it is too large
Load Diff
+382
-128
@@ -2,31 +2,140 @@
|
||||
|
||||
使用标准库的 :mod:`graphlib`(3.9+)或 :mod:`graphlib_backport`(3.8)
|
||||
进行拓扑排序。图以增量方式构建并即时校验,使配置错误在构建时(而非执行时)快速失败。
|
||||
|
||||
支持:
|
||||
* 图级默认值 :class:`GraphDefaults`,TaskSpec 字段为 ``None`` 时回退。
|
||||
* :meth:`Graph.map` 工厂批量生成 fan-out 任务。
|
||||
* 字符串引用与 :func:`compose` 编程式组合多个图。
|
||||
* 软依赖:仅用于上下文注入,不参与拓扑分层。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = [
|
||||
"Graph",
|
||||
"GraphDefaults",
|
||||
]
|
||||
|
||||
import inspect
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, Mapping, Sequence
|
||||
|
||||
from .errors import CycleError, DuplicateTaskError, MissingDependencyError
|
||||
from .task import TaskSpec
|
||||
from .task import Context, RetryPolicy, TaskSpec
|
||||
|
||||
# graphlib 自 3.9 起进入标准库;3.8 回退到 backport。
|
||||
if sys.version_info >= (3, 9): # pragma: no cover
|
||||
import graphlib # pyright: ignore[reportUnreachable]
|
||||
|
||||
_TopologicalSorter = graphlib.TopologicalSorter
|
||||
else: # pragma: no cover
|
||||
import graphlib # type: ignore[import-untyped] # pragma: no cover
|
||||
import graphlib # type: ignore[import-untyped]
|
||||
|
||||
_TopologicalSorter = graphlib.TopologicalSorter # pragma: no cover
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@dataclass
|
||||
class GraphDefaults:
|
||||
"""图级默认值。TaskSpec 对应字段为 ``None`` 时回退到此处。
|
||||
|
||||
仅对可空字段生效(retry/timeout/strategy/env/cwd/tags/priority/
|
||||
continue_on_error/concurrency_key)。非空字段(name/fn/cmd)不回退。
|
||||
"""
|
||||
|
||||
retry: RetryPolicy | None = None
|
||||
timeout: float | None = None
|
||||
strategy: str | None = None
|
||||
tags: tuple[str, ...] = ()
|
||||
env: Mapping[str, str] | None = None
|
||||
cwd: Any = None # Path | None
|
||||
priority: int = 0
|
||||
continue_on_error: bool = False
|
||||
concurrency_key: str | None = None
|
||||
verbose: bool = False
|
||||
|
||||
|
||||
def _prune_deps(spec: TaskSpec[Any], keep: Callable[[str], bool]) -> TaskSpec[Any]:
|
||||
"""返回新 spec,其 ``depends_on`` / ``soft_depends_on`` 仅保留 ``keep(dep)`` 为真的依赖。"""
|
||||
return replace(
|
||||
spec,
|
||||
depends_on=tuple(d for d in spec.depends_on if keep(d)),
|
||||
soft_depends_on=tuple(d for d in spec.soft_depends_on if keep(d)),
|
||||
)
|
||||
|
||||
|
||||
def _make_namespaced_fn(orig_fn: Any, ns: str, dep_names: set[str]) -> Any:
|
||||
"""包装 fn,使其能接收带 ``ns:`` 前缀的依赖名,调用时映射回原参数名。
|
||||
|
||||
命名空间合并后,依赖名带前缀(如 ``build:extract``),但 Python 参数名
|
||||
不能含 ``:``。wrapper 用 ``**kwargs`` 接收所有依赖,内部把带前缀的依赖名
|
||||
映射回原参数名后调用原 fn。
|
||||
|
||||
无依赖参数时直接返回原 fn。
|
||||
"""
|
||||
if not dep_names or orig_fn is None:
|
||||
return orig_fn
|
||||
try:
|
||||
orig_sig = inspect.signature(orig_fn)
|
||||
except (TypeError, ValueError):
|
||||
return orig_fn
|
||||
|
||||
# 带前缀依赖名 -> 原参数名
|
||||
name_map: dict[str, str] = {f"{ns}:{orig}": orig for orig in dep_names}
|
||||
prefix = f"{ns}:"
|
||||
|
||||
# 检查原 fn 是否有 Context 标注参数
|
||||
context_param_name: str | None = None
|
||||
for p in orig_sig.parameters.values():
|
||||
ann = p.annotation
|
||||
if ann is not Context and not (isinstance(ann, str) and ann.endswith("Context")):
|
||||
continue
|
||||
context_param_name = p.name
|
||||
break
|
||||
|
||||
if context_param_name is not None:
|
||||
|
||||
def wrapper(ctx: Any = None, **kwargs: Any) -> Any:
|
||||
# ctx 是 dep_context,键为带前缀的依赖名;映射回原始键
|
||||
orig_ctx: dict[str, Any] = {}
|
||||
for k, v in (ctx or {}).items():
|
||||
orig_ctx[name_map.get(k, k)] = v
|
||||
# kwargs 中带前缀的依赖也映射回原参数名
|
||||
for k, v in kwargs.items():
|
||||
if k in name_map:
|
||||
orig_ctx[name_map[k]] = v
|
||||
return orig_fn(**{context_param_name: orig_ctx})
|
||||
|
||||
ctx_param = inspect.Parameter("ctx", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=Context)
|
||||
kw_param = inspect.Parameter("kwargs", inspect.Parameter.VAR_KEYWORD)
|
||||
wrapper.__signature__ = inspect.Signature( # type: ignore[attr-defined]
|
||||
parameters=[ctx_param, kw_param],
|
||||
return_annotation=orig_sig.return_annotation,
|
||||
)
|
||||
else:
|
||||
|
||||
def wrapper(**kwargs: Any) -> Any: # type: ignore[no-redef]
|
||||
orig_kwargs: dict[str, Any] = {}
|
||||
for k, v in kwargs.items():
|
||||
if k.startswith(prefix):
|
||||
orig_kwargs[k[len(prefix) :]] = v
|
||||
return orig_fn(**orig_kwargs)
|
||||
|
||||
kw_param = inspect.Parameter("kwargs", inspect.Parameter.VAR_KEYWORD)
|
||||
wrapper.__signature__ = inspect.Signature( # type: ignore[attr-defined]
|
||||
parameters=[kw_param],
|
||||
return_annotation=orig_sig.return_annotation,
|
||||
)
|
||||
|
||||
wrapper.__name__ = f"{ns}_{getattr(orig_fn, '__name__', 'fn')}"
|
||||
wrapper.__doc__ = getattr(orig_fn, "__doc__", None)
|
||||
return wrapper
|
||||
|
||||
|
||||
@dataclass
|
||||
class Graph:
|
||||
"""校验后不可变的有向无环任务图。
|
||||
"""校验后的有向无环任务图。
|
||||
|
||||
通过添加 :class:`~pyflowx.task.TaskSpec` 实例构建。每次 ``add`` 都
|
||||
执行即时校验(重名、缺失依赖),:meth:`validate` / :meth:`layers`
|
||||
@@ -38,106 +147,186 @@ class Graph:
|
||||
|
||||
specs: dict[str, TaskSpec[Any]] = field(default_factory=dict)
|
||||
deps: dict[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
defaults: GraphDefaults = field(default_factory=GraphDefaults)
|
||||
namespace: str | None = None
|
||||
|
||||
# 待解析的字符串引用列表(由 GraphComposer 消费);为空表示无引用。
|
||||
_pending_refs: list[str] = field(default_factory=list)
|
||||
|
||||
# resolved_spec 缓存:避免执行期每个任务多次重复 dataclasses.replace 判断。
|
||||
# 在 specs / defaults 变更时失效。
|
||||
_resolved_cache: dict[str, TaskSpec[Any]] = field(default_factory=dict)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 构建
|
||||
# ------------------------------------------------------------------ #
|
||||
def add(self, spec: TaskSpec[Any]) -> Graph:
|
||||
"""注册一个任务 spec,并即时校验。
|
||||
|
||||
返回 ``self`` 以支持链式调用,但推荐入口是 :meth:`from_specs`,
|
||||
它会整批校验(允许单次调用中的前向引用)。
|
||||
"""
|
||||
if spec.name in self.specs:
|
||||
raise DuplicateTaskError(spec.name)
|
||||
self.specs[spec.name] = spec
|
||||
self.deps[spec.name] = spec.depends_on
|
||||
# 为增量 API 即时检查重名与缺失依赖。
|
||||
"""注册一个任务 spec,并即时校验。返回 ``self`` 支持链式调用。"""
|
||||
self._register(spec)
|
||||
self._validate_references()
|
||||
return self
|
||||
|
||||
@classmethod
|
||||
def from_specs(cls, specs: Iterable[TaskSpec[Any] | str]) -> Graph:
|
||||
"""从可迭代的 task spec 构建图.
|
||||
def chain(self, *specs: TaskSpec[Any]) -> Graph:
|
||||
"""链式注册任务:每个 spec 自动依赖前一个。
|
||||
|
||||
先收集所有 spec,再统一校验。这意味着任务可以引用*后出现*的
|
||||
依赖——顺序无关,就像声明式配置文件的读取方式。
|
||||
|
||||
支持字符串引用,允许引用其他命令图中的任务。
|
||||
字符串引用将在CliRunner中解析展开。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
specs : Iterable[TaskSpec[Any] | str]
|
||||
TaskSpec对象或字符串引用的列表
|
||||
|
||||
Returns
|
||||
-------
|
||||
Graph
|
||||
构建完成的图
|
||||
|
||||
Note
|
||||
-----
|
||||
字符串引用格式:
|
||||
- "command_name" - 引用整个命令图
|
||||
- "command_name.task_name" - 引用特定任务
|
||||
``chain(a, b, c)`` 等价于 ``b`` 依赖 ``a``,``c`` 依赖 ``b``。
|
||||
若 spec 已带 ``depends_on``,则前驱名追加到现有依赖前。
|
||||
返回 ``self`` 支持链式调用。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> graph = Graph.from_specs([
|
||||
... TaskSpec("build", cmd=["uv", "build"]),
|
||||
... "test", # 引用test命令图
|
||||
... ])
|
||||
>>> graph = px.Graph().chain(extract, transform, load)
|
||||
"""
|
||||
graph = cls()
|
||||
pending_refs: list[str] = []
|
||||
prev_name: str | None = None
|
||||
for s in specs:
|
||||
current = s
|
||||
if prev_name is not None:
|
||||
# 将前驱追加到 depends_on 最前(保持显式依赖优先)
|
||||
new_deps = (prev_name, *s.depends_on) if prev_name not in s.depends_on else s.depends_on
|
||||
current = replace(s, depends_on=new_deps)
|
||||
self.add(current)
|
||||
prev_name = current.name
|
||||
return self
|
||||
|
||||
def _register(self, spec: TaskSpec[Any]) -> None:
|
||||
if spec.name in self.specs:
|
||||
raise DuplicateTaskError(spec.name)
|
||||
self.specs[spec.name] = spec
|
||||
# 拓扑依赖仅含硬依赖;软依赖仅用于注入,不影响分层。
|
||||
self.deps[spec.name] = spec.depends_on
|
||||
self._resolved_cache.clear()
|
||||
|
||||
@classmethod
|
||||
def from_specs(
|
||||
cls,
|
||||
specs: Iterable[TaskSpec[Any] | str],
|
||||
defaults: GraphDefaults | None = None,
|
||||
*,
|
||||
namespace: str | None = None,
|
||||
) -> Graph:
|
||||
"""从可迭代的 task spec 构建图。
|
||||
|
||||
先收集所有 spec,再统一校验。允许前向引用。支持字符串引用,
|
||||
由 :func:`compose` 或 :class:`GraphComposer` 解析展开。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
specs:
|
||||
TaskSpec 对象或字符串引用的列表。
|
||||
defaults:
|
||||
图级默认值。``None`` 使用空 :class:`GraphDefaults`。
|
||||
namespace:
|
||||
可选命名空间,用于 :meth:`add_subgraph` 合并时加前缀。
|
||||
"""
|
||||
graph = cls(defaults=defaults or GraphDefaults(), namespace=namespace)
|
||||
pending_refs: list[str] = []
|
||||
for spec in specs:
|
||||
if isinstance(spec, str):
|
||||
# 字符串引用,稍后解析
|
||||
pending_refs.append(spec)
|
||||
elif isinstance(spec, TaskSpec):
|
||||
if spec.name in graph.specs:
|
||||
raise DuplicateTaskError(spec.name)
|
||||
graph.specs[spec.name] = spec
|
||||
graph.deps[spec.name] = spec.depends_on
|
||||
graph._register(spec)
|
||||
else:
|
||||
raise TypeError(f"from_specs只接受TaskSpec或str,收到: {type(spec)}")
|
||||
raise TypeError(f"from_specs 只接受 TaskSpec 或 str,收到: {type(spec)}")
|
||||
|
||||
# 存储待解析的引用
|
||||
if pending_refs:
|
||||
# 使用特殊属性存储引用,稍后在CliRunner中解析
|
||||
# 由于Graph是frozen dataclass,我们需要特殊处理
|
||||
object.__setattr__(graph, "_pending_refs", pending_refs)
|
||||
graph._pending_refs = pending_refs
|
||||
|
||||
graph._validate_references()
|
||||
graph.validate()
|
||||
return graph
|
||||
|
||||
@classmethod
|
||||
def from_yaml(
|
||||
cls,
|
||||
path: str | Path,
|
||||
variables: Mapping[str, Any] | None = None,
|
||||
) -> Graph:
|
||||
"""从 YAML 文件构建任务图。
|
||||
|
||||
参考 GitHub Actions 风格 schema, 支持 jobs/needs/strategy.matrix/if
|
||||
等 CI/CD 概念。详见 :mod:`pyflowx.yaml_loader`。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
YAML 文件路径
|
||||
variables : Mapping[str, Any] | None
|
||||
运行时变量, 用于替换 ``${VAR}`` 占位符
|
||||
|
||||
Returns
|
||||
-------
|
||||
Graph
|
||||
构建好的任务图
|
||||
|
||||
Raises
|
||||
------
|
||||
YamlLoadError
|
||||
文件不存在、YAML 格式错误、schema 校验失败、循环依赖等
|
||||
"""
|
||||
from .yaml_loader import load_yaml
|
||||
|
||||
return load_yaml(path, variables=variables)
|
||||
|
||||
def add_subgraph(self, sub: Graph, *, namespace: str | None = None) -> Graph:
|
||||
"""将子图合并到当前图,任务名加命名空间前缀避免冲突。
|
||||
|
||||
参数
|
||||
----
|
||||
sub:
|
||||
待合并的子图。
|
||||
namespace:
|
||||
命名空间前缀。``None`` 时使用 ``sub.namespace``,若子图也无命名空间
|
||||
则抛出 ``ValueError``。最终任务名为 ``f"{ns}:{original_name}"``。
|
||||
|
||||
合并后,子图内任务的依赖名也会被加前缀;与子图外部任务的依赖保持原样。
|
||||
|
||||
返回 ``self`` 支持链式调用。
|
||||
"""
|
||||
ns = namespace or sub.namespace
|
||||
if not ns:
|
||||
raise ValueError("add_subgraph 需要 namespace 或子图自带 namespace")
|
||||
|
||||
def _rename(name: str) -> str:
|
||||
# 仅对子图内部任务名加前缀;外部依赖保持原样
|
||||
return f"{ns}:{name}" if name in sub.specs else name
|
||||
|
||||
sub_names = set(sub.specs.keys())
|
||||
for spec in sub.specs.values():
|
||||
# 子图内部依赖名需加前缀,对应的 fn 参数也需包装
|
||||
internal_deps = (set(spec.depends_on) | set(spec.soft_depends_on)) & sub_names
|
||||
new_fn = _make_namespaced_fn(spec.fn, ns, internal_deps) if spec.fn else spec.fn
|
||||
new_spec = replace(
|
||||
spec,
|
||||
name=_rename(spec.name),
|
||||
fn=new_fn,
|
||||
depends_on=tuple(_rename(d) for d in spec.depends_on),
|
||||
soft_depends_on=tuple(_rename(d) for d in spec.soft_depends_on),
|
||||
)
|
||||
self._register(new_spec)
|
||||
self._validate_references()
|
||||
self.validate()
|
||||
return self
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 校验
|
||||
# ------------------------------------------------------------------ #
|
||||
def _validate_references(self) -> None:
|
||||
"""确保每个依赖名都存在于图中。"""
|
||||
for name, deps in self.deps.items():
|
||||
for dep in deps:
|
||||
"""确保每个依赖名都存在于图中。硬依赖与软依赖都校验。"""
|
||||
for name, spec in self.specs.items():
|
||||
for dep in spec.depends_on:
|
||||
if dep not in self.specs:
|
||||
raise MissingDependencyError(name, dep)
|
||||
for dep in spec.soft_depends_on:
|
||||
if dep not in self.specs:
|
||||
raise MissingDependencyError(name, dep)
|
||||
|
||||
def validate(self) -> None:
|
||||
"""执行完整 DAG 校验。
|
||||
|
||||
存在环时抛出 :class:`~pyflowx.errors.CycleError`。
|
||||
依赖存在性由 :meth:`_validate_references` 检查。
|
||||
"""
|
||||
"""执行完整 DAG 校验。存在环时抛出 :class:`CycleError`。"""
|
||||
self._validate_references()
|
||||
sorter = _TopologicalSorter(self.deps)
|
||||
try:
|
||||
# prepare() 在有环时抛出 CycleError;此处不需要
|
||||
# static_order() 的结果,仅利用其校验副作用。
|
||||
sorter.prepare()
|
||||
except graphlib.CycleError as exc:
|
||||
# exc.args[1] 是构成环的节点列表。
|
||||
except graphlib.CycleError as exc: # type: ignore[name-defined]
|
||||
cycle: Sequence[str] = exc.args[1] if len(exc.args) > 1 else []
|
||||
raise CycleError(list(cycle)) from exc
|
||||
|
||||
@@ -153,10 +342,54 @@ class Graph:
|
||||
"""返回 ``name`` 的 spec;不存在则 ``KeyError``。"""
|
||||
return self.specs[name]
|
||||
|
||||
def resolved_spec(self, name: str) -> TaskSpec[Any]:
|
||||
"""返回应用图级默认值后的 spec(不修改原图)。
|
||||
|
||||
对于 ``retry``/``timeout``/``strategy``/``env``/``cwd`` 等可空
|
||||
字段,若 spec 字段为默认空值且图级默认值非空,则用
|
||||
:func:`dataclasses.replace` 生成带默认值的副本。
|
||||
|
||||
结果按 ``name`` 缓存;specs / defaults 变更时缓存失效。
|
||||
"""
|
||||
cached = self._resolved_cache.get(name)
|
||||
if cached is not None:
|
||||
return cached
|
||||
spec = self.specs[name]
|
||||
d = self.defaults
|
||||
overrides: dict[str, Any] = {}
|
||||
if spec.retry == RetryPolicy() and d.retry is not None:
|
||||
overrides["retry"] = d.retry
|
||||
if spec.timeout is None and d.timeout is not None:
|
||||
overrides["timeout"] = d.timeout
|
||||
if spec.strategy is None and d.strategy is not None:
|
||||
overrides["strategy"] = d.strategy
|
||||
if spec.env is None and d.env is not None:
|
||||
overrides["env"] = d.env
|
||||
if spec.cwd is None and d.cwd is not None:
|
||||
overrides["cwd"] = d.cwd
|
||||
if spec.priority == 0 and d.priority != 0:
|
||||
overrides["priority"] = d.priority
|
||||
if not spec.continue_on_error and d.continue_on_error:
|
||||
overrides["continue_on_error"] = True
|
||||
if spec.concurrency_key is None and d.concurrency_key is not None:
|
||||
overrides["concurrency_key"] = d.concurrency_key
|
||||
if not spec.verbose and d.verbose:
|
||||
overrides["verbose"] = True
|
||||
if not spec.tags and d.tags:
|
||||
overrides["tags"] = d.tags
|
||||
resolved = spec if not overrides else replace(spec, **overrides)
|
||||
self._resolved_cache[name] = resolved
|
||||
return resolved
|
||||
|
||||
def dependencies(self, name: str) -> tuple[str, ...]:
|
||||
"""``name`` 的直接前驱。"""
|
||||
"""``name`` 的直接硬依赖前驱。"""
|
||||
return self.deps[name]
|
||||
|
||||
def all_deps(self, name: str) -> tuple[str, ...]:
|
||||
"""``name`` 的硬依赖 + 软依赖。"""
|
||||
spec = self.specs[name]
|
||||
return tuple(spec.depends_on) + tuple(spec.soft_depends_on)
|
||||
|
||||
def all_specs(self) -> Mapping[str, TaskSpec[Any]]:
|
||||
"""name -> spec 的只读视图。"""
|
||||
return self.specs
|
||||
@@ -164,18 +397,18 @@ class Graph:
|
||||
def layers(self) -> list[list[str]]:
|
||||
"""将任务分组为可并行执行的层(Kahn 算法)。
|
||||
|
||||
同层任务无相互依赖,可并发执行。层按执行顺序返回。
|
||||
同层任务无相互硬依赖,可并发执行。软依赖不参与分层。
|
||||
层按执行顺序返回。图有环时抛出 :class:`CycleError`。
|
||||
|
||||
图有环时抛出 :class:`~pyflowx.errors.CycleError`。
|
||||
.. note::
|
||||
本方法假定图已通过 :meth:`validate` 校验(由 :func:`pyflowx.run`
|
||||
在入口统一执行一次)。若直接调用本方法,需自行先校验。
|
||||
"""
|
||||
self.validate()
|
||||
sorter = _TopologicalSorter(self.deps)
|
||||
result: list[list[str]] = []
|
||||
# ``get_ready`` + ``done`` 每次给出一层,正好是并行执行所需的分组。
|
||||
sorter.prepare()
|
||||
while sorter.is_active():
|
||||
ready = list(sorter.get_ready())
|
||||
# 排序以保证确定性、可复现的执行计划。
|
||||
ready.sort()
|
||||
result.append(ready)
|
||||
for node in ready:
|
||||
@@ -186,35 +419,16 @@ class Graph:
|
||||
# 子图 / 标签过滤
|
||||
# ------------------------------------------------------------------ #
|
||||
def subgraph(self, tags: Iterable[str]) -> Graph:
|
||||
"""返回仅包含匹配任意标签的任务的新图。
|
||||
|
||||
依赖会被修剪,仅保留被保留任务之间的边;指向被丢弃任务的边
|
||||
会被移除(被保留的任务不再等待它们)。用于调试时运行大型
|
||||
DAG 的切片。
|
||||
"""
|
||||
"""返回仅包含匹配任意标签的任务的新图。依赖边被修剪。"""
|
||||
wanted: set[str] = set(tags)
|
||||
kept: list[TaskSpec[Any]] = []
|
||||
for spec in self.specs.values():
|
||||
if wanted & set(spec.tags):
|
||||
pruned_deps = tuple(
|
||||
d for d in spec.depends_on if d in self.specs and (wanted & set(self.specs[d].tags))
|
||||
)
|
||||
kept.append(
|
||||
TaskSpec[Any](
|
||||
name=spec.name,
|
||||
fn=spec.fn,
|
||||
cmd=spec.cmd,
|
||||
depends_on=pruned_deps,
|
||||
args=spec.args,
|
||||
kwargs=spec.kwargs,
|
||||
retries=spec.retries,
|
||||
timeout=spec.timeout,
|
||||
tags=spec.tags,
|
||||
conditions=spec.conditions,
|
||||
cwd=spec.cwd,
|
||||
)
|
||||
)
|
||||
return Graph.from_specs(kept)
|
||||
|
||||
def _dep_kept(dep: str) -> bool:
|
||||
return dep in self.specs and bool(wanted & set(self.specs[dep].tags))
|
||||
|
||||
kept: list[TaskSpec[Any]] = [
|
||||
_prune_deps(spec, _dep_kept) for spec in self.specs.values() if wanted & set(spec.tags)
|
||||
]
|
||||
return Graph.from_specs(kept, defaults=self.defaults)
|
||||
|
||||
def subgraph_by_names(self, names: Iterable[str]) -> Graph:
|
||||
"""返回限定于 ``names`` 的新图(边已修剪)。"""
|
||||
@@ -222,36 +436,72 @@ class Graph:
|
||||
for n in wanted:
|
||||
if n not in self.specs:
|
||||
raise KeyError(f"Unknown task name: {n!r}")
|
||||
kept: list[TaskSpec[Any]] = []
|
||||
for spec in self.specs.values():
|
||||
if spec.name in wanted:
|
||||
pruned_deps = tuple(d for d in spec.depends_on if d in wanted)
|
||||
kept.append(
|
||||
TaskSpec[Any](
|
||||
name=spec.name,
|
||||
fn=spec.fn,
|
||||
cmd=spec.cmd,
|
||||
depends_on=pruned_deps,
|
||||
args=spec.args,
|
||||
kwargs=spec.kwargs,
|
||||
retries=spec.retries,
|
||||
timeout=spec.timeout,
|
||||
tags=spec.tags,
|
||||
conditions=spec.conditions,
|
||||
cwd=spec.cwd,
|
||||
)
|
||||
)
|
||||
return Graph.from_specs(kept)
|
||||
kept: list[TaskSpec[Any]] = [
|
||||
_prune_deps(spec, lambda d: d in wanted) for spec in self.specs.values() if spec.name in wanted
|
||||
]
|
||||
return Graph.from_specs(kept, defaults=self.defaults)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Fan-out / map-reduce
|
||||
# ------------------------------------------------------------------ #
|
||||
def map(
|
||||
self,
|
||||
name_fn: Callable[[int], str],
|
||||
spec: TaskSpec[Any],
|
||||
items: Sequence[Any],
|
||||
arg_factory: Callable[[Any], tuple[Any, ...]] | None = None,
|
||||
depends_on_per: Callable[[int], tuple[str, ...]] | None = None,
|
||||
) -> list[TaskSpec[Any]]:
|
||||
"""为 ``items`` 中每个元素生成一个 TaskSpec 并加入图。
|
||||
|
||||
用于 fan-out / map-reduce 模式。返回生成的 spec 列表,便于
|
||||
后续 reduce 任务依赖。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name_fn:
|
||||
接受索引 ``i``,返回任务名。需保证唯一。
|
||||
spec:
|
||||
模板 spec。其 ``name`` 与 ``args`` 会被覆盖。
|
||||
items:
|
||||
待分发的数据序列。
|
||||
arg_factory:
|
||||
接受一个 item,返回位置参数元组,覆盖 spec.args。
|
||||
``None`` 则将单个 item 作为唯一位置参数。
|
||||
depends_on_per:
|
||||
接受索引 ``i``,返回该任务的额外硬依赖。``None`` 则继承 spec.depends_on。
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[TaskSpec]
|
||||
生成的 spec 列表(已加入图)。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> fetch_tmpl = px.TaskSpec("", fn=fetch_user)
|
||||
>>> specs = graph.map(lambda i: f"fetch_{i}", fetch_tmpl, [1, 2, 3])
|
||||
>>> reduce_spec = px.TaskSpec("reduce", fn=reduce_fn, depends_on=tuple(s.name for s in specs))
|
||||
"""
|
||||
generated: list[TaskSpec[Any]] = []
|
||||
for i, item in enumerate(items):
|
||||
name = name_fn(i)
|
||||
args = arg_factory(item) if arg_factory is not None else (item,)
|
||||
extra_deps = depends_on_per(i) if depends_on_per is not None else ()
|
||||
new_spec = replace(
|
||||
spec,
|
||||
name=name,
|
||||
args=tuple(args),
|
||||
depends_on=tuple(spec.depends_on) + tuple(extra_deps),
|
||||
)
|
||||
self.add(new_spec)
|
||||
generated.append(new_spec)
|
||||
return generated
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 可视化
|
||||
# ------------------------------------------------------------------ #
|
||||
def to_mermaid(self, orientation: str = "TD") -> str:
|
||||
"""将 DAG 渲染为 Mermaid ``graph`` 定义字符串。
|
||||
|
||||
无外部依赖;输出可粘贴到 Markdown、由 VS Code 的 Mermaid 预览
|
||||
渲染,或保存为文件。
|
||||
"""
|
||||
"""将 DAG 渲染为 Mermaid ``graph`` 定义字符串。"""
|
||||
valid = {"TD", "TB", "BT", "LR", "RL"}
|
||||
orientation = orientation.upper()
|
||||
if orientation not in valid:
|
||||
@@ -262,6 +512,10 @@ class Graph:
|
||||
for name, deps in self.deps.items():
|
||||
for dep in deps:
|
||||
lines.append(f" {dep} --> {name}")
|
||||
# 软依赖用虚线
|
||||
for name, spec in self.specs.items():
|
||||
for dep in spec.soft_depends_on:
|
||||
lines.append(f" {dep} -.-> {name}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
"""工具函数模块.
|
||||
|
||||
按类别组织 CLI 工具中可复用的函数, 每个子模块使用 ``@px.register_fn`` 注册函数,
|
||||
供 YAML 任务编排通过 ``fn`` 字段引用.
|
||||
|
||||
子模块
|
||||
------
|
||||
- :mod:`files` —— 文件日期/等级/备份/压缩相关函数
|
||||
- :mod:`dev` —— 开发工具 (ruff/版本号/pip/git) 相关函数
|
||||
- :mod:`media` —— PDF/截图相关函数
|
||||
- :mod:`system` —— LS-DYNA/SSH/打包相关函数
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from . import dev, files, media, system
|
||||
|
||||
__all__ = ["dev", "files", "media", "system"]
|
||||
@@ -0,0 +1,618 @@
|
||||
"""开发工具类函数模块.
|
||||
|
||||
聚合自动格式化 (autofmt)、版本号管理 (bumpversion)、pip 包管理 (piptool)、
|
||||
git 工具 (gittool) 的可复用函数. 所有公共函数通过 ``@px.register_fn`` 注册,
|
||||
供 YAML 任务编排引用.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import fnmatch
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"IGNORE_PATTERNS",
|
||||
"PACKAGE_DIR",
|
||||
"REQUIREMENTS_FILE",
|
||||
"_PROTECTED_PACKAGES",
|
||||
"BumpVersionType",
|
||||
"add_docstring",
|
||||
"auto_add_docstrings",
|
||||
"bump_file_version",
|
||||
"bump_project_version",
|
||||
"format_all",
|
||||
"format_with_ruff",
|
||||
"generate_module_docstring",
|
||||
"git_add_commit",
|
||||
"git_init_add_commit",
|
||||
"has_files",
|
||||
"init_sub_dirs",
|
||||
"lint_with_ruff",
|
||||
"not_has_git_repo",
|
||||
"pip_download",
|
||||
"pip_freeze",
|
||||
"pip_reinstall",
|
||||
"pip_uninstall",
|
||||
"sync_pyproject_config",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# autofmt 配置
|
||||
# ============================================================================
|
||||
|
||||
IGNORE_PATTERNS = [
|
||||
"__pycache__",
|
||||
"*.pyc",
|
||||
"*.pyo",
|
||||
".git",
|
||||
".venv",
|
||||
".idea",
|
||||
".vscode",
|
||||
"*.egg-info",
|
||||
"dist",
|
||||
"build",
|
||||
".pytest_cache",
|
||||
".tox",
|
||||
".mypy_cache",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# bumpversion 配置
|
||||
# ============================================================================
|
||||
|
||||
BumpVersionType = Literal["patch", "minor", "major"]
|
||||
|
||||
_PYPROJECT_VERSION_PATTERN = re.compile(
|
||||
r'(?:^|\n)\s*version\s*=\s*["\']'
|
||||
r"(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
|
||||
r"(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
|
||||
r"(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?"
|
||||
r'["\']',
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
_INIT_VERSION_PATTERN = re.compile(
|
||||
r'(?:^|\n)\s*__version__\s*=\s*["\']'
|
||||
r"(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
|
||||
r"(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
|
||||
r"(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?"
|
||||
r'["\']',
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
# ============================================================================
|
||||
# piptool 配置
|
||||
# ============================================================================
|
||||
|
||||
PACKAGE_DIR = "packages"
|
||||
REQUIREMENTS_FILE = "requirements.txt"
|
||||
|
||||
_PROTECTED_PACKAGES: frozenset[str] = frozenset(
|
||||
{
|
||||
"pyflowx",
|
||||
"bitool",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# autofmt 私有辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# autofmt 函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def format_with_ruff(target: Path, fix: bool = True) -> None:
|
||||
"""使用 ruff 格式化代码.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
target : Path
|
||||
目标路径
|
||||
fix : bool
|
||||
是否自动修复
|
||||
"""
|
||||
cmd = ["ruff", "format", str(target)]
|
||||
if fix:
|
||||
cmd.append("--fix")
|
||||
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"ruff format 完成: {target}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def lint_with_ruff(target: Path, fix: bool = True) -> None:
|
||||
"""使用 ruff 检查代码.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
target : Path
|
||||
目标路径
|
||||
fix : bool
|
||||
是否自动修复
|
||||
"""
|
||||
cmd = ["ruff", "check", str(target)]
|
||||
if fix:
|
||||
cmd.extend(["--fix", "--unsafe-fixes"])
|
||||
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"ruff check 完成: {target}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def add_docstring(file_path: Path, docstring: str) -> bool:
|
||||
"""为文件添加 docstring.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_path : Path
|
||||
文件路径
|
||||
docstring : str
|
||||
docstring 内容
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
是否成功添加
|
||||
"""
|
||||
try:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
tree = ast.parse(content)
|
||||
|
||||
first_node = tree.body[0] if tree.body else None
|
||||
if first_node and isinstance(first_node, ast.Expr) and isinstance(first_node.value, ast.Constant):
|
||||
return False
|
||||
|
||||
lines = content.splitlines()
|
||||
doc_lines = docstring.splitlines()
|
||||
doc_lines.append("")
|
||||
new_content = "\n".join(doc_lines + lines)
|
||||
|
||||
file_path.write_text(new_content, encoding="utf-8")
|
||||
print(f"添加 docstring: {file_path}")
|
||||
return True
|
||||
|
||||
except (OSError, UnicodeDecodeError, SyntaxError) as e:
|
||||
print(f"处理失败: {file_path} - {e}")
|
||||
return False
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def generate_module_docstring(file_path: Path) -> str:
|
||||
"""生成模块 docstring.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_path : Path
|
||||
文件路径
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
生成的 docstring
|
||||
"""
|
||||
stem = file_path.stem
|
||||
parent = file_path.parent.name
|
||||
|
||||
keywords = {
|
||||
"cli": f"Command-line interface for {parent}",
|
||||
"gui": f"Graphical user interface for {parent}",
|
||||
"core": f"Core functionality for {parent}",
|
||||
"util": f"Utility functions for {parent}",
|
||||
"model": f"Data models for {parent}",
|
||||
"test": f"Tests for {parent}",
|
||||
}
|
||||
|
||||
for key, desc in keywords.items():
|
||||
if key in stem.lower():
|
||||
return f'"""{desc}."""'
|
||||
|
||||
return f'"""{stem.replace("_", " ").title()} module."""'
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def auto_add_docstrings(root_dir: Path) -> int:
|
||||
"""自动为所有 Python 文件添加 docstring.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root_dir : Path
|
||||
根目录
|
||||
|
||||
Returns
|
||||
-------
|
||||
int
|
||||
添加的 docstring 数量
|
||||
"""
|
||||
count = 0
|
||||
for py_file in root_dir.rglob("*.py"):
|
||||
if any(pattern in str(py_file) for pattern in IGNORE_PATTERNS):
|
||||
continue
|
||||
|
||||
docstring = generate_module_docstring(py_file)
|
||||
if add_docstring(py_file, docstring):
|
||||
count += 1
|
||||
|
||||
print(f"共添加 {count} 个 docstring")
|
||||
return count
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def sync_pyproject_config(root_dir: Path) -> None:
|
||||
"""同步 pyproject.toml 配置到子项目.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root_dir : Path
|
||||
根目录
|
||||
"""
|
||||
main_toml = root_dir / "pyproject.toml"
|
||||
if not main_toml.exists():
|
||||
print(f"主项目配置文件不存在: {main_toml}")
|
||||
return
|
||||
|
||||
sub_tomls = [p for p in root_dir.rglob("pyproject.toml") if p != main_toml and ".venv" not in str(p)]
|
||||
|
||||
if not sub_tomls:
|
||||
print("没有找到子项目的 pyproject.toml")
|
||||
return
|
||||
|
||||
print(f"找到 {len(sub_tomls)} 个子项目配置文件")
|
||||
|
||||
for sub_toml in sub_tomls:
|
||||
subprocess.run(["ruff", "format", str(sub_toml)], check=False)
|
||||
|
||||
print("配置同步完成")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def format_all(root_dir: Path) -> None:
|
||||
"""格式化所有 Python 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root_dir : Path
|
||||
根目录
|
||||
"""
|
||||
subprocess.run(["ruff", "format", str(root_dir)], check=True)
|
||||
subprocess.run(["ruff", "check", "--fix", "--unsafe-fixes", str(root_dir)], check=True)
|
||||
print(f"格式化完成: {root_dir}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# bumpversion 私有辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _get_pattern_for_file(file_name: str) -> re.Pattern[str] | None:
|
||||
"""根据文件类型获取对应的正则表达式."""
|
||||
if file_name == "pyproject.toml":
|
||||
return _PYPROJECT_VERSION_PATTERN
|
||||
if file_name == "__init__.py":
|
||||
return _INIT_VERSION_PATTERN
|
||||
return None
|
||||
|
||||
|
||||
def _calculate_new_version(major: int, minor: int, patch: int, part: BumpVersionType) -> str:
|
||||
"""计算新版本号."""
|
||||
if part == "major":
|
||||
return f"{major + 1}.0.0"
|
||||
if part == "minor":
|
||||
return f"{major}.{minor + 1}.0"
|
||||
return f"{major}.{minor}.{patch + 1}"
|
||||
|
||||
|
||||
def _build_replacement_string(original_match: str, new_version: str, file_name: str) -> str:
|
||||
"""构建替换字符串, 保留原始格式."""
|
||||
quote_char = '"' if '"' in original_match else "'"
|
||||
|
||||
if file_name == "pyproject.toml":
|
||||
prefix_match = re.match(r'(\s*version\s*=\s*)["\']', original_match)
|
||||
prefix = prefix_match.group(1) if prefix_match else "version = "
|
||||
return f"{prefix}{quote_char}{new_version}{quote_char}"
|
||||
|
||||
if file_name == "__init__.py":
|
||||
prefix_match = re.match(r'(\s*__version__\s*=\s*)["\']', original_match)
|
||||
prefix = prefix_match.group(1) if prefix_match else "__version__ = "
|
||||
return f"{prefix}{quote_char}{new_version}{quote_char}"
|
||||
|
||||
return new_version
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# bumpversion 函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def bump_file_version(file_path: Path, part: BumpVersionType = "patch") -> str | None:
|
||||
"""更新文件中的版本号.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_path : Path
|
||||
要更新的文件路径
|
||||
part : BumpVersionType
|
||||
版本部分: patch, minor, major
|
||||
|
||||
Returns
|
||||
-------
|
||||
str | None
|
||||
更新后的新版本号, 如果文件中未找到版本号则返回 None
|
||||
"""
|
||||
try:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
except Exception as e:
|
||||
print(f"读取文件 {file_path} 时出错: {e}")
|
||||
raise
|
||||
|
||||
pattern = _get_pattern_for_file(file_path.name)
|
||||
|
||||
if pattern:
|
||||
match = pattern.search(content)
|
||||
else:
|
||||
match = _PYPROJECT_VERSION_PATTERN.search(content) or _INIT_VERSION_PATTERN.search(content)
|
||||
|
||||
if not match:
|
||||
print(f"文件 {file_path} 中未找到版本号模式")
|
||||
return None
|
||||
|
||||
major = int(match.group("major"))
|
||||
minor = int(match.group("minor"))
|
||||
patch = int(match.group("patch"))
|
||||
|
||||
new_version = _calculate_new_version(major, minor, patch, part)
|
||||
|
||||
original_match = match.group(0)
|
||||
replacement = _build_replacement_string(original_match, new_version, file_path.name)
|
||||
|
||||
content = content.replace(original_match, replacement)
|
||||
|
||||
try:
|
||||
file_path.write_text(content, encoding="utf-8")
|
||||
except Exception as e:
|
||||
print(f"更新文件 {file_path} 版本号时出错: {e}")
|
||||
raise
|
||||
|
||||
return new_version
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def bump_project_version(part: BumpVersionType = "patch", no_tag: bool = False) -> str | None:
|
||||
"""批量更新项目所有文件的版本号并提交.
|
||||
|
||||
搜索当前目录下所有 ``__init__.py`` 和 ``pyproject.toml`` 文件
|
||||
(排除虚拟环境和缓存目录), 更新版本号, 然后执行 git add + commit + tag.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
part : BumpVersionType
|
||||
版本部分: patch, minor, major
|
||||
no_tag : bool
|
||||
提交后不创建 git tag
|
||||
|
||||
Returns
|
||||
-------
|
||||
str | None
|
||||
更新后的新版本号, 如果未找到版本号文件则返回 None
|
||||
"""
|
||||
ignore_dirs = {".venv", "venv", ".git", "__pycache__", ".tox", "node_modules", "build", "dist", ".eggs"}
|
||||
all_files: set[Path] = set()
|
||||
|
||||
for pattern in ["__init__.py", "pyproject.toml"]:
|
||||
for file in Path.cwd().rglob(pattern):
|
||||
if not any(ignore_dir in file.parts for ignore_dir in ignore_dirs):
|
||||
all_files.add(file)
|
||||
|
||||
if not all_files:
|
||||
print("未找到包含版本号的文件")
|
||||
return None
|
||||
|
||||
print(f"找到 {len(all_files)} 个文件需要更新版本号")
|
||||
for file in sorted(all_files):
|
||||
print(f" - {file.relative_to(Path.cwd())}")
|
||||
|
||||
new_version: str | None = None
|
||||
for file in sorted(all_files):
|
||||
version = bump_file_version(file, part)
|
||||
if version is not None and new_version is None:
|
||||
new_version = version
|
||||
|
||||
if not new_version:
|
||||
print("未能获取新版本号")
|
||||
return None
|
||||
|
||||
print(f"版本号已更新为: {new_version}")
|
||||
|
||||
subprocess.run(["git", "add", "."], check=False)
|
||||
subprocess.run(["git", "commit", "-m", f"bump version to {new_version}"], check=False)
|
||||
|
||||
if not no_tag:
|
||||
tag_name = f"v{new_version}"
|
||||
subprocess.run(["git", "tag", "-a", tag_name, "-m", f"Release {tag_name}"], check=False)
|
||||
print(f"已创建标签: {tag_name}")
|
||||
|
||||
return new_version
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# piptool 私有辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _get_installed_packages() -> list[str]:
|
||||
"""获取当前环境中所有已安装的包名."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pip", "list", "--format=freeze"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
packages: list[str] = []
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
if line and "==" in line:
|
||||
pkg_name = line.split("==")[0].strip()
|
||||
packages.append(pkg_name)
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
return []
|
||||
return packages
|
||||
|
||||
|
||||
def _expand_wildcard_packages(pattern: str) -> list[str]:
|
||||
"""展开通配符模式为实际的包名列表."""
|
||||
if not any(char in pattern for char in ["*", "?", "[", "]"]):
|
||||
return [pattern]
|
||||
|
||||
installed_packages = _get_installed_packages()
|
||||
matched = [pkg for pkg in installed_packages if fnmatch.fnmatchcase(pkg.lower(), pattern.lower())]
|
||||
return matched
|
||||
|
||||
|
||||
def _filter_protected_packages(packages: list[str]) -> list[str]:
|
||||
"""过滤掉受保护的包名."""
|
||||
safe = [p for p in packages if p.lower() not in {p.lower() for p in _PROTECTED_PACKAGES}]
|
||||
filtered = [p for p in packages if p.lower() in {p.lower() for p in _PROTECTED_PACKAGES}]
|
||||
if filtered:
|
||||
print(f"跳过受保护的包: {', '.join(filtered)}")
|
||||
return safe
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# piptool 函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pip_uninstall(pkg_names: list[str]) -> None:
|
||||
"""卸载包."""
|
||||
packages_to_uninstall: list[str] = []
|
||||
for pattern in pkg_names:
|
||||
packages_to_uninstall.extend(_expand_wildcard_packages(pattern))
|
||||
|
||||
packages_to_uninstall = _filter_protected_packages(packages_to_uninstall)
|
||||
|
||||
if not packages_to_uninstall:
|
||||
return
|
||||
|
||||
subprocess.run(["pip", "uninstall", "-y", *packages_to_uninstall], check=True)
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pip_reinstall(pkg_names: list[str], offline: bool = False) -> None:
|
||||
"""重新安装包."""
|
||||
safe_ps = _filter_protected_packages(pkg_names)
|
||||
if not safe_ps:
|
||||
print("所有指定的包均为受保护包, 跳过重装")
|
||||
return
|
||||
|
||||
subprocess.run(["pip", "uninstall", "-y", *safe_ps], check=True)
|
||||
|
||||
options = ["--no-index", "--find-links", "."] if offline else []
|
||||
subprocess.run(["pip", "install", *options, *safe_ps], check=True)
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pip_download(pkg_names: list[str], offline: bool = False) -> None:
|
||||
"""下载包."""
|
||||
options = ["--no-index", "--find-links", "."] if offline else []
|
||||
subprocess.run(
|
||||
["pip", "download", *pkg_names, *options, "-d", PACKAGE_DIR],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pip_freeze() -> None:
|
||||
"""冻结依赖."""
|
||||
result = subprocess.run(
|
||||
["pip", "freeze", "--exclude-editable"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
Path(REQUIREMENTS_FILE).write_text(result.stdout)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# gittool 函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def init_sub_dirs() -> None:
|
||||
"""初始化子目录的 Git 仓库."""
|
||||
sub_dirs = [subdir for subdir in Path.cwd().iterdir() if subdir.is_dir()]
|
||||
for subdir in sub_dirs:
|
||||
px.run(
|
||||
px.Graph().chain(
|
||||
px.cmd(["git", "init"], conditions=(lambda _: not_has_git_repo(),), cwd=subdir),
|
||||
px.cmd(["git", "add", "."]),
|
||||
px.cmd(["git", "commit", "-m", "init commit"]),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def not_has_git_repo() -> bool:
|
||||
"""检查当前目录没有 Git 仓库."""
|
||||
return not Path.cwd().exists() or not (Path.cwd() / ".git").is_dir()
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def has_files() -> bool:
|
||||
"""检查当前 Git 仓库是否有未提交的更改."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "status", "--porcelain"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return bool(result.stdout.strip())
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def git_add_commit(message: str = "chore: update") -> None:
|
||||
"""执行 git add + git commit (仅当有未提交更改时).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
message : str
|
||||
提交信息
|
||||
"""
|
||||
if not has_files():
|
||||
print("没有文件需要提交")
|
||||
return
|
||||
subprocess.run(["git", "add", "."], check=True)
|
||||
subprocess.run(["git", "commit", "-m", message], check=True)
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def git_init_add_commit(message: str = "init commit") -> None:
|
||||
"""执行 git init (若需) + git add + git commit (若有更改).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
message : str
|
||||
提交信息
|
||||
"""
|
||||
if not_has_git_repo():
|
||||
subprocess.run(["git", "init"], check=True)
|
||||
if has_files():
|
||||
subprocess.run(["git", "add", "."], check=True)
|
||||
subprocess.run(["git", "commit", "-m", message], check=True)
|
||||
else:
|
||||
print("没有文件需要提交")
|
||||
@@ -0,0 +1,327 @@
|
||||
"""文件类函数模块.
|
||||
|
||||
聚合文件日期处理、文件等级重命名、文件夹备份、文件夹压缩工具的可复用函数.
|
||||
所有公共函数通过 ``@px.register_fn`` 注册, 供 YAML 任务编排引用.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"BRACKETS",
|
||||
"DATE_PATTERN",
|
||||
"IGNORE_DIRS",
|
||||
"IGNORE_EXT",
|
||||
"IGNORE_FILES",
|
||||
"LEVELS",
|
||||
"SEP",
|
||||
"add_date_prefix",
|
||||
"archive_folder",
|
||||
"backup_folder",
|
||||
"folderback_default",
|
||||
"folderzip_default",
|
||||
"get_file_timestamp",
|
||||
"process_file_date",
|
||||
"process_file_level",
|
||||
"process_files_date",
|
||||
"process_files_level",
|
||||
"remove_date_prefix",
|
||||
"remove_dump",
|
||||
"remove_marks",
|
||||
"zip_folders",
|
||||
"zip_target",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# filedate 配置
|
||||
# ============================================================================
|
||||
|
||||
DATE_PATTERN = re.compile(r"(20|19)\d{2}[-_#.~]?((0[1-9])|(1[012]))[-_#.~]?((0[1-9])|([12]\d)|(3[01]))[-_#.~]?")
|
||||
SEP = "_"
|
||||
|
||||
# ============================================================================
|
||||
# filelevel 配置
|
||||
# ============================================================================
|
||||
|
||||
LEVELS: dict[str, str] = {
|
||||
"0": "",
|
||||
"1": "PUB,NOR",
|
||||
"2": "INT",
|
||||
"3": "CON",
|
||||
"4": "CLA",
|
||||
}
|
||||
|
||||
BRACKETS: tuple[str, str] = (" ([_(【-", " )]_)】")
|
||||
|
||||
# ============================================================================
|
||||
# folderzip 配置
|
||||
# ============================================================================
|
||||
|
||||
IGNORE_DIRS: list[str] = [".git", ".idea", ".vscode", "__pycache__"]
|
||||
IGNORE_FILES: list[str] = [".gitignore"]
|
||||
IGNORE: list[str] = [*IGNORE_DIRS, *IGNORE_FILES]
|
||||
IGNORE_EXT: list[str] = [".zip", ".rar", ".7z", ".tar", ".gz"]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# filedate 函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def get_file_timestamp(filepath: Path) -> str:
|
||||
"""获取文件时间戳."""
|
||||
modified_time = filepath.stat().st_mtime
|
||||
created_time = filepath.stat().st_ctime
|
||||
return time.strftime("%Y%m%d", time.localtime(max((modified_time, created_time))))
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def remove_date_prefix(filepath: Path) -> Path:
|
||||
"""移除文件日期前缀."""
|
||||
stem = filepath.stem
|
||||
new_stem = DATE_PATTERN.sub("", stem)
|
||||
if new_stem != stem:
|
||||
new_path = filepath.with_name(new_stem + filepath.suffix)
|
||||
filepath.rename(new_path)
|
||||
return new_path
|
||||
return filepath
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def add_date_prefix(filepath: Path) -> Path:
|
||||
"""添加文件日期前缀."""
|
||||
timestamp = get_file_timestamp(filepath)
|
||||
stem = filepath.stem
|
||||
new_stem = f"{timestamp}{SEP}{stem}"
|
||||
new_path = filepath.with_name(new_stem + filepath.suffix)
|
||||
if new_path != filepath:
|
||||
filepath.rename(new_path)
|
||||
return new_path
|
||||
return filepath
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def process_file_date(filepath: Path, clear: bool = False) -> None:
|
||||
"""处理单个文件的日期前缀.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filepath : Path
|
||||
文件路径
|
||||
clear : bool
|
||||
是否清除日期前缀
|
||||
"""
|
||||
if clear:
|
||||
remove_date_prefix(filepath)
|
||||
else:
|
||||
new_path = remove_date_prefix(filepath)
|
||||
add_date_prefix(new_path)
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def process_files_date(targets: list[Path], clear: bool = False) -> None:
|
||||
"""批量处理文件日期前缀.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
targets : list[Path]
|
||||
文件路径列表
|
||||
clear : bool
|
||||
是否清除日期前缀
|
||||
"""
|
||||
for target in targets:
|
||||
if target.exists() and not target.name.startswith("."):
|
||||
process_file_date(target, clear)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# filelevel 函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def remove_marks(stem: str, marks: list[str]) -> str:
|
||||
"""从文件名主干中移除所有标记."""
|
||||
left_brackets, right_brackets = BRACKETS
|
||||
for mark in marks:
|
||||
pos = 0
|
||||
while True:
|
||||
pos = stem.find(mark, pos)
|
||||
if pos == -1:
|
||||
break
|
||||
b, e = pos - 1, pos + len(mark)
|
||||
if b >= 0 and e < len(stem) and stem[b] in left_brackets and stem[e] in right_brackets:
|
||||
stem = stem[:b] + stem[e + 1 :]
|
||||
else:
|
||||
pos = e
|
||||
return stem
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def process_file_level(filepath: Path, level: int = 0) -> None:
|
||||
"""处理单个文件的等级标记.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filepath : Path
|
||||
文件路径
|
||||
level : int
|
||||
文件等级 (0-4), 0 用于清除等级
|
||||
"""
|
||||
if not (0 <= level < len(LEVELS)):
|
||||
print(f"无效的等级 {level}, 必须在 0 和 {len(LEVELS) - 1} 之间")
|
||||
return
|
||||
|
||||
if not filepath.exists():
|
||||
print(f"文件不存在: {filepath}")
|
||||
return
|
||||
|
||||
filestem = filepath.stem
|
||||
original_stem = filestem
|
||||
|
||||
for level_names in LEVELS.values():
|
||||
if level_names:
|
||||
filestem = remove_marks(filestem, level_names.split(","))
|
||||
|
||||
for digit in map(str, range(1, 10)):
|
||||
filestem = remove_marks(filestem, [digit])
|
||||
|
||||
if level > 0:
|
||||
levelstr = LEVELS.get(str(level), "").split(",")[0]
|
||||
if levelstr:
|
||||
filestem = f"{filestem}({levelstr})"
|
||||
|
||||
if filestem != original_stem:
|
||||
new_path = filepath.with_name(filestem + filepath.suffix)
|
||||
filepath.rename(new_path)
|
||||
print(f"重命名: {filepath} -> {new_path}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def process_files_level(targets: list[Path], level: int = 0) -> None:
|
||||
"""批量处理文件等级标记.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
targets : list[Path]
|
||||
文件路径列表
|
||||
level : int
|
||||
文件等级 (0-4)
|
||||
"""
|
||||
for target in targets:
|
||||
process_file_level(target, level)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# folderback 函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def remove_dump(src: Path, dst: Path, max_zip: int) -> None:
|
||||
"""递归删除旧的备份 zip 文件."""
|
||||
zip_paths = [filepath for filepath in dst.rglob("*.zip") if src.stem in str(filepath)]
|
||||
zip_files = sorted(zip_paths, key=lambda fn: str(fn)[-19:-4])
|
||||
if len(zip_files) > max_zip:
|
||||
zip_files[0].unlink()
|
||||
remove_dump(src, dst, max_zip)
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def zip_target(src: Path, dst: Path, max_zip: int) -> None:
|
||||
"""将单个文件或文件夹压缩为 zip 文件."""
|
||||
files = [str(_) for _ in src.rglob("*")]
|
||||
timestamp = time.strftime("_%Y%m%d_%H%M%S")
|
||||
target_path = dst / (src.stem + timestamp + ".zip")
|
||||
|
||||
with zipfile.ZipFile(target_path, "w") as zip_file:
|
||||
for file in files:
|
||||
zip_file.write(file, arcname=file.replace(str(src.parent), ""))
|
||||
|
||||
remove_dump(src, dst, max_zip)
|
||||
print(f"备份完成: {target_path}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def backup_folder(src: str, dst: str, max_zip: int = 5) -> None:
|
||||
"""备份文件夹.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
src : str
|
||||
源文件夹路径
|
||||
dst : str
|
||||
目标文件夹路径
|
||||
max_zip : int
|
||||
最大备份数量
|
||||
"""
|
||||
src_path = Path(src)
|
||||
dst_path = Path(dst)
|
||||
|
||||
if not src_path.exists():
|
||||
print(f"源文件夹不存在: {src_path}")
|
||||
return
|
||||
|
||||
if not dst_path.exists():
|
||||
dst_path.mkdir(parents=True, exist_ok=True)
|
||||
print(f"创建目标文件夹: {dst_path}")
|
||||
|
||||
zip_target(src_path, dst_path, max_zip)
|
||||
|
||||
|
||||
@px.register_fn("folderback_default")
|
||||
def folderback_default() -> None:
|
||||
"""备份当前目录到 ./backup."""
|
||||
backup_folder(".", "./backup", 5)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# folderzip 函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def archive_folder(folder: Path) -> None:
|
||||
"""压缩单个文件夹."""
|
||||
shutil.make_archive(
|
||||
str(folder.with_name(folder.name)),
|
||||
format="zip",
|
||||
base_dir=folder,
|
||||
)
|
||||
print(f"压缩完成: {folder.name}.zip")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def zip_folders(cwd: str = ".") -> None:
|
||||
"""压缩目录下的所有文件夹.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
cwd : str
|
||||
工作目录
|
||||
"""
|
||||
cwd_path = Path(cwd)
|
||||
if not cwd_path.exists():
|
||||
print(f"目录不存在: {cwd_path}")
|
||||
return
|
||||
|
||||
dirs: list[Path] = [
|
||||
e for e in cwd_path.iterdir() if e.is_dir() and e.name not in IGNORE_DIRS and e.suffix not in IGNORE_EXT
|
||||
]
|
||||
|
||||
for dir_path in dirs:
|
||||
archive_folder(dir_path)
|
||||
|
||||
|
||||
@px.register_fn("folderzip_default")
|
||||
def folderzip_default() -> None:
|
||||
"""压缩当前目录下的所有文件夹."""
|
||||
zip_folders(".")
|
||||
@@ -0,0 +1,498 @@
|
||||
"""媒体类函数模块.
|
||||
|
||||
聚合 PDF 工具 (pdftool) 和截图工具 (screenshot) 的可复用函数.
|
||||
所有公共函数通过 ``@px.register_fn`` 注册, 供 YAML 任务编排引用.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_PASSWORD",
|
||||
"DEFAULT_QUALITY",
|
||||
"PDF_SUFFIX",
|
||||
"get_screenshot_path",
|
||||
"pdf_add_watermark",
|
||||
"pdf_compress",
|
||||
"pdf_crop",
|
||||
"pdf_decrypt",
|
||||
"pdf_encrypt",
|
||||
"pdf_extract_images",
|
||||
"pdf_extract_text",
|
||||
"pdf_info",
|
||||
"pdf_merge",
|
||||
"pdf_ocr",
|
||||
"pdf_reorder",
|
||||
"pdf_repair",
|
||||
"pdf_rotate",
|
||||
"pdf_split",
|
||||
"pdf_to_images",
|
||||
"take_screenshot_area",
|
||||
"take_screenshot_full",
|
||||
]
|
||||
|
||||
try:
|
||||
import fitz # PyMuPDF
|
||||
|
||||
HAS_PYMUPDF = True
|
||||
except ImportError:
|
||||
HAS_PYMUPDF = False
|
||||
|
||||
try:
|
||||
import pypdf
|
||||
|
||||
HAS_PYPDF = True
|
||||
except ImportError:
|
||||
HAS_PYPDF = False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
PDF_SUFFIX = ".pdf"
|
||||
DEFAULT_QUALITY = 75
|
||||
DEFAULT_PASSWORD = ""
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# PDF 函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_merge(input_paths: list[Path], output_path: Path) -> None:
|
||||
"""合并多个 PDF 文件."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库, 请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
writer = pypdf.PdfWriter()
|
||||
for input_path in input_paths:
|
||||
if input_path.exists():
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"合并完成: {output_path}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_split(input_path: Path, output_dir: Path) -> None:
|
||||
"""拆分 PDF 文件为单页."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库, 请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for i, page in enumerate(reader.pages):
|
||||
writer = pypdf.PdfWriter()
|
||||
writer.add_page(page)
|
||||
output_file = output_dir / f"{input_path.stem}_page_{i + 1}.pdf"
|
||||
with open(output_file, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"拆分完成: {output_dir}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_compress(input_path: Path, output_path: Path) -> None:
|
||||
"""压缩 PDF 文件."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path), garbage=4, deflate=True, clean=True)
|
||||
doc.close()
|
||||
|
||||
original_size = input_path.stat().st_size
|
||||
new_size = output_path.stat().st_size
|
||||
ratio = (1 - new_size / original_size) * 100
|
||||
print(f"压缩完成: {output_path} (缩小 {ratio:.1f}%)")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_encrypt(input_path: Path, output_path: Path, password: str) -> None:
|
||||
"""加密 PDF 文件."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库, 请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
writer = pypdf.PdfWriter()
|
||||
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
writer.encrypt(user_password=password, owner_password=password, use_128bit=True)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"加密完成: {output_path}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_decrypt(input_path: Path, output_path: Path, password: str) -> None:
|
||||
"""解密 PDF 文件."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库, 请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
if reader.is_encrypted:
|
||||
reader.decrypt(password)
|
||||
|
||||
writer = pypdf.PdfWriter()
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"解密完成: {output_path}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_extract_text(input_path: Path, output_path: Path) -> None:
|
||||
"""提取 PDF 文本."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
text = ""
|
||||
for page in doc:
|
||||
text += str(page.get_text()) + "\n\n"
|
||||
doc.close()
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(text, encoding="utf-8")
|
||||
print(f"文本提取完成: {output_path}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_extract_images(input_path: Path, output_dir: Path) -> None:
|
||||
"""提取 PDF 图片."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
image_count = 0
|
||||
# pyrefly: ignore [bad-argument-type]
|
||||
for page_num, page in enumerate(doc):
|
||||
images = page.get_images(full=True)
|
||||
for img_idx, img in enumerate(images):
|
||||
xref = img[0]
|
||||
base_image = doc.extract_image(xref)
|
||||
image_data = base_image["image"]
|
||||
image_ext = base_image["ext"]
|
||||
image_path = output_dir / f"page_{page_num + 1}_img_{img_idx + 1}.{image_ext}"
|
||||
image_path.write_bytes(image_data)
|
||||
image_count += 1
|
||||
|
||||
doc.close()
|
||||
print(f"图片提取完成: {output_dir} (共 {image_count} 张)")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_add_watermark(input_path: Path, output_path: Path, text: str = "CONFIDENTIAL") -> None:
|
||||
"""添加 PDF 水印."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
for page in doc:
|
||||
rect = page.rect
|
||||
text_width = fitz.get_text_length(text, fontsize=48)
|
||||
x = (rect.width - text_width) / 2
|
||||
y = rect.height / 2
|
||||
page.insert_text((x, y), text, fontsize=48, rotate=45, color=(0, 0, 0))
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"水印添加完成: {output_path}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_rotate(input_path: Path, output_path: Path, rotation: int = 90) -> None:
|
||||
"""旋转 PDF 页面."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
for page in doc:
|
||||
page.set_rotation(rotation)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"旋转完成: {output_path}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_crop(input_path: Path, output_path: Path, margins: tuple[int, int, int, int]) -> None:
|
||||
"""裁剪 PDF 页面."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
left, top, right, bottom = margins
|
||||
|
||||
for page in doc:
|
||||
rect = page.rect
|
||||
new_rect = fitz.Rect(
|
||||
rect.x0 + left,
|
||||
rect.y0 + top,
|
||||
rect.x1 - right,
|
||||
rect.y1 - bottom,
|
||||
)
|
||||
page.set_cropbox(new_rect)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"裁剪完成: {output_path}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_info(input_path: Path) -> None:
|
||||
"""显示 PDF 信息."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
print(f"文件: {input_path}")
|
||||
print(f"页数: {doc.page_count}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"标题: {doc.metadata.get('title', 'N/A')}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"作者: {doc.metadata.get('author', 'N/A')}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"创建日期: {doc.metadata.get('creationDate', 'N/A')}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"修改日期: {doc.metadata.get('modDate', 'N/A')}")
|
||||
print(f"文件大小: {input_path.stat().st_size / 1024:.1f} KB")
|
||||
doc.close()
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_ocr(input_path: Path, output_path: Path, lang: str = "chi_sim+eng") -> None:
|
||||
"""PDF OCR 识别."""
|
||||
try:
|
||||
import pytesseract
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
print("未安装 OCR 相关库, 请安装: pip install pytesseract pillow")
|
||||
return
|
||||
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
new_doc = fitz.open()
|
||||
|
||||
for page in doc:
|
||||
pix = page.get_pixmap()
|
||||
img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
|
||||
ocr_text = pytesseract.image_to_string(img, lang=lang)
|
||||
|
||||
new_page = new_doc.new_page(width=page.rect.width, height=page.rect.height)
|
||||
new_page.insert_image(new_page.rect, pixmap=pix)
|
||||
text_rect = fitz.Rect(0, 0, page.rect.width, page.rect.height)
|
||||
# pyrefly: ignore [bad-argument-type]
|
||||
new_page.insert_textbox(text_rect, ocr_text, fontname="china-ss", fontsize=11)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
new_doc.save(str(output_path))
|
||||
new_doc.close()
|
||||
doc.close()
|
||||
print(f"OCR 识别完成: {output_path}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_reorder(input_path: Path, output_path: Path, order: list[int]) -> None:
|
||||
"""重排 PDF 页面顺序."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库, 请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
writer = pypdf.PdfWriter()
|
||||
|
||||
for page_num in order:
|
||||
if 0 <= page_num < len(reader.pages):
|
||||
writer.add_page(reader.pages[page_num])
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"重排完成: {output_path}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_to_images(input_path: Path, output_dir: Path, dpi: int = 300) -> None:
|
||||
"""PDF 转图片."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# pyrefly: ignore [bad-argument-type]
|
||||
for page_num, page in enumerate(doc):
|
||||
pix = page.get_pixmap(dpi=dpi)
|
||||
image_path = output_dir / f"{input_path.stem}_page_{page_num + 1}.png"
|
||||
pix.save(str(image_path))
|
||||
|
||||
doc.close()
|
||||
print(f"转换完成: {output_dir}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pdf_repair(input_path: Path, output_path: Path) -> None:
|
||||
"""修复 PDF 文件."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path), garbage=4, deflate=True, clean=True)
|
||||
doc.close()
|
||||
print(f"修复完成: {output_path}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# screenshot 函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def get_screenshot_path(filename: str | None = None) -> Path:
|
||||
"""获取截图保存路径.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filename : str | None
|
||||
文件名, 如果为 None 则自动生成
|
||||
|
||||
Returns
|
||||
-------
|
||||
Path
|
||||
截图保存路径
|
||||
"""
|
||||
if filename is None:
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
filename = f"screenshot_{timestamp}.png"
|
||||
|
||||
screenshots_dir = Path.home() / "Pictures" / "screenshots"
|
||||
screenshots_dir.mkdir(parents=True, exist_ok=True)
|
||||
return screenshots_dir / filename
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def take_screenshot_full(filename: str | None = None) -> None:
|
||||
"""全屏截图.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filename : str | None
|
||||
文件名
|
||||
"""
|
||||
output_path = get_screenshot_path(filename)
|
||||
|
||||
if Constants.IS_WINDOWS:
|
||||
ps_script = f"""
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
$screen = [System.Windows.Forms.Screen]::PrimaryScreen
|
||||
$bounds = $screen.Bounds
|
||||
$bitmap = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height
|
||||
$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
|
||||
$graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)
|
||||
$bitmap.Save('{output_path.as_posix()}')
|
||||
$graphics.Dispose()
|
||||
$bitmap.Dispose()
|
||||
"""
|
||||
subprocess.run(["powershell", "-Command", ps_script], check=True)
|
||||
elif Constants.IS_MACOS:
|
||||
subprocess.run(["screencapture", "-x", str(output_path)], check=True)
|
||||
else:
|
||||
try:
|
||||
subprocess.run(["gnome-screenshot", "-f", str(output_path)], check=True)
|
||||
except FileNotFoundError:
|
||||
subprocess.run(["scrot", str(output_path)], check=True)
|
||||
|
||||
print(f"截图已保存: {output_path}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def take_screenshot_area(filename: str | None = None) -> None:
|
||||
"""区域截图.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filename : str | None
|
||||
文件名
|
||||
"""
|
||||
output_path = get_screenshot_path(filename)
|
||||
|
||||
if Constants.IS_WINDOWS:
|
||||
ps_script = f"""
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
$form = New-Object System.Windows.Forms.Form
|
||||
$form.WindowState = 'Maximized'
|
||||
$form.FormBorderStyle = 'None'
|
||||
$form.BackColor = [System.Drawing.Color]::FromArgb(1, 0, 0)
|
||||
$form.Opacity = 0.5
|
||||
$form.TopMost = $true
|
||||
$form.Show()
|
||||
Start-Sleep -Milliseconds 100
|
||||
$screen = [System.Windows.Forms.Screen]::PrimaryScreen
|
||||
$bounds = $screen.Bounds
|
||||
$bitmap = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height
|
||||
$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
|
||||
$graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)
|
||||
$form.Close()
|
||||
$bitmap.Save('{output_path.as_posix()}')
|
||||
$graphics.Dispose()
|
||||
$bitmap.Dispose()
|
||||
"""
|
||||
subprocess.run(["powershell", "-Command", ps_script], check=True)
|
||||
elif Constants.IS_MACOS:
|
||||
subprocess.run(["screencapture", "-i", str(output_path)], check=True)
|
||||
else:
|
||||
try:
|
||||
subprocess.run(["gnome-screenshot", "-a", "-f", str(output_path)], check=True)
|
||||
except FileNotFoundError:
|
||||
subprocess.run(["scrot", "-s", str(output_path)], check=True)
|
||||
|
||||
print(f"截图已保存: {output_path}")
|
||||
@@ -0,0 +1,504 @@
|
||||
"""系统类函数模块.
|
||||
|
||||
聚合 LS-DYNA 计算 (lscalc)、SSH 密钥部署 (sshcopyid)、Python 打包 (packtool)、
|
||||
重置图标缓存 (reset_icon_cache) 的可复用函数. 所有公共函数通过 ``@px.register_fn``
|
||||
注册, 供 YAML 任务编排引用.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_BUILD_DIR",
|
||||
"DEFAULT_CACHE_DIR",
|
||||
"DEFAULT_DIST_DIR",
|
||||
"DEFAULT_INPUT_FILE",
|
||||
"DEFAULT_LIB_DIR",
|
||||
"DEFAULT_NCPU",
|
||||
"IGNORE_PATTERNS",
|
||||
"LS_DYNA_COMMANDS",
|
||||
"check_ls_dyna_status",
|
||||
"clean_build_dir",
|
||||
"create_zip_package",
|
||||
"get_ls_dyna_command",
|
||||
"install_embed_python",
|
||||
"pack_dependencies",
|
||||
"pack_source",
|
||||
"pack_wheel",
|
||||
"reset_icon_cache_run",
|
||||
"run_ls_dyna",
|
||||
"run_ls_dyna_mpi",
|
||||
"ssh_copy_id",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# lscalc 配置
|
||||
# ============================================================================
|
||||
|
||||
LS_DYNA_COMMANDS: dict[str, list[str]] = {
|
||||
"windows": ["ls-dyna_mpp", "i=input.k", "ncpu=4"],
|
||||
"linux": ["ls-dyna_mpp", "i=input.k", "ncpu=8"],
|
||||
"macos": ["ls-dyna_mpp", "i=input.k", "ncpu=4"],
|
||||
}
|
||||
|
||||
DEFAULT_INPUT_FILE: str = "input.k"
|
||||
DEFAULT_NCPU: int = 4
|
||||
|
||||
# ============================================================================
|
||||
# packtool 配置
|
||||
# ============================================================================
|
||||
|
||||
DEFAULT_BUILD_DIR = ".pypack"
|
||||
DEFAULT_DIST_DIR = "dist"
|
||||
DEFAULT_LIB_DIR = "libs"
|
||||
DEFAULT_CACHE_DIR = ".cache/pypack"
|
||||
|
||||
IGNORE_PATTERNS = [
|
||||
"__pycache__",
|
||||
"*.pyc",
|
||||
"*.pyo",
|
||||
".git",
|
||||
".venv",
|
||||
".idea",
|
||||
".vscode",
|
||||
"*.egg-info",
|
||||
"dist",
|
||||
"build",
|
||||
".pytest_cache",
|
||||
".tox",
|
||||
".mypy_cache",
|
||||
]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# lscalc 函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def get_ls_dyna_command(input_file: str, ncpu: int) -> list[str]:
|
||||
"""获取 LS-DYNA 命令.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_file : str
|
||||
输入文件路径
|
||||
ncpu : int
|
||||
CPU 核心数
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[str]
|
||||
LS-DYNA 命令列表
|
||||
"""
|
||||
if Constants.IS_WINDOWS or Constants.IS_MACOS:
|
||||
return ["ls-dyna_mpp", f"i={input_file}", f"ncpu={ncpu}"]
|
||||
else:
|
||||
return ["ls-dyna_mpp", f"i={input_file}", f"ncpu={ncpu}"]
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def run_ls_dyna(input_file: str, ncpu: int = DEFAULT_NCPU) -> None:
|
||||
"""运行 LS-DYNA 计算.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_file : str
|
||||
输入文件路径
|
||||
ncpu : int
|
||||
CPU 核心数
|
||||
"""
|
||||
input_path = Path(input_file)
|
||||
if not input_path.exists():
|
||||
print(f"输入文件不存在: {input_path}")
|
||||
return
|
||||
|
||||
cmd = get_ls_dyna_command(input_file, ncpu)
|
||||
try:
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"LS-DYNA 计算完成: {input_file}")
|
||||
except FileNotFoundError:
|
||||
print("未找到 ls-dyna_mpp 命令")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"LS-DYNA 计算失败: {e}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def run_ls_dyna_mpi(input_file: str, ncpu: int = DEFAULT_NCPU) -> None:
|
||||
"""运行 LS-DYNA MPI 计算.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_file : str
|
||||
输入文件路径
|
||||
ncpu : int
|
||||
CPU 核心数
|
||||
"""
|
||||
input_path = Path(input_file)
|
||||
if not input_path.exists():
|
||||
print(f"输入文件不存在: {input_path}")
|
||||
return
|
||||
|
||||
cmd = ["mpirun", "-np", str(ncpu), "ls-dyna_mpp", f"i={input_file}"]
|
||||
try:
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"LS-DYNA MPI 计算完成: {input_file}")
|
||||
except FileNotFoundError:
|
||||
print("未找到 mpirun 或 ls-dyna_mpp 命令")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"LS-DYNA MPI 计算失败: {e}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def check_ls_dyna_status() -> None:
|
||||
"""检查 LS-DYNA 进程状态."""
|
||||
try:
|
||||
if Constants.IS_WINDOWS:
|
||||
result = subprocess.run(
|
||||
["tasklist", "/fi", "imagename eq ls-dyna_mpp.exe"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
print(result.stdout)
|
||||
else:
|
||||
result = subprocess.run(
|
||||
["pgrep", "-f", "ls-dyna"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.stdout.strip():
|
||||
print(f"运行中的 LS-DYNA 进程 PID: {result.stdout.strip()}")
|
||||
else:
|
||||
print("没有运行中的 LS-DYNA 进程")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"检查进程状态失败: {e}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# sshcopyid 函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def ssh_copy_id(
|
||||
hostname: str,
|
||||
username: str,
|
||||
password: str,
|
||||
port: int = 22,
|
||||
keypath: str = "~/.ssh/id_rsa.pub",
|
||||
timeout: int = 30,
|
||||
) -> None:
|
||||
"""将 SSH 公钥部署到远程服务器.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
hostname : str
|
||||
远程服务器主机名或 IP 地址
|
||||
username : str
|
||||
远程服务器用户名
|
||||
password : str
|
||||
远程服务器密码
|
||||
port : int
|
||||
SSH 端口, 默认 22
|
||||
keypath : str
|
||||
公钥文件路径, 默认 ~/.ssh/id_rsa.pub
|
||||
timeout : int
|
||||
SSH 操作超时秒数, 默认 30
|
||||
"""
|
||||
pub_key_path = Path(keypath).expanduser()
|
||||
if not pub_key_path.exists():
|
||||
print(f"公钥文件不存在: {pub_key_path}")
|
||||
sys.exit(1)
|
||||
|
||||
pub_key = pub_key_path.read_text().strip()
|
||||
|
||||
script = f"""mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||
cd ~/.ssh && touch authorized_keys && chmod 600 authorized_keys
|
||||
grep -qF '{pub_key.split()[1]}' authorized_keys 2>/dev/null || echo '{pub_key}' >> authorized_keys"""
|
||||
|
||||
try:
|
||||
subprocess.run(
|
||||
[
|
||||
"sshpass",
|
||||
"-p",
|
||||
password,
|
||||
"ssh",
|
||||
"-p",
|
||||
str(port),
|
||||
"-o",
|
||||
"StrictHostKeyChecking=no",
|
||||
"-o",
|
||||
"UserKnownHostsFile=/dev/null",
|
||||
"-o",
|
||||
f"ConnectTimeout={timeout}",
|
||||
f"{username}@{hostname}",
|
||||
script,
|
||||
],
|
||||
check=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
print(f"SSH 密钥已部署到 {username}@{hostname}:{port}")
|
||||
except FileNotFoundError:
|
||||
print(f"未找到 sshpass 工具, 请手动执行: ssh-copy-id -p {port} {username}@{hostname}")
|
||||
sys.exit(1)
|
||||
except subprocess.TimeoutExpired:
|
||||
print("SSH 连接超时")
|
||||
sys.exit(1)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"SSH 执行失败: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# packtool 函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pack_source(project_dir: Path, output_dir: Path) -> None:
|
||||
"""打包项目源码.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
project_dir : Path
|
||||
项目目录
|
||||
output_dir : Path
|
||||
输出目录
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pyproject_file = project_dir / "pyproject.toml"
|
||||
project_name = project_dir.name
|
||||
|
||||
if pyproject_file.exists():
|
||||
try:
|
||||
import tomllib
|
||||
|
||||
content = pyproject_file.read_text(encoding="utf-8")
|
||||
data = tomllib.loads(content)
|
||||
project_name = data.get("project", {}).get("name", project_name)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
source_dir = output_dir / "src" / project_name
|
||||
source_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
src_subdir = project_dir / "src"
|
||||
if src_subdir.exists():
|
||||
shutil.copytree(
|
||||
src_subdir,
|
||||
source_dir / "src",
|
||||
ignore=shutil.ignore_patterns(*IGNORE_PATTERNS),
|
||||
dirs_exist_ok=True,
|
||||
)
|
||||
else:
|
||||
for item in project_dir.iterdir():
|
||||
if item.name in IGNORE_PATTERNS or item.name.startswith("."):
|
||||
continue
|
||||
dst_item = source_dir / item.name
|
||||
if item.is_dir():
|
||||
shutil.copytree(
|
||||
item,
|
||||
dst_item,
|
||||
ignore=shutil.ignore_patterns(*IGNORE_PATTERNS),
|
||||
dirs_exist_ok=True,
|
||||
)
|
||||
else:
|
||||
shutil.copy2(item, dst_item)
|
||||
|
||||
print(f"源码打包完成: {source_dir}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pack_dependencies(lib_dir: Path, dependencies: list[str]) -> None:
|
||||
"""打包项目依赖.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lib_dir : Path
|
||||
依赖库目录
|
||||
dependencies : list[str]
|
||||
依赖列表
|
||||
"""
|
||||
lib_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if not dependencies:
|
||||
print("没有依赖需要打包")
|
||||
return
|
||||
|
||||
cmd = [
|
||||
"pip",
|
||||
"install",
|
||||
"--target",
|
||||
str(lib_dir),
|
||||
"--no-compile",
|
||||
"--no-warn-script-location",
|
||||
]
|
||||
cmd.extend(dependencies)
|
||||
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"依赖打包完成: {lib_dir}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def pack_wheel(project_dir: Path, output_dir: Path) -> None:
|
||||
"""打包项目为 wheel 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
project_dir : Path
|
||||
项目目录
|
||||
output_dir : Path
|
||||
输出目录
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
cmd = [
|
||||
"pip",
|
||||
"wheel",
|
||||
"--no-deps",
|
||||
"--wheel-dir",
|
||||
str(output_dir),
|
||||
str(project_dir),
|
||||
]
|
||||
|
||||
subprocess.run(cmd, check=True)
|
||||
print(f"Wheel 打包完成: {output_dir}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def install_embed_python(version: str, output_dir: Path) -> None:
|
||||
"""安装嵌入式 Python.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
version : str
|
||||
Python 版本 (如: 3.10, 3.11)
|
||||
output_dir : Path
|
||||
输出目录
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
arch = platform.machine().lower()
|
||||
if arch in ["x86_64", "amd64"]:
|
||||
arch = "amd64"
|
||||
elif arch in ["arm64", "aarch64"]:
|
||||
arch = "arm64"
|
||||
|
||||
version_map = {
|
||||
"3.8": "3.8.10",
|
||||
"3.9": "3.9.13",
|
||||
"3.10": "3.10.11",
|
||||
"3.11": "3.11.9",
|
||||
"3.12": "3.12.4",
|
||||
}
|
||||
full_version = version_map.get(version, f"{version}.0")
|
||||
|
||||
url = f"https://www.python.org/ftp/python/{full_version}/python-{full_version}-embed-{arch}.zip"
|
||||
|
||||
cache_file = Path(DEFAULT_CACHE_DIR) / f"python-{full_version}-embed-{arch}.zip"
|
||||
cache_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if not cache_file.exists():
|
||||
print(f"正在下载嵌入式 Python {full_version}...")
|
||||
urllib.request.urlretrieve(url, cache_file)
|
||||
print(f"下载完成: {cache_file}")
|
||||
|
||||
with zipfile.ZipFile(cache_file, "r") as zf:
|
||||
zf.extractall(output_dir)
|
||||
|
||||
print(f"嵌入式 Python 安装完成: {output_dir}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def create_zip_package(source_dir: Path, output_file: Path) -> None:
|
||||
"""创建 ZIP 打包文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
source_dir : Path
|
||||
源目录
|
||||
output_file : Path
|
||||
输出文件
|
||||
"""
|
||||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with zipfile.ZipFile(output_file, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
for file in source_dir.rglob("*"):
|
||||
if file.is_file():
|
||||
arcname = file.relative_to(source_dir)
|
||||
zf.write(file, arcname)
|
||||
|
||||
print(f"ZIP 打包完成: {output_file}")
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def clean_build_dir(build_dir: Path) -> None:
|
||||
"""清理构建目录.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
build_dir : Path
|
||||
构建目录
|
||||
"""
|
||||
if build_dir.exists():
|
||||
shutil.rmtree(build_dir)
|
||||
print(f"清理完成: {build_dir}")
|
||||
else:
|
||||
print(f"目录不存在: {build_dir}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# reseticoncache 函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.register_fn
|
||||
def reset_icon_cache_run() -> None:
|
||||
"""重置 Windows 图标缓存.
|
||||
|
||||
执行流程: 杀掉 explorer → 删除 IconCache.db → 删除 iconcache* → 重启 explorer.
|
||||
仅在 Windows 上执行, 非 Windows 平台打印提示并跳过.
|
||||
"""
|
||||
if not Constants.IS_WINDOWS:
|
||||
print("reset_icon_cache: 仅在 Windows 上支持")
|
||||
return
|
||||
|
||||
local_app_data = os.environ.get("LOCALAPPDATA", "")
|
||||
if not local_app_data:
|
||||
print("reset_icon_cache: LOCALAPPDATA 环境变量未设置")
|
||||
return
|
||||
|
||||
icon_cache_db = Path(local_app_data) / "IconCache.db"
|
||||
explorer_cache_dir = Path(local_app_data) / "Microsoft" / "Windows" / "Explorer"
|
||||
|
||||
print("正在终止 explorer 进程...")
|
||||
subprocess.run(["taskkill", "/f", "/im", "explorer.exe"], check=False)
|
||||
|
||||
if icon_cache_db.exists():
|
||||
print(f"删除图标缓存: {icon_cache_db}")
|
||||
subprocess.run(["cmd", "/c", "del", "/a", "/q", str(icon_cache_db)], check=False)
|
||||
|
||||
if explorer_cache_dir.exists():
|
||||
print(f"清理 Explorer 缓存: {explorer_cache_dir}")
|
||||
subprocess.run(
|
||||
["cmd", "/c", "del", "/a", "/q", str(explorer_cache_dir / "iconcache*")],
|
||||
check=False,
|
||||
)
|
||||
|
||||
print("重启 explorer...")
|
||||
subprocess.run(["cmd", "/c", "start", "explorer.exe"], check=False)
|
||||
print("图标缓存已重置")
|
||||
@@ -0,0 +1,705 @@
|
||||
"""工作流执行性能评估。
|
||||
|
||||
基于 :class:`~pyflowx.report.RunReport` 中已有的 ``started_at`` /
|
||||
``finished_at`` 时间戳进行离线分析,**零运行时开销**——不修改执行流程,
|
||||
不注册回调,不引入额外计时器。
|
||||
|
||||
核心指标
|
||||
--------
|
||||
* **任务级**:每个任务的 wall-clock 耗时、状态、重试次数、等待时间
|
||||
(从最早依赖完成到本任务开始)。
|
||||
* **图级**:总耗时(wall-clock)、关键路径耗时(理论最短耗时)、
|
||||
并行度效率(关键路径耗时 / 总耗时)。
|
||||
* **关键路径**:从源点到汇点的最长依赖路径,识别真正的串行瓶颈。
|
||||
* **并行度**:基于时间线重叠计算瞬时并行度,给出平均并行度与峰值并行度。
|
||||
* **瓶颈识别**:按耗时排序的 Top-N 任务。
|
||||
|
||||
设计原则
|
||||
--------
|
||||
* 数据来源于 ``RunReport`` + ``Graph``,无副作用。
|
||||
* 计算复杂度 O(V+E):拓扑排序 + 单次松弛,适合大规模图。
|
||||
* 所有时间戳用 ``datetime``,与 :class:`TaskResult` 保持一致。
|
||||
|
||||
快速上手
|
||||
--------
|
||||
import pyflowx as px
|
||||
|
||||
report = px.run(graph)
|
||||
profile = px.ProfileReport.from_report(report, graph)
|
||||
print(profile.describe())
|
||||
bottlenecks = profile.top_bottlenecks(3)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = [
|
||||
"ProfileReport",
|
||||
"TaskProfile",
|
||||
]
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from .graph import Graph
|
||||
from .report import RunReport
|
||||
from .task import TaskResult, TaskStatus
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TaskProfile:
|
||||
"""单个任务的性能剖面。
|
||||
|
||||
属性
|
||||
----
|
||||
name:
|
||||
任务名。
|
||||
status:
|
||||
终态(SUCCESS/FAILED/SKIPPED)。
|
||||
duration:
|
||||
wall-clock 执行耗时(秒)。SKIPPED 任务为 0.0。
|
||||
attempts:
|
||||
尝试次数(含首次)。
|
||||
wait_time:
|
||||
从最早硬依赖完成到本任务开始的等待时间(秒)。
|
||||
无硬依赖或 SKIPPED 时为 0.0。
|
||||
is_on_critical_path:
|
||||
是否位于关键路径上。
|
||||
deps:
|
||||
硬依赖任务名列表。
|
||||
"""
|
||||
|
||||
name: str
|
||||
status: TaskStatus
|
||||
duration: float
|
||||
attempts: int
|
||||
wait_time: float
|
||||
is_on_critical_path: bool
|
||||
deps: tuple[str, ...]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""转为 JSON 友好的字典。"""
|
||||
return {
|
||||
"name": self.name,
|
||||
"status": self.status.value,
|
||||
"duration_seconds": round(self.duration, 6),
|
||||
"attempts": self.attempts,
|
||||
"wait_time_seconds": round(self.wait_time, 6),
|
||||
"is_on_critical_path": self.is_on_critical_path,
|
||||
"deps": list(self.deps),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProfileReport:
|
||||
"""工作流执行的性能剖面报告。
|
||||
|
||||
通过 :meth:`from_report` 从 :class:`RunReport` + :class:`Graph` 构建。
|
||||
所有字段在构造时一次性计算完毕,后续访问为 O(1)。
|
||||
"""
|
||||
|
||||
tasks: tuple[TaskProfile, ...]
|
||||
"""所有任务的性能剖面(按拓扑序)。"""
|
||||
|
||||
total_duration: float
|
||||
"""整次运行的 wall-clock 耗时(秒)。"""
|
||||
|
||||
critical_path_duration: float
|
||||
"""关键路径耗时(秒):从最早任务开始到最晚任务结束的最长依赖路径。"""
|
||||
|
||||
critical_path: tuple[str, ...]
|
||||
"""关键路径上的任务名序列(按执行顺序)。"""
|
||||
|
||||
avg_parallelism: float
|
||||
"""平均并行度 = 任务总耗时 / wall-clock 总耗时。"""
|
||||
|
||||
peak_parallelism: int
|
||||
"""峰值并行度:任一时刻同时运行的任务数最大值。"""
|
||||
|
||||
parallelism_efficiency: float
|
||||
"""并行度效率 = 关键路径耗时 / wall-clock 总耗时。``1.0`` 表示完全串行,
|
||||
越大表示并行化收益越低(瓶颈在关键路径上)。"""
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 构建
|
||||
# ------------------------------------------------------------------ #
|
||||
@classmethod
|
||||
def from_report(cls, report: RunReport, graph: Graph) -> ProfileReport:
|
||||
"""从运行报告与图构建性能剖面。
|
||||
|
||||
参数
|
||||
----
|
||||
report:
|
||||
已完成的 :class:`RunReport`,需包含 ``started_at``/``finished_at``。
|
||||
graph:
|
||||
对应的 :class:`Graph`,用于依赖关系与关键路径分析。
|
||||
|
||||
Note
|
||||
-----
|
||||
本方法不修改 ``report`` 或 ``graph``,纯函数式计算。
|
||||
"""
|
||||
task_profiles = cls._build_task_profiles(report, graph)
|
||||
total_duration = cls._calc_total_duration(report)
|
||||
critical_path, critical_duration = cls._calc_critical_path(graph, report)
|
||||
avg_par, peak_par = cls._calc_parallelism(report)
|
||||
efficiency = critical_duration / total_duration if total_duration > 0 else 0.0
|
||||
|
||||
# 标记关键路径上的任务
|
||||
critical_set = set(critical_path)
|
||||
marked = tuple(
|
||||
TaskProfile(
|
||||
name=t.name,
|
||||
status=t.status,
|
||||
duration=t.duration,
|
||||
attempts=t.attempts,
|
||||
wait_time=t.wait_time,
|
||||
is_on_critical_path=t.name in critical_set,
|
||||
deps=t.deps,
|
||||
)
|
||||
for t in task_profiles
|
||||
)
|
||||
|
||||
return cls(
|
||||
tasks=marked,
|
||||
total_duration=total_duration,
|
||||
critical_path_duration=critical_duration,
|
||||
critical_path=critical_path,
|
||||
avg_parallelism=avg_par,
|
||||
peak_parallelism=peak_par,
|
||||
parallelism_efficiency=efficiency,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_task_profiles(report: RunReport, graph: Graph) -> tuple[TaskProfile, ...]:
|
||||
"""构建每个任务的性能剖面。"""
|
||||
profiles: list[TaskProfile] = []
|
||||
for name, result in report.results.items():
|
||||
spec = graph.specs.get(name)
|
||||
deps = tuple(spec.depends_on) if spec is not None else ()
|
||||
duration = result.duration or 0.0
|
||||
wait_time = ProfileReport._calc_wait_time(result, deps, report)
|
||||
profiles.append(
|
||||
TaskProfile(
|
||||
name=name,
|
||||
status=result.status,
|
||||
duration=duration,
|
||||
attempts=result.attempts,
|
||||
wait_time=wait_time,
|
||||
is_on_critical_path=False, # 后续标记
|
||||
deps=deps,
|
||||
)
|
||||
)
|
||||
return tuple(profiles)
|
||||
|
||||
@staticmethod
|
||||
def _calc_wait_time(
|
||||
result: TaskResult[Any],
|
||||
deps: tuple[str, ...],
|
||||
report: RunReport,
|
||||
) -> float:
|
||||
"""计算等待时间:从最早依赖完成到本任务开始。
|
||||
|
||||
无硬依赖、SKIPPED 任务或时间戳缺失时返回 0.0。
|
||||
"""
|
||||
if not deps or result.started_at is None or result.status == TaskStatus.SKIPPED:
|
||||
return 0.0
|
||||
# 找出所有已完成依赖的最晚完成时间
|
||||
dep_end_times: list[datetime] = []
|
||||
for dep in deps:
|
||||
dep_result = report.results.get(dep)
|
||||
if dep_result is not None and dep_result.finished_at is not None:
|
||||
dep_end_times.append(dep_result.finished_at)
|
||||
if not dep_end_times:
|
||||
return 0.0
|
||||
latest_dep_end = max(dep_end_times)
|
||||
delta = (result.started_at - latest_dep_end).total_seconds()
|
||||
return max(0.0, delta)
|
||||
|
||||
@staticmethod
|
||||
def _calc_total_duration(report: RunReport) -> float:
|
||||
"""计算 wall-clock 总耗时:最早开始到最晚结束。"""
|
||||
starts: list[datetime] = []
|
||||
ends: list[datetime] = []
|
||||
for r in report.results.values():
|
||||
if r.started_at is not None:
|
||||
starts.append(r.started_at)
|
||||
if r.finished_at is not None:
|
||||
ends.append(r.finished_at)
|
||||
if not starts or not ends:
|
||||
return 0.0
|
||||
return (max(ends) - min(starts)).total_seconds()
|
||||
|
||||
@staticmethod
|
||||
def _calc_critical_path(graph: Graph, report: RunReport) -> tuple[tuple[str, ...], float]:
|
||||
"""计算关键路径:DAG 最长路径(按实际执行耗时)。
|
||||
|
||||
使用拓扑排序 + 动态规划,O(V+E)。SKIPPED 任务耗时按 0 计。
|
||||
"""
|
||||
# 构建耗时映射
|
||||
durations: dict[str, float] = {}
|
||||
for name, result in report.results.items():
|
||||
durations[name] = result.duration or 0.0
|
||||
|
||||
# 拓扑序(使用 graph.layers 保证与分层一致)
|
||||
try:
|
||||
layers = graph.layers()
|
||||
except Exception:
|
||||
# 图校验失败时回退为空
|
||||
return (), 0.0
|
||||
|
||||
# earliest_finish[name] = duration[name] + max(earliest_finish[dep] for dep in deps)
|
||||
earliest_finish: dict[str, float] = {}
|
||||
predecessor: dict[str, str | None] = {}
|
||||
|
||||
for layer in layers:
|
||||
for name in layer:
|
||||
spec = graph.specs.get(name)
|
||||
deps = spec.depends_on if spec is not None else ()
|
||||
if not deps:
|
||||
earliest_finish[name] = durations.get(name, 0.0)
|
||||
predecessor[name] = None
|
||||
else:
|
||||
best_dep: str | None = None
|
||||
best_ef = 0.0
|
||||
for dep in deps:
|
||||
ef = earliest_finish.get(dep, 0.0)
|
||||
if ef >= best_ef:
|
||||
best_ef = ef
|
||||
best_dep = dep
|
||||
earliest_finish[name] = best_ef + durations.get(name, 0.0)
|
||||
predecessor[name] = best_dep
|
||||
|
||||
if not earliest_finish:
|
||||
return (), 0.0
|
||||
|
||||
# 找到 earliest_finish 最大的节点作为终点
|
||||
end_node = max(earliest_finish, key=lambda n: earliest_finish[n])
|
||||
total = earliest_finish[end_node]
|
||||
|
||||
# 回溯关键路径
|
||||
path: list[str] = []
|
||||
node: str | None = end_node
|
||||
while node is not None:
|
||||
path.append(node)
|
||||
node = predecessor.get(node)
|
||||
path.reverse()
|
||||
|
||||
return tuple(path), total
|
||||
|
||||
@staticmethod
|
||||
def _calc_parallelism(report: RunReport) -> tuple[float, int]:
|
||||
"""计算平均并行度与峰值并行度。
|
||||
|
||||
基于时间线扫描:将每个任务的 [started_at, finished_at] 区间
|
||||
转为事件点(+1/-1),排序后扫描得到瞬时并行度序列。
|
||||
|
||||
返回 (avg_parallelism, peak_parallelism)。
|
||||
无有效时间戳时返回 (0.0, 0)。
|
||||
"""
|
||||
events: list[tuple[float, int]] = [] # (timestamp, delta)
|
||||
for r in report.results.values():
|
||||
if r.started_at is None or r.finished_at is None:
|
||||
continue
|
||||
if r.status == TaskStatus.SKIPPED:
|
||||
continue
|
||||
start_ts = r.started_at.timestamp()
|
||||
end_ts = r.finished_at.timestamp()
|
||||
if end_ts <= start_ts:
|
||||
continue
|
||||
events.append((start_ts, 1))
|
||||
events.append((end_ts, -1))
|
||||
|
||||
if not events:
|
||||
return 0.0, 0
|
||||
|
||||
# 排序:同一时间点先处理结束(-1)再处理开始(+1),避免虚假峰值
|
||||
events.sort(key=lambda e: (e[0], e[1]))
|
||||
|
||||
current = 0
|
||||
peak = 0
|
||||
# 加权面积用于计算平均并行度
|
||||
area = 0.0
|
||||
prev_ts = events[0][0]
|
||||
for ts, delta in events:
|
||||
if ts > prev_ts:
|
||||
area += current * (ts - prev_ts)
|
||||
current += delta
|
||||
peak = max(peak, current)
|
||||
prev_ts = ts
|
||||
|
||||
total_span = events[-1][0] - events[0][0]
|
||||
avg = area / total_span if total_span > 0 else 0.0
|
||||
return avg, peak
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 查询
|
||||
# ------------------------------------------------------------------ #
|
||||
def task(self, name: str) -> TaskProfile:
|
||||
"""返回指定任务的剖面。不存在则 ``KeyError``。"""
|
||||
for t in self.tasks:
|
||||
if t.name == name:
|
||||
return t
|
||||
raise KeyError(name)
|
||||
|
||||
def top_bottlenecks(self, n: int = 5) -> tuple[TaskProfile, ...]:
|
||||
"""返回耗时最长的 Top-N 任务(按 duration 降序)。
|
||||
|
||||
参数
|
||||
----
|
||||
n:
|
||||
返回数量。``n <= 0`` 返回空元组。
|
||||
"""
|
||||
if n <= 0:
|
||||
return ()
|
||||
return tuple(sorted(self.tasks, key=lambda t: t.duration, reverse=True)[:n])
|
||||
|
||||
def critical_tasks(self) -> tuple[TaskProfile, ...]:
|
||||
"""返回关键路径上的所有任务(按路径顺序)。"""
|
||||
critical_set = set(self.critical_path)
|
||||
# 保持关键路径顺序
|
||||
order = {name: i for i, name in enumerate(self.critical_path)}
|
||||
return tuple(sorted((t for t in self.tasks if t.name in critical_set), key=lambda t: order[t.name]))
|
||||
|
||||
def failed_tasks(self) -> tuple[TaskProfile, ...]:
|
||||
"""返回 FAILED 状态的任务。"""
|
||||
return tuple(t for t in self.tasks if t.status == TaskStatus.FAILED)
|
||||
|
||||
def skipped_tasks(self) -> tuple[TaskProfile, ...]:
|
||||
"""返回 SKIPPED 状态的任务。"""
|
||||
return tuple(t for t in self.tasks if t.status == TaskStatus.SKIPPED)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 输出
|
||||
# ------------------------------------------------------------------ #
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""转为 JSON 友好的字典。"""
|
||||
return {
|
||||
"tasks": [t.to_dict() for t in self.tasks],
|
||||
"total_duration_seconds": round(self.total_duration, 6),
|
||||
"critical_path_duration_seconds": round(self.critical_path_duration, 6),
|
||||
"critical_path": list(self.critical_path),
|
||||
"avg_parallelism": round(self.avg_parallelism, 4),
|
||||
"peak_parallelism": self.peak_parallelism,
|
||||
"parallelism_efficiency": round(self.parallelism_efficiency, 4),
|
||||
"bottlenecks": [t.to_dict() for t in self.top_bottlenecks(5)],
|
||||
}
|
||||
|
||||
def to_html(self) -> str:
|
||||
"""生成自包含的 HTML 报告(含 CSS,无外部依赖)。
|
||||
|
||||
报告含:图级指标卡片、关键路径、时间线甘特图、Top 瓶颈表格、
|
||||
全部任务表格。适合直接用浏览器打开查看。
|
||||
"""
|
||||
return _render_html(self)
|
||||
|
||||
def describe(self) -> str:
|
||||
lines: list[str] = []
|
||||
lines.append("=" * 70)
|
||||
lines.append("PyFlowX 性能剖面报告")
|
||||
lines.append("=" * 70)
|
||||
lines.append("")
|
||||
lines.append("【图级指标】")
|
||||
lines.append(f" 总耗时 (wall-clock): {self.total_duration:.3f}s")
|
||||
lines.append(f" 关键路径耗时: {self.critical_path_duration:.3f}s")
|
||||
lines.append(f" 平均并行度: {self.avg_parallelism:.2f}")
|
||||
lines.append(f" 峰值并行度: {self.peak_parallelism}")
|
||||
lines.append(f" 并行度效率: {self.parallelism_efficiency:.2%}")
|
||||
lines.append(f" 任务总数: {len(self.tasks)}")
|
||||
lines.append("")
|
||||
|
||||
# 关键路径
|
||||
lines.append("【关键路径】")
|
||||
if self.critical_path:
|
||||
lines.append(f" {' -> '.join(self.critical_path)}")
|
||||
else:
|
||||
lines.append(" (无)")
|
||||
lines.append("")
|
||||
|
||||
# Top 瓶颈
|
||||
bottlenecks = self.top_bottlenecks(5)
|
||||
lines.append(f"【Top {len(bottlenecks)} 瓶颈任务】")
|
||||
if bottlenecks:
|
||||
lines.append(f" {'任务':<30} {'耗时':>10} {'等待':>10} {'尝试':>6} {'关键路径':>8} {'状态':>8}")
|
||||
lines.append(f" {'-' * 30} {'-' * 10} {'-' * 10} {'-' * 6} {'-' * 8} {'-' * 8}")
|
||||
for t in bottlenecks:
|
||||
critical_flag = "✓" if t.is_on_critical_path else ""
|
||||
lines.append(
|
||||
f" {t.name:<30} {t.duration:>9.3f}s {t.wait_time:>9.3f}s {t.attempts:>6} "
|
||||
f"{critical_flag:>8} {t.status.value:>8}",
|
||||
)
|
||||
else:
|
||||
lines.append(" (无)")
|
||||
lines.append("")
|
||||
|
||||
# 全部任务详情
|
||||
lines.append("【全部任务】")
|
||||
if self.tasks:
|
||||
lines.append(f" {'任务':<30} {'耗时':>10} {'等待':>10} {'尝试':>6} {'关键路径':>8} {'状态':>8}")
|
||||
lines.append(f" {'-' * 30} {'-' * 10} {'-' * 10} {'-' * 6} {'-' * 8} {'-' * 8}")
|
||||
for t in self.tasks:
|
||||
critical_flag = "✓" if t.is_on_critical_path else ""
|
||||
lines.append(
|
||||
f" {t.name:<30} {t.duration:>9.3f}s {t.wait_time:>9.3f}s {t.attempts:>6} "
|
||||
f"{critical_flag:>8} {t.status.value:>8}",
|
||||
)
|
||||
else:
|
||||
lines.append(" (无)")
|
||||
lines.append("")
|
||||
lines.append("=" * 70)
|
||||
return "\n".join(lines)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"ProfileReport(tasks={len(self.tasks)}, "
|
||||
f"total={self.total_duration:.3f}s, "
|
||||
f"critical={self.critical_path_duration:.3f}s, "
|
||||
f"avg_par={self.avg_parallelism:.2f}, "
|
||||
f"peak_par={self.peak_parallelism})"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# HTML 渲染(私有,零依赖)
|
||||
# ---------------------------------------------------------------------- #
|
||||
_HTML_TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>PyFlowX 性能剖面报告</title>
|
||||
<style>
|
||||
:root {{
|
||||
--bg: #f5f5f7;
|
||||
--card: #ffffff;
|
||||
--border: #d2d2d7;
|
||||
--text: #1d1d1f;
|
||||
--muted: #6e6e73;
|
||||
--accent: #0071e3;
|
||||
--success: #34c759;
|
||||
--warning: #ff9f0a;
|
||||
--danger: #ff3b30;
|
||||
--critical: #af52de;
|
||||
}}
|
||||
* {{ box-sizing: border-box; }}
|
||||
body {{
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
margin: 0;
|
||||
padding: 24px;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
line-height: 1.5;
|
||||
}}
|
||||
h1 {{ margin: 0 0 8px; font-size: 28px; }}
|
||||
h2 {{ margin: 32px 0 12px; font-size: 20px; border-bottom: 1px solid var(--border); padding-bottom: 6px; }}
|
||||
.subtitle {{ color: var(--muted); margin: 0 0 24px; font-size: 14px; }}
|
||||
.cards {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 12px; margin-bottom: 8px; }}
|
||||
.card {{
|
||||
background: var(--card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 10px;
|
||||
padding: 16px;
|
||||
}}
|
||||
.card .label {{ font-size: 12px; color: var(--muted); margin-bottom: 4px; text-transform: uppercase; letter-spacing: 0.5px; }}
|
||||
.card .value {{ font-size: 22px; font-weight: 600; }}
|
||||
.card .unit {{ font-size: 13px; color: var(--muted); margin-left: 2px; }}
|
||||
.critical-path {{
|
||||
background: var(--card);
|
||||
border: 1px solid var(--border);
|
||||
border-left: 4px solid var(--critical);
|
||||
border-radius: 10px;
|
||||
padding: 16px;
|
||||
margin-bottom: 8px;
|
||||
}}
|
||||
.critical-path .label {{ font-size: 12px; color: var(--muted); margin-bottom: 8px; text-transform: uppercase; letter-spacing: 0.5px; }}
|
||||
.critical-path .chain {{ font-family: ui-monospace, "SF Mono", Menlo, monospace; font-size: 13px; word-break: break-all; }}
|
||||
.critical-path .arrow {{ color: var(--critical); margin: 0 6px; font-weight: 600; }}
|
||||
/* 甘特图 */
|
||||
.gantt {{
|
||||
background: var(--card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 10px;
|
||||
padding: 16px;
|
||||
overflow-x: auto;
|
||||
}}
|
||||
.gantt-row {{ display: flex; align-items: center; margin-bottom: 6px; min-width: 600px; }}
|
||||
.gantt-label {{ width: 200px; flex-shrink: 0; font-size: 13px; font-family: ui-monospace, monospace; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }}
|
||||
.gantt-track {{ flex: 1; height: 22px; background: #f0f0f3; border-radius: 4px; position: relative; }}
|
||||
.gantt-bar {{ position: absolute; height: 100%; border-radius: 4px; min-width: 2px; }}
|
||||
.gantt-bar.success {{ background: var(--success); }}
|
||||
.gantt-bar.failed {{ background: var(--danger); }}
|
||||
.gantt-bar.skipped {{ background: var(--muted); }}
|
||||
.gantt-bar.critical {{ box-shadow: 0 0 0 2px var(--critical) inset; }}
|
||||
.gantt-bar:hover {{ opacity: 0.85; }}
|
||||
.gantt-tooltip {{ position: absolute; bottom: 100%; left: 50%; transform: translateX(-50%); background: #1d1d1f; color: #fff; padding: 4px 8px; border-radius: 4px; font-size: 11px; white-space: nowrap; opacity: 0; pointer-events: none; transition: opacity 0.15s; }}
|
||||
.gantt-bar:hover .gantt-tooltip {{ opacity: 1; }}
|
||||
/* 表格 */
|
||||
table {{ width: 100%; border-collapse: collapse; background: var(--card); border-radius: 10px; overflow: hidden; border: 1px solid var(--border); }}
|
||||
th, td {{ padding: 10px 12px; text-align: left; font-size: 13px; }}
|
||||
th {{ background: #fafafa; font-weight: 600; color: var(--muted); text-transform: uppercase; font-size: 11px; letter-spacing: 0.5px; }}
|
||||
tbody tr {{ border-top: 1px solid var(--border); }}
|
||||
tbody tr:hover {{ background: #fafafa; }}
|
||||
td.num {{ font-family: ui-monospace, monospace; text-align: right; }}
|
||||
.badge {{ display: inline-block; padding: 2px 8px; border-radius: 10px; font-size: 11px; font-weight: 500; }}
|
||||
.badge.success {{ background: rgba(52,199,89,0.15); color: var(--success); }}
|
||||
.badge.failed {{ background: rgba(255,59,48,0.15); color: var(--danger); }}
|
||||
.badge.skipped {{ background: rgba(110,110,115,0.15); color: var(--muted); }}
|
||||
.star {{ color: var(--critical); font-weight: 700; }}
|
||||
.footer {{ margin-top: 32px; color: var(--muted); font-size: 12px; text-align: center; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>PyFlowX 性能剖面报告</h1>
|
||||
<p class="subtitle">由 <code>pxp</code> 生成 · {generated_at}</p>
|
||||
|
||||
<h2>图级指标</h2>
|
||||
<div class="cards">
|
||||
<div class="card"><div class="label">总耗时</div><div class="value">{total_duration:.3f}<span class="unit">s</span></div></div>
|
||||
<div class="card"><div class="label">关键路径耗时</div><div class="value">{critical_duration:.3f}<span class="unit">s</span></div></div>
|
||||
<div class="card"><div class="label">平均并行度</div><div class="value">{avg_par:.2f}</div></div>
|
||||
<div class="card"><div class="label">峰值并行度</div><div class="value">{peak_par}</div></div>
|
||||
<div class="card"><div class="label">并行度效率</div><div class="value">{efficiency:.1f}<span class="unit">%</span></div></div>
|
||||
<div class="card"><div class="label">任务总数</div><div class="value">{task_count}</div></div>
|
||||
</div>
|
||||
|
||||
<h2>关键路径</h2>
|
||||
<div class="critical-path">
|
||||
<div class="label">最长依赖路径(串行瓶颈)</div>
|
||||
<div class="chain">{critical_chain}</div>
|
||||
</div>
|
||||
|
||||
<h2>任务时间线</h2>
|
||||
<div class="gantt">
|
||||
{gantt_rows}
|
||||
</div>
|
||||
|
||||
<h2>Top 瓶颈任务</h2>
|
||||
<table>
|
||||
<thead><tr><th>任务</th><th class="num">耗时</th><th class="num">等待</th><th class="num">尝试</th><th>关键路径</th><th>状态</th></tr></thead>
|
||||
<tbody>
|
||||
{bottleneck_rows}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<h2>全部任务</h2>
|
||||
<table>
|
||||
<thead><tr><th>任务</th><th class="num">耗时</th><th class="num">等待</th><th class="num">尝试</th><th>关键路径</th><th>状态</th><th>依赖</th></tr></thead>
|
||||
<tbody>
|
||||
{all_task_rows}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<div class="footer">由 PyFlowX · pxp 生成</div>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
def _status_badge(status: TaskStatus) -> str:
|
||||
"""生成状态徽章 HTML。"""
|
||||
cls = status.value
|
||||
return f'<span class="badge {cls}">{cls}</span>'
|
||||
|
||||
|
||||
def _format_critical_chain(path: tuple[str, ...]) -> str:
|
||||
"""格式化关键路径为 HTML 链。"""
|
||||
if not path:
|
||||
return '<em style="color:var(--muted)">(无)</em>'
|
||||
arrow = '<span class="arrow">→</span>'
|
||||
return arrow.join(f"<strong>{name}</strong>" for name in path)
|
||||
|
||||
|
||||
def _render_gantt(profile: ProfileReport) -> str:
|
||||
"""渲染甘特图行 HTML。
|
||||
|
||||
每个任务一行:标签 + 时间条。时间条位置基于 wait_time + 依赖关系
|
||||
重建相对开始时间(相对最早任务起点),归一化到 0-100% 宽度。
|
||||
SKIPPED 任务不显示(无时间戳)。
|
||||
"""
|
||||
visible = [t for t in profile.tasks if t.status != TaskStatus.SKIPPED and t.duration > 0]
|
||||
if not visible:
|
||||
return '<div style="color:var(--muted);padding:12px;">(无时间线数据)</div>'
|
||||
|
||||
# 重建相对开始时间:start[name] = max(end[dep]) + wait_time
|
||||
# profile.tasks 已是拓扑序,可直接按序计算
|
||||
start: dict[str, float] = {}
|
||||
end: dict[str, float] = {}
|
||||
for t in profile.tasks:
|
||||
if t.status == TaskStatus.SKIPPED:
|
||||
continue
|
||||
dep_end = 0.0
|
||||
for dep in t.deps:
|
||||
dep_end = max(dep_end, end.get(dep, 0.0))
|
||||
s = dep_end + t.wait_time
|
||||
start[t.name] = s
|
||||
end[t.name] = s + t.duration
|
||||
|
||||
# 归一化:以最早开始时间为 0,最晚结束为 100%
|
||||
min_start = min(start.get(t.name, 0.0) for t in visible)
|
||||
max_end = max(end.get(t.name, 0.0) for t in visible)
|
||||
span = max_end - min_start
|
||||
if span <= 0:
|
||||
span = 1.0
|
||||
|
||||
rows: list[str] = []
|
||||
for t in visible:
|
||||
s = start.get(t.name, 0.0) - min_start
|
||||
left_pct = (s / span) * 100
|
||||
width_pct = (t.duration / span) * 100
|
||||
cls = t.status.value
|
||||
critical_cls = " critical" if t.is_on_critical_path else ""
|
||||
tooltip = f"{t.name}: {t.duration:.3f}s @ +{s:.3f}s ({t.status.value})"
|
||||
rows.append(
|
||||
f' <div class="gantt-row">'
|
||||
f'<div class="gantt-label" title="{t.name}">{t.name}</div>'
|
||||
f'<div class="gantt-track">'
|
||||
f'<div class="gantt-bar {cls}{critical_cls}" style="left:{left_pct:.2f}%;width:{width_pct:.2f}%">'
|
||||
f'<span class="gantt-tooltip">{tooltip}</span>'
|
||||
f"</div></div></div>"
|
||||
)
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def _render_task_row(t: TaskProfile, show_deps: bool = False) -> str:
|
||||
"""渲染任务表格行 HTML。"""
|
||||
star = '<span class="star">★</span>' if t.is_on_critical_path else ""
|
||||
deps = ", ".join(t.deps) if show_deps and t.deps else ""
|
||||
deps_cell = f"<td>{deps}</td>" if show_deps else ""
|
||||
return (
|
||||
f" <tr>"
|
||||
f"<td><code>{t.name}</code></td>"
|
||||
f'<td class="num">{t.duration:.3f}s</td>'
|
||||
f'<td class="num">{t.wait_time:.3f}s</td>'
|
||||
f'<td class="num">{t.attempts}</td>'
|
||||
f"<td>{star}</td>"
|
||||
f"<td>{_status_badge(t.status)}</td>"
|
||||
f"{deps_cell}"
|
||||
f"</tr>"
|
||||
)
|
||||
|
||||
|
||||
def _render_html(profile: ProfileReport) -> str:
|
||||
"""渲染完整 HTML 报告。"""
|
||||
from datetime import datetime as _dt
|
||||
|
||||
bottlenecks = profile.top_bottlenecks(5)
|
||||
bottleneck_rows = (
|
||||
"\n".join(_render_task_row(t) for t in bottlenecks)
|
||||
or ' <tr><td colspan="6" style="color:var(--muted);">(无)</td></tr>'
|
||||
)
|
||||
all_task_rows = (
|
||||
"\n".join(_render_task_row(t, show_deps=True) for t in profile.tasks)
|
||||
or ' <tr><td colspan="7" style="color:var(--muted);">(无)</td></tr>'
|
||||
)
|
||||
|
||||
return _HTML_TEMPLATE.format(
|
||||
generated_at=_dt.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
total_duration=profile.total_duration,
|
||||
critical_duration=profile.critical_path_duration,
|
||||
avg_par=profile.avg_parallelism,
|
||||
peak_par=profile.peak_parallelism,
|
||||
efficiency=profile.parallelism_efficiency * 100,
|
||||
task_count=len(profile.tasks),
|
||||
critical_chain=_format_critical_chain(profile.critical_path),
|
||||
gantt_rows=_render_gantt(profile),
|
||||
bottleneck_rows=bottleneck_rows,
|
||||
all_task_rows=all_task_rows,
|
||||
)
|
||||
@@ -0,0 +1,159 @@
|
||||
"""函数注册表.
|
||||
|
||||
提供全局函数注册机制, 供 YAML 任务编排通过 ``fn`` 字段引用 Python 函数.
|
||||
|
||||
使用方式
|
||||
--------
|
||||
import pyflowx as px
|
||||
|
||||
@px.register_fn("pack_source")
|
||||
def pack_source(project_dir, output_dir):
|
||||
...
|
||||
|
||||
# YAML 中引用:
|
||||
# jobs:
|
||||
# pack:
|
||||
# fn: pack_source
|
||||
# args: ["./project", "./dist"]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from typing import Any, Callable, TypeVar, overload
|
||||
|
||||
if sys.version_info >= (3, 10):
|
||||
from typing import ParamSpec
|
||||
else:
|
||||
from typing_extensions import ParamSpec # pragma: no cover
|
||||
|
||||
__all__ = ["FnRegistry", "get_fn", "has_fn", "register_fn"]
|
||||
|
||||
P = ParamSpec("P")
|
||||
T = TypeVar("T")
|
||||
|
||||
_REGISTRY: dict[str, Callable[..., Any]] = {}
|
||||
|
||||
|
||||
@overload
|
||||
def register_fn(name: Callable[P, T]) -> Callable[P, T]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def register_fn(name: str | None = None) -> Callable[[Callable[P, T]], Callable[P, T]]: ...
|
||||
|
||||
|
||||
def register_fn(name: str | Callable[..., Any] | None = None) -> Callable[..., Any]:
|
||||
"""装饰器:将函数注册到全局 registry.
|
||||
|
||||
支持两种用法::
|
||||
|
||||
@register_fn # 使用函数 __name__ 作为注册名
|
||||
def my_func(): ...
|
||||
|
||||
@register_fn("custom") # 显式指定注册名
|
||||
def my_func(): ...
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str | Callable | None
|
||||
注册名或被装饰函数; 为 None 时使用函数 ``__name__``
|
||||
|
||||
Returns
|
||||
-------
|
||||
Callable
|
||||
装饰器函数或被装饰函数
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
名称已注册或无法推断函数名
|
||||
"""
|
||||
if callable(name):
|
||||
fn = name
|
||||
key = getattr(fn, "__name__", None)
|
||||
if key is None:
|
||||
raise ValueError("无法推断函数名, 请显式提供 name 参数")
|
||||
if key in _REGISTRY:
|
||||
raise ValueError(f"函数 {key!r} 已注册")
|
||||
_REGISTRY[key] = fn
|
||||
return fn
|
||||
|
||||
def decorator(fn: Callable[P, T]) -> Callable[P, T]:
|
||||
key = name if name is not None else getattr(fn, "__name__", None)
|
||||
if key is None:
|
||||
raise ValueError("无法推断函数名, 请显式提供 name 参数")
|
||||
if key in _REGISTRY:
|
||||
raise ValueError(f"函数 {key!r} 已注册")
|
||||
_REGISTRY[key] = fn
|
||||
return fn
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def get_fn(name: str) -> Callable[..., Any]:
|
||||
"""按名称获取已注册的函数.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
函数名
|
||||
|
||||
Returns
|
||||
-------
|
||||
Callable
|
||||
已注册的函数
|
||||
|
||||
Raises
|
||||
------
|
||||
KeyError
|
||||
函数未注册
|
||||
"""
|
||||
if name not in _REGISTRY:
|
||||
raise KeyError(f"函数 {name!r} 未注册")
|
||||
return _REGISTRY[name]
|
||||
|
||||
|
||||
def has_fn(name: str) -> bool:
|
||||
"""检查函数是否已注册.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
函数名
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
是否已注册
|
||||
"""
|
||||
return name in _REGISTRY
|
||||
|
||||
|
||||
class FnRegistry:
|
||||
"""函数注册表的面向对象访问接口."""
|
||||
|
||||
@staticmethod
|
||||
def register(name: str | None = None) -> Callable[[Callable[..., T]], Callable[..., T]]:
|
||||
"""注册装饰器, 等价于 :func:`register_fn`."""
|
||||
return register_fn(name)
|
||||
|
||||
@staticmethod
|
||||
def get(name: str) -> Callable[..., Any]:
|
||||
"""获取已注册函数, 等价于 :func:`get_fn`."""
|
||||
return get_fn(name)
|
||||
|
||||
@staticmethod
|
||||
def has(name: str) -> bool:
|
||||
"""检查是否已注册, 等价于 :func:`has_fn`."""
|
||||
return has_fn(name)
|
||||
|
||||
@staticmethod
|
||||
def clear() -> None:
|
||||
"""清空注册表."""
|
||||
_REGISTRY.clear()
|
||||
|
||||
@staticmethod
|
||||
def names() -> list[str]:
|
||||
"""返回所有已注册函数名."""
|
||||
return list(_REGISTRY.keys())
|
||||
@@ -69,6 +69,22 @@ class RunReport:
|
||||
"""以 FAILED 状态结束的任务名列表。"""
|
||||
return [name for name, r in self.results.items() if r.status == TaskStatus.FAILED]
|
||||
|
||||
def succeeded_tasks(self) -> list[str]:
|
||||
"""以 SUCCESS 状态结束的任务名列表。"""
|
||||
return [name for name, r in self.results.items() if r.status == TaskStatus.SUCCESS]
|
||||
|
||||
def skipped_tasks(self) -> list[str]:
|
||||
"""以 SKIPPED 状态结束的任务名列表。"""
|
||||
return [name for name, r in self.results.items() if r.status == TaskStatus.SKIPPED]
|
||||
|
||||
def tasks_by_status(self, status: TaskStatus) -> list[str]:
|
||||
"""返回指定状态的任务名列表。"""
|
||||
return [name for name, r in self.results.items() if r.status == status]
|
||||
|
||||
def durations(self) -> dict[str, float]:
|
||||
"""任务名 -> 执行时长(秒)。无时长记录的为 0.0。"""
|
||||
return {name: (r.duration or 0.0) for name, r in self.results.items()}
|
||||
|
||||
def describe(self) -> str:
|
||||
"""用于调试的人类可读多行报告。"""
|
||||
lines: list[str] = [f"RunReport(success={self.success})"]
|
||||
|
||||
+97
-215
@@ -14,9 +14,11 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import enum
|
||||
import sys
|
||||
from dataclasses import dataclass, field, replace
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Sequence, get_args
|
||||
|
||||
from .compose import GraphComposer
|
||||
from .errors import PyFlowXError
|
||||
from .executors import Strategy, run
|
||||
from .graph import Graph
|
||||
@@ -33,253 +35,137 @@ class CliExitCode(enum.IntEnum):
|
||||
INTERRUPTED = 130 # 与 POSIX 信号中断一致
|
||||
|
||||
|
||||
def _apply_verbose_to_graph(graph: Graph, verbose: bool) -> Graph:
|
||||
"""创建新图, 其中所有 TaskSpec 的 verbose 字段被设置为指定值.
|
||||
|
||||
使用 ``dataclasses.replace`` 在不可变的 TaskSpec 上创建带 verbose 标记的副本.
|
||||
依赖关系、标签等元数据全部保留.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph : Graph
|
||||
原始图.
|
||||
verbose : bool
|
||||
要设置的 verbose 值.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Graph
|
||||
所有 spec 的 verbose 字段已更新的新图.
|
||||
"""
|
||||
new_specs: list[TaskSpec[Any]] = []
|
||||
for spec in graph.all_specs().values():
|
||||
if spec.verbose == verbose:
|
||||
new_specs.append(spec)
|
||||
else:
|
||||
new_specs.append(replace(spec, verbose=verbose))
|
||||
return Graph.from_specs(new_specs)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@dataclass
|
||||
class CliRunner:
|
||||
"""命令行运行器: 根据用户输入执行对应的任务流图.
|
||||
|
||||
将命令名映射到 Graph 实例.
|
||||
通过 ``sys.argv`` 解析用户输入的命令, 执行对应的图.
|
||||
将命令别名映射到 Graph 实例. 通过 ``sys.argv`` 解析用户输入的命令,
|
||||
执行对应的图.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
aliases : dict[str, str | list[str] | Graph]
|
||||
命令别名到任务引用的映射. 每个值可以是:
|
||||
* ``str`` —— 单个任务名 (引用 ``tasks`` 中注册的任务),
|
||||
生成单任务图.
|
||||
* ``list[str]`` —— 任务名列表, 自动 :meth:`Graph.chain` 建立链式依赖,
|
||||
即后一个任务依赖前一个.
|
||||
* :class:`~pyflowx.graph.Graph` —— 直接使用该图 (用于复杂场景, 如
|
||||
自定义 ``conditions``、并行分支等).
|
||||
tasks : list[TaskSpec]
|
||||
扁平注册的任务列表. ``aliases`` 中的字符串引用这些任务名.
|
||||
未被任何 alias 引用的任务不会被执行.
|
||||
strategy : str | Strategy
|
||||
默认执行策略 (``Strategy.SEQUENTIAL`` / ``Strategy.THREAD`` /
|
||||
``Strategy.ASYNC`` 或对应字符串). 可被命令行 ``--strategy`` 覆盖.
|
||||
默认执行策略. 可被命令行 ``--strategy`` 覆盖.
|
||||
description : str
|
||||
CLI 帮助文本.
|
||||
verbose : bool
|
||||
是否显示详细执行过程. ``True`` 时打印任务生命周期和 subprocess 输出.
|
||||
默认 ``True``. 可被命令行 ``--quiet`` 关闭.
|
||||
**graphs : Graph
|
||||
命令名到图的映射. 每个 key 是一个命令名, value 是对应的
|
||||
:class:`~pyflowx.graph.Graph`.
|
||||
是否显示详细执行过程. 默认 ``True``, 可被命令行 ``--quiet`` 关闭.
|
||||
|
||||
Examples
|
||||
--------
|
||||
基本用法::
|
||||
简单场景 (tasks + aliases)::
|
||||
|
||||
runner = px.CliRunner(
|
||||
clean=px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("cargo_clean", cmd=["cargo", "clean"]),
|
||||
]
|
||||
),
|
||||
build=px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("uv_build", cmd=["uv", "build"]),
|
||||
]
|
||||
),
|
||||
tasks=[
|
||||
px.cmd(["uv", "build"]), # name="uv_build"
|
||||
px.cmd(["maturin", "build"], name="maturin_build"),
|
||||
px.cmd(["ruff", "check", "--fix"], name="lint"),
|
||||
],
|
||||
aliases={
|
||||
"b": "uv_build",
|
||||
"ba": ["uv_build", "maturin_build"], # chain: maturin 依赖 uv
|
||||
"lint": "lint",
|
||||
},
|
||||
)
|
||||
runner.run() # 解析 sys.argv
|
||||
runner.run()
|
||||
|
||||
指定策略与描述::
|
||||
复杂场景 (直接用 Graph)::
|
||||
|
||||
runner = px.CliRunner(
|
||||
strategy=px.Strategy.THREAD,
|
||||
aliases={
|
||||
"a": px.Graph.from_specs([
|
||||
px.TaskSpec("add", cmd=["git", "add", "."], conditions=(...)),
|
||||
px.TaskSpec("commit", cmd=["git", "commit"], depends_on=("add",)),
|
||||
]),
|
||||
},
|
||||
)
|
||||
runner.run(["test", "--strategy", "sequential"])
|
||||
"""
|
||||
|
||||
graphs: dict[str, Graph] = field(default_factory=dict)
|
||||
strategy: Strategy = field(default="sequential")
|
||||
aliases: dict[str, str | list[str | TaskSpec[Any]] | TaskSpec[Any] | Graph] = field(default_factory=dict)
|
||||
tasks: list[TaskSpec[Any]] = field(default_factory=list)
|
||||
strategy: Strategy = field(default="dependency")
|
||||
description: str = field(default_factory=str)
|
||||
verbose: bool = field(default_factory=lambda: True)
|
||||
# 解析后的命令→图映射,__post_init__ 填充
|
||||
graphs: dict[str, Graph] = field(default_factory=dict, init=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.graphs:
|
||||
raise ValueError("CliRunner 至少需要一个命令 (通过关键字参数提供)")
|
||||
if not self.aliases:
|
||||
raise ValueError("CliRunner 至少需要一个别名 (通过 aliases= 提供)")
|
||||
|
||||
# 解析并展开字符串引用
|
||||
self._resolve_graph_refs()
|
||||
# 1. 把 tasks 注册为虚拟命令图(每个 task 一个图),加入 raw_graphs
|
||||
# 使 GraphComposer 能解析对它们的字符串引用
|
||||
raw_graphs: dict[str, Graph] = {}
|
||||
for spec in self.tasks:
|
||||
if spec.name in raw_graphs:
|
||||
raise ValueError(f"任务名重复: {spec.name!r}")
|
||||
raw_graphs[spec.name] = Graph.from_specs([spec])
|
||||
|
||||
def _resolve_graph_refs(self) -> None:
|
||||
"""解析并展开图中的字符串引用.
|
||||
# 2. 把每个 alias 转为 Graph(alias 名可与 task 名相同,覆盖 task 注册)
|
||||
for alias, value in self.aliases.items():
|
||||
raw_graphs[alias] = self._alias_to_graph(alias, value)
|
||||
|
||||
支持两种引用格式:
|
||||
1. "command_name" - 引用整个命令图
|
||||
2. "command_name.task_name" - 引用特定任务
|
||||
# 3. 解析图间字符串引用(str / list[str] 引用其他 alias 或任务)
|
||||
self.graphs = GraphComposer(raw_graphs).resolve_all()
|
||||
|
||||
递归解析所有引用,直到所有图都只包含TaskSpec对象。
|
||||
@staticmethod
|
||||
def _alias_to_graph(
|
||||
alias: str,
|
||||
value: str | list[str | TaskSpec[Any]] | TaskSpec[Any] | Graph,
|
||||
) -> Graph:
|
||||
"""把 alias 的值转换为 Graph.
|
||||
|
||||
* ``str`` —— 对其他 alias 或已注册任务名的引用, 由 GraphComposer 展开.
|
||||
* ``TaskSpec`` —— 单个内联任务, 生成单任务图.
|
||||
* ``list[str | TaskSpec]`` —— 引用/任务混合列表, GraphComposer 展开时
|
||||
自动让后续引用依赖前面 (chain 语义). 元素为 alias 名、任务名或
|
||||
:class:`TaskSpec` 对象 (内联任务).
|
||||
* ``Graph`` —— 原样返回 (用于复杂场景: conditions、并行分支等).
|
||||
"""
|
||||
resolved_graphs: dict[str, Graph] = {}
|
||||
|
||||
for cmd_name, graph in self.graphs.items():
|
||||
resolved_graph = self._expand_refs(graph, cmd_name)
|
||||
resolved_graphs[cmd_name] = resolved_graph
|
||||
|
||||
# 更新graphs字典
|
||||
object.__setattr__(self, "graphs", resolved_graphs)
|
||||
|
||||
def _expand_refs(self, graph: Graph, current_cmd: str) -> Graph:
|
||||
"""展开图中的字符串引用.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph : Graph
|
||||
包含可能的字符串引用的图
|
||||
current_cmd : str
|
||||
当前命令名(用于避免循环引用)
|
||||
|
||||
Returns
|
||||
-------
|
||||
Graph
|
||||
展开后的图,只包含TaskSpec对象
|
||||
|
||||
Note
|
||||
-----
|
||||
引用按顺序展开,后续引用的任务依赖于前面引用的任务完成。
|
||||
例如:["c", "tc", bump] 会展开为:
|
||||
- c的所有任务(无依赖)
|
||||
- tc的所有任务(依赖于c的最后一个任务)
|
||||
- bump任务(依赖于tc的最后一个任务)
|
||||
"""
|
||||
# 检查是否有待解析的引用
|
||||
pending_refs = getattr(graph, "_pending_refs", None)
|
||||
if not pending_refs:
|
||||
return graph
|
||||
|
||||
# 收集所有TaskSpec(按正确顺序:先引用,后原始TaskSpec)
|
||||
all_specs: list[TaskSpec[Any]] = []
|
||||
|
||||
# 记录每个引用展开后的所有任务名,用于建立依赖链
|
||||
previous_ref_last_task: str | None = None
|
||||
|
||||
# 先解析每个引用,并建立依赖关系
|
||||
for ref in pending_refs:
|
||||
expanded_specs = self._parse_ref(ref, current_cmd)
|
||||
|
||||
# 如果有前面的引用,让当前引用的所有任务依赖于前面引用的最后一个任务
|
||||
if previous_ref_last_task and expanded_specs:
|
||||
# 为当前引用的每个任务添加依赖
|
||||
for i, task in enumerate(expanded_specs):
|
||||
# 只为没有依赖的任务添加依赖,或者为第一个任务添加依赖
|
||||
if i == 0 or not task.depends_on:
|
||||
updated_task = replace(task, depends_on=tuple({*task.depends_on, previous_ref_last_task}))
|
||||
expanded_specs[i] = updated_task
|
||||
|
||||
# 记录当前引用的最后一个任务名
|
||||
if expanded_specs:
|
||||
previous_ref_last_task = expanded_specs[-1].name
|
||||
|
||||
all_specs.extend(expanded_specs)
|
||||
|
||||
# 然后添加原始图中的TaskSpec,并让它们按顺序执行
|
||||
original_specs = list(graph.all_specs().values())
|
||||
if original_specs:
|
||||
# 第一个原始TaskSpec依赖于最后一个引用的任务
|
||||
if previous_ref_last_task:
|
||||
first_original = original_specs[0]
|
||||
updated_first = replace(
|
||||
first_original, depends_on=tuple({*first_original.depends_on, previous_ref_last_task})
|
||||
)
|
||||
all_specs.append(updated_first)
|
||||
else:
|
||||
# 如果没有引用,直接添加第一个原始TaskSpec
|
||||
all_specs.append(original_specs[0])
|
||||
|
||||
# 后续的原始TaskSpec依赖于前一个原始TaskSpec
|
||||
for i in range(1, len(original_specs)):
|
||||
current_task = original_specs[i]
|
||||
previous_task_name = original_specs[i - 1].name
|
||||
# 更新依赖,确保顺序执行
|
||||
updated_task = replace(current_task, depends_on=tuple({*current_task.depends_on, previous_task_name}))
|
||||
all_specs.append(updated_task)
|
||||
|
||||
# 创建新的图(不包含引用)
|
||||
return Graph.from_specs(all_specs)
|
||||
|
||||
def _parse_ref(self, ref: str, current_cmd: str) -> list[TaskSpec[Any]]:
|
||||
"""解析单个字符串引用.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
ref : str
|
||||
引用字符串(如"tc"或"tc.lint")
|
||||
current_cmd : str
|
||||
当前命令名(用于避免循环引用)
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[TaskSpec[Any]]
|
||||
解析后的TaskSpec列表
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
如果引用无效或存在循环引用
|
||||
"""
|
||||
# 避免循环引用
|
||||
if ref == current_cmd:
|
||||
raise ValueError(f"循环引用: 命令 '{current_cmd}' 引用了自己")
|
||||
|
||||
# 解析引用格式
|
||||
if "." in ref:
|
||||
# 特定任务引用: "command_name.task_name"
|
||||
cmd_name, task_name = ref.split(".", 1)
|
||||
if cmd_name not in self.graphs:
|
||||
raise ValueError(f"引用的命令 '{cmd_name}' 不存在")
|
||||
|
||||
# 获取特定任务
|
||||
ref_graph = self.graphs[cmd_name]
|
||||
if task_name not in ref_graph.all_specs():
|
||||
raise ValueError(f"任务 '{task_name}' 不存在于命令 '{cmd_name}' 中")
|
||||
|
||||
return [ref_graph.all_specs()[task_name]]
|
||||
else:
|
||||
# 整个命令图引用: "command_name"
|
||||
cmd_name = ref
|
||||
if cmd_name not in self.graphs:
|
||||
raise ValueError(f"引用的命令 '{cmd_name}' 不存在")
|
||||
|
||||
# 获取整个图的所有任务
|
||||
ref_graph = self.graphs[cmd_name]
|
||||
|
||||
# 递归展开引用(如果引用的图也有引用)
|
||||
ref_graph = self._expand_refs(ref_graph, cmd_name)
|
||||
|
||||
return list(ref_graph.all_specs().values())
|
||||
if isinstance(value, Graph):
|
||||
return value
|
||||
if isinstance(value, TaskSpec):
|
||||
return Graph.from_specs([value])
|
||||
if isinstance(value, str):
|
||||
# 字符串引用,用 _pending_refs 占位,GraphComposer 后续展开
|
||||
return Graph.from_specs([value]) # type: ignore[arg-type]
|
||||
if isinstance(value, list):
|
||||
if not value:
|
||||
raise ValueError(f"别名 {alias!r} 的任务列表为空")
|
||||
for item in value:
|
||||
if not isinstance(item, (str, TaskSpec)):
|
||||
raise TypeError(f"别名 {alias!r} 的列表元素类型无效: {type(item).__name__}, 预期 str 或 TaskSpec")
|
||||
# str/TaskSpec 混合列表,由 GraphComposer 展开(自动建立 chain 依赖)
|
||||
return Graph.from_specs(value)
|
||||
raise TypeError(
|
||||
f"别名 {alias!r} 的值类型无效: {type(value).__name__}, 预期 str/TaskSpec/list[str|TaskSpec]/Graph"
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 内省
|
||||
# ------------------------------------------------------------------ #
|
||||
@property
|
||||
def commands(self) -> list[str]:
|
||||
"""可用的命令列表 (按插入顺序)."""
|
||||
return list(self.graphs.keys())
|
||||
"""可用的命令列表 (按 aliases 定义顺序, 不含 tasks 中未引用的任务)."""
|
||||
return list(self.aliases.keys())
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 参数解析
|
||||
# ------------------------------------------------------------------ #
|
||||
def _prog_name(self) -> str:
|
||||
"""从 sys.argv[0] 推导程序名."""
|
||||
import os
|
||||
|
||||
return os.path.basename(sys.argv[0]) if sys.argv else "pyflowx"
|
||||
return Path(sys.argv[0]).name if sys.argv else "pyflowx"
|
||||
|
||||
def create_parser(self) -> argparse.ArgumentParser:
|
||||
"""创建参数解析器.
|
||||
@@ -365,9 +251,9 @@ class CliRunner:
|
||||
parser.print_help()
|
||||
return CliExitCode.FAILURE.value
|
||||
|
||||
# 验证命令
|
||||
if parsed.command not in self.graphs:
|
||||
available = ", ".join(self.graphs.keys())
|
||||
# 验证命令(必须是已注册的 alias,不接受裸任务名)
|
||||
if parsed.command not in self.aliases:
|
||||
available = ", ".join(self.commands)
|
||||
print(
|
||||
f"错误: 未知命令 {parsed.command!r} (可用命令: {available})",
|
||||
file=sys.stderr,
|
||||
@@ -377,12 +263,8 @@ class CliRunner:
|
||||
# 确定是否 verbose: --quiet 覆盖默认值
|
||||
verbose = self.verbose and not parsed.quiet
|
||||
|
||||
# 对图应用 verbose 设置 (重建带 verbose 标记的 spec)
|
||||
# 执行对应的图 (verbose 标记由 run() 统一应用到各 spec)
|
||||
graph = self.graphs[parsed.command]
|
||||
if verbose:
|
||||
graph = _apply_verbose_to_graph(graph, verbose=True)
|
||||
|
||||
# 执行对应的图
|
||||
try:
|
||||
report = run(
|
||||
graph,
|
||||
|
||||
+197
-47
@@ -4,83 +4,197 @@
|
||||
执行器向后端查询某任务是否已有存储结果;若有则跳过该任务,并将其
|
||||
存储值注入下游任务。
|
||||
|
||||
本模块刻意保持最小化:仅持久化*成功*结果(失败任务会重跑),存储
|
||||
形态为扁平的 ``{task_name: result}`` 映射。内置两个后端:
|
||||
存储键由 :meth:`TaskSpec.storage_key` 计算,默认为任务名;若任务配置
|
||||
了 ``cache_key``,则键为 ``"name:cache_key_value"``,使不同输入产生
|
||||
独立缓存条目。
|
||||
|
||||
* :class:`MemoryBackend` —— 快速、进程内、无 I/O。默认。
|
||||
* :class:`JSONBackend` —— 持久化到 JSON 文件,支持跨进程续跑。
|
||||
|
||||
两者均零依赖(``json`` 为标准库)。用户可子类化
|
||||
:class:`StateBackend` 接入 SQLite、Redis 等。
|
||||
支持 TTL:``has`` 在条目过期时返回 ``False``。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
from typing import Any, ContextManager, Mapping
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override
|
||||
else:
|
||||
from typing_extensions import override # pragma: no cover
|
||||
|
||||
from .errors import StorageError
|
||||
|
||||
|
||||
class StateBackend(ABC):
|
||||
"""可续跑状态存储的抽象基类。"""
|
||||
"""可续跑状态存储的抽象基类。
|
||||
|
||||
所有方法以 ``key`` 为参数(通常为任务名或 ``name:cache_key``)。
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def load(self) -> Mapping[str, Any]:
|
||||
"""返回完整的存储映射(可能为空)。"""
|
||||
|
||||
@abstractmethod
|
||||
def save(self, name: str, value: Any) -> None:
|
||||
def save(self, key: str, value: Any) -> None:
|
||||
"""持久化单个任务的成功结果。"""
|
||||
|
||||
@abstractmethod
|
||||
def has(self, name: str) -> bool:
|
||||
"""``name`` 是否已有存储结果。"""
|
||||
def has(self, key: str) -> bool:
|
||||
"""``key`` 是否已有未过期的存储结果。"""
|
||||
|
||||
@abstractmethod
|
||||
def get(self, name: str) -> Any:
|
||||
"""返回 ``name`` 的存储结果(不存在则抛 ``KeyError``)。"""
|
||||
def get(self, key: str) -> Any:
|
||||
"""返回 ``key`` 的存储结果(不存在则抛 ``KeyError``)。"""
|
||||
|
||||
@abstractmethod
|
||||
def clear(self) -> None:
|
||||
"""清除所有存储状态。"""
|
||||
|
||||
def flush(self) -> None: # noqa: B027
|
||||
"""将内存中暂存的状态持久化到外部介质。
|
||||
|
||||
class MemoryBackend(StateBackend):
|
||||
"""进程内 dict 后端。进程退出即丢失。"""
|
||||
默认无操作(如 :class:`MemoryBackend` 无需落盘)。
|
||||
:class:`JSONBackend` 在 :meth:`batch` 期间会延迟落盘,需在退出时调用。
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._store: dict[str, Any] = {}
|
||||
def batch(self) -> ContextManager[None]:
|
||||
"""返回一个上下文管理器,期间 :meth:`save` 可延迟 :meth:`flush`。
|
||||
|
||||
默认实现为 no-op(如 :class:`MemoryBackend`)。:class:`JSONBackend`
|
||||
覆盖为:进入时标记延迟,退出时统一 flush 一次,将每任务一次落盘
|
||||
(N 次写入)降为整次运行一次(O(N) 而非 O(N²))。
|
||||
"""
|
||||
return nullcontext()
|
||||
|
||||
|
||||
class _TTLStateBackendMixin(StateBackend):
|
||||
"""TTL 状态后端共享逻辑。
|
||||
|
||||
将 ``has`` / ``get`` / ``load`` / ``save`` / ``clear`` 的统一实现
|
||||
委托给四个原始存取原语::meth:`_get_raw`、:meth:`_put_raw`、
|
||||
:meth:`_iter_raw`、:meth:`_clear_raw`,并基于 :meth:`_now` 与
|
||||
``self._ttl`` 提供统一的过期判断 :meth:`_is_expired`。
|
||||
|
||||
子类需设置 ``self._ttl`` 并实现上述四个原语;如需自定义时间源
|
||||
(如 ``time.monotonic``)可覆盖 :meth:`_now`。
|
||||
"""
|
||||
|
||||
_ttl: float | None
|
||||
|
||||
# ---- 原语:由子类实现 ---- #
|
||||
@abstractmethod
|
||||
def _get_raw(self, key: str) -> tuple[Any, float] | None:
|
||||
"""返回 ``(value, ts)``;键不存在时返回 ``None``。"""
|
||||
|
||||
@abstractmethod
|
||||
def _put_raw(self, key: str, value: Any, ts: float) -> None:
|
||||
"""写入一条记录。"""
|
||||
|
||||
@abstractmethod
|
||||
def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
|
||||
"""迭代所有记录(不做过期过滤),yield ``(key, value, ts)``。"""
|
||||
|
||||
@abstractmethod
|
||||
def _clear_raw(self) -> None:
|
||||
"""清空所有记录。"""
|
||||
|
||||
# ---- 共享实现 ---- #
|
||||
def _now(self) -> float:
|
||||
"""当前时间戳,默认为 wall-clock 秒。"""
|
||||
return time.time()
|
||||
|
||||
def _is_expired(self, ts: float) -> bool:
|
||||
"""时间戳 ``ts`` 是否已过期。"""
|
||||
if self._ttl is None:
|
||||
return False
|
||||
return (self._now() - ts) > self._ttl
|
||||
|
||||
@override
|
||||
def load(self) -> Mapping[str, Any]:
|
||||
return dict(self._store)
|
||||
return {k: v for k, v, ts in self._iter_raw() if not self._is_expired(ts)}
|
||||
|
||||
def save(self, name: str, value: Any) -> None:
|
||||
self._store[name] = value
|
||||
@override
|
||||
def save(self, key: str, value: Any) -> None:
|
||||
self._put_raw(key, value, self._now())
|
||||
|
||||
def has(self, name: str) -> bool:
|
||||
return name in self._store
|
||||
@override
|
||||
def has(self, key: str) -> bool:
|
||||
entry = self._get_raw(key)
|
||||
return entry is not None and not self._is_expired(entry[1])
|
||||
|
||||
def get(self, name: str) -> Any:
|
||||
return self._store[name]
|
||||
@override
|
||||
def get(self, key: str) -> Any:
|
||||
entry = self._get_raw(key)
|
||||
if entry is None or self._is_expired(entry[1]):
|
||||
raise KeyError(key)
|
||||
return entry[0]
|
||||
|
||||
@override
|
||||
def clear(self) -> None:
|
||||
self._clear_raw()
|
||||
|
||||
|
||||
class MemoryBackend(_TTLStateBackendMixin):
|
||||
"""进程内 dict 后端。进程退出即丢失。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
ttl:
|
||||
条目存活秒数。``None`` 表示永不过期。``has`` 在条目超过 ttl 后
|
||||
返回 ``False``(但不主动删除,下次 ``save`` 覆盖)。
|
||||
"""
|
||||
|
||||
def __init__(self, ttl: float | None = None) -> None:
|
||||
self._store: dict[str, tuple[Any, float]] = {}
|
||||
self._ttl = ttl
|
||||
|
||||
@override
|
||||
def _now(self) -> float:
|
||||
return time.monotonic()
|
||||
|
||||
@override
|
||||
def _get_raw(self, key: str) -> tuple[Any, float] | None:
|
||||
return self._store.get(key)
|
||||
|
||||
@override
|
||||
def _put_raw(self, key: str, value: Any, ts: float) -> None:
|
||||
self._store[key] = (value, ts)
|
||||
|
||||
@override
|
||||
def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
|
||||
for k, (v, ts) in self._store.items():
|
||||
yield k, v, ts
|
||||
|
||||
@override
|
||||
def _clear_raw(self) -> None:
|
||||
self._store.clear()
|
||||
|
||||
|
||||
class JSONBackend(StateBackend):
|
||||
class JSONBackend(_TTLStateBackendMixin):
|
||||
"""基于文件的 JSON 存储,用于跨进程续跑。
|
||||
|
||||
结果必须可 JSON 序列化。不可序列化的值会抛出
|
||||
:class:`~pyflowx.errors.StorageError`(运行本身不会中止;仅该条
|
||||
结果的持久化失败)。
|
||||
存储格式:``{key: {"value": v, "ts": epoch_seconds}}``。
|
||||
``ts`` 用于 TTL 判断。结果必须可 JSON 序列化。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path:
|
||||
JSON 文件路径。
|
||||
ttl:
|
||||
条目存活秒数。``None`` 表示永不过期。
|
||||
"""
|
||||
|
||||
def __init__(self, path: str) -> None:
|
||||
def __init__(self, path: str, ttl: float | None = None) -> None:
|
||||
self._path: str = path
|
||||
self._store: dict[str, Any] = {}
|
||||
self._ttl = ttl
|
||||
self._store: dict[str, dict[str, Any]] = {}
|
||||
self._defer_flush: bool = False
|
||||
self._load()
|
||||
|
||||
def _load(self) -> None:
|
||||
@@ -90,7 +204,13 @@ class JSONBackend(StateBackend):
|
||||
with open(self._path, encoding="utf-8") as fh:
|
||||
data: Any = json.load(fh)
|
||||
if isinstance(data, dict):
|
||||
self._store = data
|
||||
# 兼容纯值格式与带元数据格式
|
||||
self._store = {}
|
||||
for k, v in data.items():
|
||||
if isinstance(v, dict) and "value" in v and "ts" in v:
|
||||
self._store[k] = v
|
||||
else:
|
||||
self._store[k] = {"value": v, "ts": time.time()}
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise StorageError(f"cannot read state file {self._path!r}", exc) from exc
|
||||
|
||||
@@ -99,32 +219,62 @@ class JSONBackend(StateBackend):
|
||||
try:
|
||||
with open(tmp, "w", encoding="utf-8") as fh:
|
||||
json.dump(self._store, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
_ = Path(tmp).replace(Path(self._path))
|
||||
except (OSError, TypeError) as exc:
|
||||
raise StorageError(f"cannot write state file {self._path!r}", exc) from exc
|
||||
|
||||
def load(self) -> Mapping[str, Any]:
|
||||
return dict(self._store)
|
||||
@override
|
||||
def _get_raw(self, key: str) -> tuple[Any, float] | None:
|
||||
entry = self._store.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
return entry["value"], float(entry.get("ts", 0))
|
||||
|
||||
def save(self, name: str, value: Any) -> None:
|
||||
# 在修改内存状态前先校验可序列化性。
|
||||
@override
|
||||
def _put_raw(self, key: str, value: Any, ts: float) -> None:
|
||||
self._store[key] = {"value": value, "ts": ts}
|
||||
|
||||
@override
|
||||
def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
|
||||
for k, entry in self._store.items():
|
||||
yield k, entry["value"], float(entry.get("ts", 0))
|
||||
|
||||
@override
|
||||
def _clear_raw(self) -> None:
|
||||
self._store.clear()
|
||||
|
||||
@override
|
||||
def clear(self) -> None:
|
||||
super().clear()
|
||||
self._flush()
|
||||
|
||||
@override
|
||||
def save(self, key: str, value: Any) -> None:
|
||||
try:
|
||||
_ = json.dumps(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise StorageError(f"result of task {name!r} is not JSON-serialisable", exc) from exc
|
||||
self._store[name] = value
|
||||
raise StorageError(f"result of key {key!r} is not JSON-serialisable", exc) from exc
|
||||
super().save(key, value)
|
||||
if not self._defer_flush:
|
||||
self._flush()
|
||||
|
||||
@override
|
||||
def flush(self) -> None:
|
||||
self._flush()
|
||||
|
||||
def has(self, name: str) -> bool:
|
||||
return name in self._store
|
||||
@override
|
||||
@contextmanager
|
||||
def batch(self) -> Iterator[None]:
|
||||
"""进入批量模式:``save`` 暂不落盘,退出时统一 flush 一次。
|
||||
|
||||
def get(self, name: str) -> Any:
|
||||
return self._store[name]
|
||||
|
||||
def clear(self) -> None:
|
||||
self._store.clear()
|
||||
self._flush()
|
||||
将整次运行 N 个任务的 N 次全量落盘降为 1 次。
|
||||
"""
|
||||
self._defer_flush = True
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._defer_flush = False
|
||||
self._flush()
|
||||
|
||||
|
||||
def resolve_backend(backend: StateBackend | None) -> StateBackend:
|
||||
|
||||
+479
-183
@@ -15,25 +15,38 @@
|
||||
* ``TaskStatus`` 是封闭枚举;执行器绝不发明临时字符串。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
ContextManager,
|
||||
Coroutine,
|
||||
Generator,
|
||||
Generic,
|
||||
List,
|
||||
Mapping,
|
||||
Optional,
|
||||
Tuple,
|
||||
TypeVar,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
T = TypeVar("T")
|
||||
if sys.version_info >= (3, 13):
|
||||
from typing import TypeVar
|
||||
else:
|
||||
from typing_extensions import TypeVar # pragma: no cover
|
||||
|
||||
T = TypeVar("T", default=Any)
|
||||
|
||||
# 任务可调用对象可以是同步或异步的。显式保留联合类型,让 mypy 理解两种形态。
|
||||
TaskFn = Union[
|
||||
@@ -52,8 +65,104 @@ TaskCmd = Union[
|
||||
Callable[..., Any], # Python 函数
|
||||
]
|
||||
|
||||
# 条件判断函数类型
|
||||
Condition = Callable[[], bool]
|
||||
# 执行策略:sequential/thread/async 为层屏障模型,dependency 为依赖驱动模型。
|
||||
Strategy = Union[str, "StrategyKind"]
|
||||
StrategyKind = Any # 占位,避免循环;executors 模块用 Literal 约束
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 条件判断函数类型:接收依赖上下文(可能为空映射),返回是否应执行。
|
||||
Condition = Callable[[Context], bool]
|
||||
|
||||
# 缓存键计算函数:基于依赖上下文计算稳定字符串键。
|
||||
CacheKeyFn = Callable[[Context], str]
|
||||
|
||||
|
||||
def _format_skip_reason(failed_conditions: list[str]) -> str:
|
||||
"""格式化跳过原因:≤2 个全展示,>2 个仅展示前 2 个并附总数。"""
|
||||
if len(failed_conditions) <= 2:
|
||||
return f"条件不满足: {', '.join(failed_conditions)}"
|
||||
return f"条件不满足: {', '.join(failed_conditions[:2])} 等{len(failed_conditions)}个条件"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 重试策略
|
||||
# ---------------------------------------------------------------------- #
|
||||
@dataclass(frozen=True)
|
||||
class RetryPolicy:
|
||||
"""任务失败重试策略。
|
||||
|
||||
参数
|
||||
----
|
||||
max_attempts:
|
||||
最大尝试次数(含首次)。``1`` 表示仅尝试一次,不重试。
|
||||
delay:
|
||||
两次尝试之间的初始等待秒数。
|
||||
backoff:
|
||||
退避倍率。第 n 次重试等待 ``delay * backoff ** (n-1)``。
|
||||
jitter:
|
||||
抖动上限秒数。每次等待加上 ``[0, jitter)`` 的随机量,避免惊群。
|
||||
retry_on:
|
||||
仅对这些异常类型重试。默认 ``(Exception,)`` 重试所有异常。
|
||||
传入空元组等价于不重试。
|
||||
|
||||
Note
|
||||
-----
|
||||
替代旧版 ``retries: int``。``retries=2`` 等价于
|
||||
``RetryPolicy(max_attempts=3)``。
|
||||
"""
|
||||
|
||||
max_attempts: int = 1
|
||||
delay: float = 0.0
|
||||
backoff: float = 1.0
|
||||
jitter: float = 0.0
|
||||
retry_on: tuple[type[BaseException], ...] = (Exception,)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.max_attempts < 1:
|
||||
raise ValueError(f"RetryPolicy.max_attempts must be >= 1, got {self.max_attempts}.")
|
||||
if self.delay < 0:
|
||||
raise ValueError(f"RetryPolicy.delay must be >= 0, got {self.delay}.")
|
||||
if self.backoff < 0:
|
||||
raise ValueError(f"RetryPolicy.backoff must be >= 0, got {self.backoff}.")
|
||||
if self.jitter < 0:
|
||||
raise ValueError(f"RetryPolicy.jitter must be >= 0, got {self.jitter}.")
|
||||
|
||||
@property
|
||||
def retries(self) -> int:
|
||||
"""重试次数(不含首次),等价于 ``max_attempts - 1``。"""
|
||||
return self.max_attempts - 1
|
||||
|
||||
def should_retry(self, exc: BaseException) -> bool:
|
||||
"""异常是否属于可重试类型。"""
|
||||
return isinstance(exc, self.retry_on)
|
||||
|
||||
def wait_seconds(self, attempt: int) -> float:
|
||||
"""第 ``attempt`` 次失败后应等待的秒数(attempt 从 1 开始)。"""
|
||||
if attempt < 1:
|
||||
return 0.0
|
||||
import random
|
||||
|
||||
base = self.delay * (self.backoff ** max(0, attempt - 1))
|
||||
jitter = random.uniform(0, self.jitter) if self.jitter > 0 else 0.0
|
||||
return base + jitter
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 任务钩子
|
||||
# ---------------------------------------------------------------------- #
|
||||
@dataclass(frozen=True)
|
||||
class TaskHooks:
|
||||
"""任务生命周期钩子。
|
||||
|
||||
所有钩子均为可选。``pre_run`` 在任务实际执行前调用;``post_run``
|
||||
在成功后调用并接收返回值;``on_failure`` 在最终失败后调用并接收异常。
|
||||
钩子异常不会影响任务状态,仅记录日志。
|
||||
"""
|
||||
|
||||
pre_run: Callable[[TaskSpec[Any]], None] | None = None
|
||||
post_run: Callable[[TaskSpec[Any], Any], None] | None = None
|
||||
on_failure: Callable[[TaskSpec[Any], BaseException], None] | None = None
|
||||
|
||||
|
||||
class TaskStatus(Enum):
|
||||
@@ -83,235 +192,426 @@ class TaskSpec(Generic[T]):
|
||||
- ``list[str]``: 命令及参数列表,如 ``["ls", "-la"]``
|
||||
- ``str``: shell 命令字符串,如 ``"pip freeze > requirements.txt"``
|
||||
- ``Callable``: Python 函数,与 ``fn`` 参数等效
|
||||
若提供此参数,会自动包装为执行函数,覆盖 ``fn`` 参数。
|
||||
depends_on:
|
||||
必须先完成才能运行本任务的任务名列表。顺序无关;框架会做
|
||||
拓扑排序。
|
||||
硬依赖任务名。必须全部成功完成才会运行本任务。
|
||||
上游被 SKIPPED 时,本任务也会被 SKIPPED(除非
|
||||
``allow_upstream_skip=True``)。
|
||||
soft_depends_on:
|
||||
软依赖任务名。会等待其完成,但其结果不影响本任务是否执行:
|
||||
- 上游成功:注入其返回值
|
||||
- 上游 SKIPPED 或失败:注入 :attr:`defaults` 中提供的默认值
|
||||
适用于"可选输入"场景。
|
||||
defaults:
|
||||
软依赖的默认值映射 ``{dep_name: default_value}``。
|
||||
软依赖未提供结果时使用。未在 defaults 中出现的软依赖默认为 ``None``。
|
||||
args:
|
||||
静态位置参数,追加在注入参数*之后*。适用于参数化任务
|
||||
(如 ``fetch_user(uid)``)。
|
||||
静态位置参数,追加在注入参数*之后*。
|
||||
kwargs:
|
||||
静态关键字参数。若与注入名冲突则抛出
|
||||
:class:`~pyflowx.errors.InjectionError`。
|
||||
retries:
|
||||
失败后的重试次数。``0`` 表示仅尝试一次。
|
||||
retry:
|
||||
:class:`RetryPolicy` 重试策略。默认仅尝试一次。
|
||||
timeout:
|
||||
最大执行时长(秒)。``None`` 表示不限制。异步任务使用
|
||||
:func:`asyncio.wait_for`;线程/异步执行器中的同步任务会
|
||||
取消 worker future。
|
||||
:func:`asyncio.wait_for`;同步任务通过线程 future 取消。
|
||||
tags:
|
||||
自由标签,供 :meth:`Graph.subgraph` 做选择性执行与调试。
|
||||
自由标签,供 :meth:`Graph.subgraph` 做选择性执行与调试,
|
||||
也可用于并发限制分组。
|
||||
conditions:
|
||||
条件判断函数列表,只有所有条件都返回 ``True`` 时才执行任务。
|
||||
若任一条件返回 ``False``,任务会被标记为 SKIPPED。
|
||||
用于平台判断、环境变量检查等场景。
|
||||
条件判断函数列表,接收依赖上下文,全部返回 ``True`` 时才执行任务。
|
||||
任一返回 ``False`` 则任务被标记为 SKIPPED。
|
||||
cwd:
|
||||
命令执行的工作目录,仅在使用 ``cmd`` 参数时有效。
|
||||
``None`` 表示当前目录。
|
||||
工作目录。对 ``cmd`` 任务作为子进程工作目录;对 ``fn`` 任务
|
||||
通过临时切换当前目录生效。
|
||||
env:
|
||||
环境变量覆盖映射。对 ``cmd`` 任务合并到子进程环境;对 ``fn``
|
||||
任务在执行期间临时设置。
|
||||
verbose:
|
||||
是否在命令执行时显示详细输出。``True`` 时会打印执行的命令
|
||||
及其标准输出/标准错误。仅在使用 ``cmd`` 参数时有效。
|
||||
``False`` 时静默捕获输出(失败时仍会包含在错误信息中)。
|
||||
是否打印详细输出。``True`` 时打印执行的命令、返回码与输出
|
||||
(仅 ``cmd``),以及任务生命周期。
|
||||
skip_if_missing:
|
||||
仅对 ``cmd`` 为 ``list[str]`` 的任务有效。``True`` 时自动检查
|
||||
命令是否存在(通过 :func:`shutil.which`),不存在则跳过任务
|
||||
(标记为 SKIPPED)而非失败。适用于构建工具场景,避免因未安装
|
||||
某些工具(如 maturin、tox)而导致整个图执行失败。
|
||||
对于 ``str`` (shell) 和 ``Callable`` 类型的 ``cmd``,此参数无效。
|
||||
仅对 ``cmd`` 为 ``list[str]`` 有效。``True`` 时通过
|
||||
:func:`shutil.which` 检查命令是否存在,不存在则跳过。
|
||||
allow_upstream_skip:
|
||||
若为 ``True``,硬依赖被 SKIPPED 时本任务仍执行(软依赖不影响)。
|
||||
适用于清理类任务。
|
||||
strategy:
|
||||
单任务执行策略覆盖。``None`` 表示继承图级策略。
|
||||
``"sequential"`` 同步直接调用;``"thread"``/``"async"`` 将同步
|
||||
任务卸载到线程池,异步任务跑在事件循环上。
|
||||
priority:
|
||||
同层任务调度优先级。数值越大越先启动。仅影响同层内启动顺序,
|
||||
不打破层屏障。默认 ``0``。
|
||||
concurrency_key:
|
||||
并发限制分组键。具有相同键的任务共享一个信号量,限制同时
|
||||
运行的实例数。具体限额由 :func:`run` 的 ``concurrency_limits``
|
||||
参数提供 ``{key: limit}`` 映射。``None`` 表示不限制。
|
||||
continue_on_error:
|
||||
若为 ``True``,任务最终失败时不中止整图,仅标记本任务 FAILED,
|
||||
其硬依赖下游被 SKIPPED,其余任务继续。默认 ``False``。
|
||||
cache_key:
|
||||
缓存键计算函数。若提供,则用其基于依赖上下文计算的字符串键
|
||||
存取状态后端,使不同输入产生独立缓存条目。``None`` 表示用任务名。
|
||||
hooks:
|
||||
:class:`TaskHooks` 生命周期钩子。
|
||||
executor:
|
||||
同步任务的执行器:``"thread"``(默认,线程池)/ ``"process"``
|
||||
(进程池,绕过 GIL,适合 CPU 密集型;``fn`` 须可 pickle)/
|
||||
``"inline"``(直接在事件循环线程调用,最快但会阻塞循环)。
|
||||
"""
|
||||
|
||||
name: str
|
||||
fn: Optional[TaskFn[T]] = None
|
||||
cmd: Optional[TaskCmd] = None
|
||||
depends_on: Tuple[str, ...] = ()
|
||||
args: Tuple[Any, ...] = ()
|
||||
fn: TaskFn[T] | None = None
|
||||
cmd: TaskCmd | None = None
|
||||
depends_on: tuple[str, ...] = ()
|
||||
soft_depends_on: tuple[str, ...] = ()
|
||||
defaults: Mapping[str, Any] = field(default_factory=dict)
|
||||
args: tuple[Any, ...] = ()
|
||||
kwargs: Mapping[str, Any] = field(default_factory=dict)
|
||||
retries: int = 0
|
||||
timeout: Optional[float] = None
|
||||
tags: Tuple[str, ...] = ()
|
||||
conditions: Tuple[Condition, ...] = ()
|
||||
cwd: Optional[Path] = None
|
||||
retry: RetryPolicy = field(default_factory=RetryPolicy)
|
||||
timeout: float | None = None
|
||||
tags: tuple[str, ...] = ()
|
||||
conditions: tuple[Condition, ...] = ()
|
||||
cwd: Path | None = None
|
||||
env: Mapping[str, str] | None = None
|
||||
verbose: bool = False
|
||||
skip_if_missing: bool = True
|
||||
skip_if_missing: bool = False
|
||||
allow_upstream_skip: bool = False
|
||||
strategy: str | None = None
|
||||
priority: int = 0
|
||||
concurrency_key: str | None = None
|
||||
continue_on_error: bool = False
|
||||
cache_key: CacheKeyFn | None = None
|
||||
hooks: TaskHooks = field(default_factory=TaskHooks)
|
||||
executor: str = "thread" # "thread" | "process" | "inline"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.name:
|
||||
raise ValueError("TaskSpec.name must be a non-empty string.")
|
||||
if self.retries < 0:
|
||||
raise ValueError(f"TaskSpec '{self.name}': retries must be >= 0.")
|
||||
if self.retry.max_attempts < 1:
|
||||
raise ValueError(f"TaskSpec '{self.name}': retry.max_attempts must be >= 1.")
|
||||
if self.timeout is not None and self.timeout <= 0:
|
||||
raise ValueError(f"TaskSpec '{self.name}': timeout must be > 0.")
|
||||
if self.name in self.depends_on:
|
||||
if self.name in self.depends_on or self.name in self.soft_depends_on:
|
||||
raise ValueError(f"TaskSpec '{self.name}' cannot depend on itself.")
|
||||
overlap = set(self.depends_on) & set(self.soft_depends_on)
|
||||
if overlap:
|
||||
raise ValueError(f"TaskSpec '{self.name}': depends_on 与 soft_depends_on 不能重叠: {sorted(overlap)}")
|
||||
if self.fn is None and self.cmd is None:
|
||||
raise ValueError(f"TaskSpec '{self.name}': 必须提供 fn 或 cmd 参数。")
|
||||
|
||||
@property
|
||||
@cached_property
|
||||
def effective_fn(self) -> TaskFn[T]:
|
||||
"""获取有效的执行函数.
|
||||
"""获取有效的执行函数。
|
||||
|
||||
若提供了 ``cmd`` 参数,则返回包装后的命令执行函数;
|
||||
否则返回 ``fn`` 参数。
|
||||
若提供 ``cmd``,返回包装后的命令执行函数;否则返回 ``fn``。
|
||||
包装函数在每次调用时从 ``self`` 读取 ``verbose``/``cwd``/``env``/
|
||||
``timeout``,避免闭包捕获运行期参数,使翻转字段无需重建 spec。
|
||||
|
||||
结果按实例缓存(:func:`functools.cached_property`):frozen dataclass
|
||||
字段不可变,``_wrap_cmd`` 生成的闭包稳定,无需每次访问重建。
|
||||
"""
|
||||
if self.cmd is not None:
|
||||
return self._wrap_cmd()
|
||||
if self.fn is not None:
|
||||
return self.fn
|
||||
|
||||
raise ValueError(f"TaskSpec '{self.name}': 没有可执行的函数或命令。") # pragma: no cover
|
||||
|
||||
def _wrap_cmd(self) -> TaskFn[Any]:
|
||||
"""将 cmd 包装为可执行函数.
|
||||
"""将 cmd 包装为可执行函数。
|
||||
|
||||
实际执行逻辑位于 :mod:`pyflowx.command`,避免 :class:`TaskSpec`
|
||||
作为纯数据结构混入命令执行逻辑。
|
||||
"""
|
||||
from .command import run_command
|
||||
|
||||
spec = self
|
||||
|
||||
def _run() -> T:
|
||||
return cast(T, run_command(spec))
|
||||
|
||||
_run.__name__ = spec.name
|
||||
return _run # type: ignore[return-value]
|
||||
|
||||
def should_execute(self, context: Context) -> tuple[bool, str | None]:
|
||||
"""检查任务是否应执行。
|
||||
|
||||
Returns
|
||||
-------
|
||||
TaskFn[Any]
|
||||
包装后的执行函数.
|
||||
(should_run, skip_reason)
|
||||
``should_run`` 为 False 时 ``skip_reason`` 描述跳过原因。
|
||||
失败条件超过 2 个时仅展示前 2 个并附总数。
|
||||
"""
|
||||
cmd = self.cmd
|
||||
cwd = self.cwd
|
||||
timeout = self.timeout
|
||||
verbose = self.verbose
|
||||
|
||||
if isinstance(cmd, list):
|
||||
cmd_list = cast(List[str], cmd)
|
||||
|
||||
def _run_list() -> T:
|
||||
import subprocess
|
||||
|
||||
cmd_str = " ".join(str(arg) for arg in cmd_list)
|
||||
if verbose:
|
||||
print(f"[verbose] 执行命令: {cmd_str}", flush=True)
|
||||
if cwd is not None:
|
||||
print(f"[verbose] 工作目录: {cwd}", flush=True)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd_list,
|
||||
cwd=cwd,
|
||||
timeout=timeout,
|
||||
capture_output=not verbose,
|
||||
text=True,
|
||||
check=False,
|
||||
# 逐个求值条件,记录失败项。
|
||||
failed_conditions: list[str] = []
|
||||
for condition in self.conditions:
|
||||
try:
|
||||
ok = condition(context)
|
||||
except Exception:
|
||||
ok = False
|
||||
failed_conditions.append("匿名条件(执行错误)")
|
||||
continue
|
||||
if not ok:
|
||||
reason = getattr(condition, "_reason", None)
|
||||
if reason is not None:
|
||||
failed_conditions.append(
|
||||
", ".join(str(r) for r in reason) if isinstance(reason, list) else str(reason),
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError(f"命令未找到: {cmd_str}") from None
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError(f"命令执行超时: {cmd_str} ({timeout}s)") from None
|
||||
except OSError as e:
|
||||
raise RuntimeError(f"命令执行异常: {cmd_str}: {e}") from e
|
||||
else:
|
||||
failed_conditions.append(getattr(condition, "__name__", None) or "匿名条件")
|
||||
|
||||
if verbose:
|
||||
print(f"[verbose] 返回码: {result.returncode}", flush=True)
|
||||
if failed_conditions:
|
||||
return False, _format_skip_reason(failed_conditions)
|
||||
|
||||
if result.returncode == 0:
|
||||
return cast(T, None) # type: ignore[return-value]
|
||||
if self.skip_if_missing and not self._is_cmd_available():
|
||||
cmd_name = self.cmd[0] if isinstance(self.cmd, list) and self.cmd else "unknown"
|
||||
return False, f"命令不存在: {cmd_name}"
|
||||
|
||||
err_msg = f"命令执行失败: `{cmd_str}`, 返回码: {result.returncode}"
|
||||
if not verbose and result.stderr.strip():
|
||||
err_msg += f"\n{result.stderr.strip()}"
|
||||
raise RuntimeError(err_msg)
|
||||
|
||||
_run_list.__name__ = self.name
|
||||
return _run_list # type: ignore[return-value]
|
||||
|
||||
if isinstance(cmd, str):
|
||||
|
||||
def _run_shell() -> T:
|
||||
import subprocess
|
||||
|
||||
if verbose:
|
||||
print(f"[verbose] 执行 Shell: {cmd}", flush=True)
|
||||
if cwd is not None:
|
||||
print(f"[verbose] 工作目录: {cwd}", flush=True)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
shell=True,
|
||||
cwd=cwd,
|
||||
timeout=timeout,
|
||||
capture_output=not verbose,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError(f"Shell 命令未找到: {cmd}") from None
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError(f"Shell 命令执行超时: {cmd} ({timeout}s)") from None
|
||||
except OSError as e:
|
||||
raise RuntimeError(f"Shell 命令执行异常: {cmd}: {e}") from e
|
||||
|
||||
if verbose:
|
||||
print(f"[verbose] 返回码: {result.returncode}", flush=True)
|
||||
|
||||
if result.returncode == 0:
|
||||
return cast(T, None) # type: ignore[return-value]
|
||||
|
||||
err_msg = f"Shell 命令执行失败: `{cmd}`, 返回码: {result.returncode}"
|
||||
if not verbose and result.stderr.strip():
|
||||
err_msg += f"\n{result.stderr.strip()}"
|
||||
raise RuntimeError(err_msg)
|
||||
|
||||
_run_shell.__name__ = self.name
|
||||
return _run_shell # type: ignore[return-value]
|
||||
|
||||
if callable(cmd):
|
||||
return cmd # type: ignore[return-value]
|
||||
|
||||
raise TypeError(f"TaskSpec '{self.name}': 不支持的 cmd 类型 {type(cmd).__name__}") # pragma: no cover
|
||||
|
||||
def should_execute(self) -> bool:
|
||||
"""检查任务是否应该执行.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
若所有条件都返回 ``True``,且 ``skip_if_missing`` 检查通过,
|
||||
则返回 ``True``;否则返回 ``False``。
|
||||
"""
|
||||
if not all(condition() for condition in self.conditions):
|
||||
return False
|
||||
|
||||
return not (self.skip_if_missing and not self._is_cmd_available())
|
||||
return True, None
|
||||
|
||||
def _is_cmd_available(self) -> bool:
|
||||
"""检查 ``cmd`` 是否可用.
|
||||
|
||||
仅对 ``list[str]`` 类型的 ``cmd`` 进行检查(通过 :func:`shutil.which`)。
|
||||
对于 ``str`` (shell) 和 ``Callable`` 类型,始终返回 ``True``。
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
命令可用返回 ``True``,否则返回 ``False``。
|
||||
"""
|
||||
import shutil
|
||||
|
||||
"""检查 ``cmd`` 是否可用(仅 list[str])。"""
|
||||
cmd = self.cmd
|
||||
if isinstance(cmd, list) and cmd:
|
||||
first_arg = cast(str, cmd[0])
|
||||
return shutil.which(first_arg) is not None
|
||||
return shutil.which(cmd[0]) is not None
|
||||
return True
|
||||
|
||||
def env_context(self) -> ContextManager[None]:
|
||||
"""返回临时应用 ``env`` 与 ``cwd`` 的上下文管理器。
|
||||
|
||||
对 ``fn`` 任务生效。``cmd`` 任务在 :func:`_run_command` 中直接
|
||||
传给子进程。
|
||||
"""
|
||||
return _env_and_cwd(self.env, self.cwd)
|
||||
|
||||
def storage_key(self, context: Context) -> str:
|
||||
"""计算状态后端存储键。"""
|
||||
if self.cache_key is None:
|
||||
return self.name
|
||||
try:
|
||||
return f"{self.name}:{self.cache_key(context)}"
|
||||
except (TypeError, ValueError, KeyError, AttributeError) as exc:
|
||||
# cache_key 抛出预期内的数据/类型异常时回退到 name,但仍记录警告
|
||||
# 以便用户发现 cache_key 实现中的 bug。
|
||||
logger.warning(
|
||||
"task %r: cache_key 回退到 name(%s: %s)",
|
||||
self.name,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
return self.name
|
||||
|
||||
|
||||
# 全局锁:序列化对进程级状态(os.environ / os.chdir)的临时修改。
|
||||
# ``fn`` 任务在 thread/async 策略下并发执行时,若各自配置了不同的
|
||||
# ``cwd``/``env``,会相互覆盖(os.chdir 与 os.environ 均为进程全局)。
|
||||
# 该锁仅包裹"切换→执行→恢复"区间,保证正确性;不使用 cwd/env 的任务不受影响。
|
||||
_env_cwd_lock = threading.RLock()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _env_and_cwd(
|
||||
env: Mapping[str, str] | None,
|
||||
cwd: Path | None,
|
||||
) -> Generator[None, None, None]:
|
||||
"""临时设置环境变量与工作目录。
|
||||
|
||||
``os.environ`` 与 ``os.chdir`` 是进程级全局状态,在 thread/async 策略下
|
||||
并发执行多个带 ``env``/``cwd`` 的 ``fn`` 任务时会相互覆盖。本函数通过
|
||||
模块级 :data:`_env_cwd_lock` 串行化"切换→执行→恢复"区间,确保正确性。
|
||||
无 ``env`` 且无 ``cwd`` 时直接 yield,不获取锁。
|
||||
"""
|
||||
if not env and cwd is None:
|
||||
yield
|
||||
return
|
||||
with _env_cwd_lock:
|
||||
saved_env: dict[str, str] = {}
|
||||
saved_cwd: str | None = None
|
||||
if env:
|
||||
for k, v in env.items():
|
||||
if k in os.environ:
|
||||
saved_env[k] = os.environ[k]
|
||||
os.environ[k] = v
|
||||
if cwd is not None:
|
||||
saved_cwd = str(Path.cwd())
|
||||
os.chdir(cwd)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if saved_cwd is not None:
|
||||
os.chdir(saved_cwd)
|
||||
# 恢复环境变量
|
||||
if env:
|
||||
for k in env:
|
||||
if k in saved_env:
|
||||
os.environ[k] = saved_env[k]
|
||||
else:
|
||||
os.environ.pop(k, None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 任务模板:批量生成相似 TaskSpec 的工厂
|
||||
# ---------------------------------------------------------------------- #
|
||||
def _task_noop() -> None:
|
||||
"""task(cmd=...) 形式下的占位 fn(cmd 任务执行期不调用 fn)。"""
|
||||
return None
|
||||
|
||||
|
||||
def task(
|
||||
fn: TaskFn[Any] | None = None,
|
||||
*,
|
||||
cmd: TaskCmd | None = None,
|
||||
depends_on: tuple[str, ...] = (),
|
||||
soft_depends_on: tuple[str, ...] = (),
|
||||
defaults: Mapping[str, Any] | None = None,
|
||||
args: tuple[Any, ...] = (),
|
||||
kwargs: Mapping[str, Any] | None = None,
|
||||
retry: RetryPolicy | None = None,
|
||||
timeout: float | None = None,
|
||||
tags: tuple[str, ...] = (),
|
||||
conditions: tuple[Condition, ...] = (),
|
||||
cwd: str | Path | None = None,
|
||||
env: Mapping[str, str] | None = None,
|
||||
verbose: bool = False,
|
||||
skip_if_missing: bool = False,
|
||||
allow_upstream_skip: bool = False,
|
||||
strategy: str | None = None,
|
||||
priority: int = 0,
|
||||
concurrency_key: str | None = None,
|
||||
continue_on_error: bool = False,
|
||||
cache_key: CacheKeyFn | None = None,
|
||||
hooks: TaskHooks | None = None,
|
||||
name: str | None = None,
|
||||
) -> Any:
|
||||
"""装饰器:将函数转为 :class:`TaskSpec`。
|
||||
|
||||
``name`` 默认取 ``fn.__name__``。可直接装饰函数,或带参数使用。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> @px.task
|
||||
... def extract(): return [1, 2, 3]
|
||||
>>> @px.task(depends_on=("extract",))
|
||||
... def double(extract): return [x * 2 for x in extract]
|
||||
>>> graph = px.Graph.from_specs([extract, double])
|
||||
"""
|
||||
|
||||
def _decorate(func: TaskFn[Any]) -> TaskSpec[Any]:
|
||||
spec_name = name or func.__name__
|
||||
return TaskSpec(
|
||||
name=spec_name,
|
||||
fn=func,
|
||||
cmd=cmd,
|
||||
depends_on=depends_on,
|
||||
soft_depends_on=soft_depends_on,
|
||||
defaults=dict(defaults) if defaults else {},
|
||||
args=args,
|
||||
kwargs=dict(kwargs) if kwargs else {},
|
||||
retry=retry if retry is not None else RetryPolicy(),
|
||||
timeout=timeout,
|
||||
tags=tags,
|
||||
conditions=conditions,
|
||||
cwd=Path(cwd) if isinstance(cwd, str) else cwd,
|
||||
env=dict(env) if env else None,
|
||||
verbose=verbose,
|
||||
skip_if_missing=skip_if_missing,
|
||||
allow_upstream_skip=allow_upstream_skip,
|
||||
strategy=strategy,
|
||||
priority=priority,
|
||||
concurrency_key=concurrency_key,
|
||||
continue_on_error=continue_on_error,
|
||||
cache_key=cache_key,
|
||||
hooks=hooks if hooks is not None else TaskHooks(),
|
||||
)
|
||||
|
||||
if fn is None and cmd is None:
|
||||
# 带参数调用:@task(depends_on=...),等待被装饰函数
|
||||
return _decorate
|
||||
if fn is None:
|
||||
# task(cmd=..., name=...) 直接构造,无被装饰函数
|
||||
if name is None:
|
||||
raise ValueError("task(cmd=...) 需要显式提供 name")
|
||||
return _decorate(_task_noop)
|
||||
return _decorate(fn)
|
||||
|
||||
|
||||
def cmd(
|
||||
command: list[str],
|
||||
*,
|
||||
name: str | None = None,
|
||||
depends_on: tuple[str, ...] = (),
|
||||
**kwargs: Any,
|
||||
) -> TaskSpec[Any]:
|
||||
"""从命令列表快速创建 :class:`TaskSpec`。
|
||||
|
||||
``name`` 默认为 ``"_".join(command[:2])``(如 ``["uv", "build"]`` → ``"uv_build"``)。
|
||||
若命令不足两个元素则用 ``"_".join(command)``。
|
||||
|
||||
其余关键字参数透传给 :class:`TaskSpec`(如 ``depends_on``、``tags`` 等)。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> uv_build = px.cmd(["uv", "build"])
|
||||
>>> uv_build.name
|
||||
'uv_build'
|
||||
>>> lint = px.cmd(["ruff", "check", "--fix"], name="lint")
|
||||
>>> lint.name
|
||||
'lint'
|
||||
"""
|
||||
spec_name = name or "_".join(command[:2]) if len(command) >= 2 else "_".join(command)
|
||||
return TaskSpec(
|
||||
name=spec_name,
|
||||
cmd=command,
|
||||
depends_on=depends_on,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def task_template(
|
||||
fn: TaskFn[Any] | None = None,
|
||||
cmd: TaskCmd | None = None,
|
||||
**defaults: Any,
|
||||
) -> Callable[..., TaskSpec[Any]]:
|
||||
"""创建任务模板工厂。
|
||||
|
||||
返回的工厂接受 ``name`` 与任意覆盖字段,生成 :class:`TaskSpec`。
|
||||
适用于批量创建相似任务(如 fan-out)。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> Fetch = px.task_template(fn=fetch_user, retry=px.RetryPolicy(max_attempts=3))
|
||||
>>> specs = [Fetch(f"fetch_{uid}", args=(uid,)) for uid in range(5)]
|
||||
"""
|
||||
base = dict(defaults)
|
||||
if fn is not None:
|
||||
base["fn"] = fn
|
||||
if cmd is not None:
|
||||
base["cmd"] = cmd
|
||||
|
||||
def _factory(name: str, **overrides: Any) -> TaskSpec[Any]:
|
||||
merged = dict(base)
|
||||
merged.update(overrides)
|
||||
return TaskSpec(name, **merged)
|
||||
|
||||
_factory.__name__ = "task_template_factory"
|
||||
return _factory
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskResult(Generic[T]):
|
||||
"""运行期间产生的可变单任务记录。
|
||||
|
||||
每次运行都会创建全新的 :class:`TaskResult`;spec 本身保持不可变。
|
||||
这让同一个图可以安全地重复运行。
|
||||
"""
|
||||
"""运行期间产生的可变单任务记录。"""
|
||||
|
||||
spec: TaskSpec[T]
|
||||
status: TaskStatus = TaskStatus.PENDING
|
||||
value: Optional[T] = None
|
||||
error: Optional[BaseException] = None
|
||||
value: T | None = None
|
||||
error: BaseException | None = None
|
||||
attempts: int = 0
|
||||
started_at: Optional[datetime] = None
|
||||
finished_at: Optional[datetime] = None
|
||||
reason: Optional[str] = None # 跳过原因
|
||||
started_at: datetime | None = None
|
||||
finished_at: datetime | None = None
|
||||
reason: str | None = None # 跳过原因
|
||||
|
||||
@property
|
||||
def duration(self) -> Optional[float]:
|
||||
def duration(self) -> float | None:
|
||||
"""从开始到结束的耗时(秒),未开始/未结束则为 ``None``。"""
|
||||
if self.started_at is None or self.finished_at is None:
|
||||
return None
|
||||
@@ -320,15 +620,11 @@ class TaskResult(Generic[T]):
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TaskEvent:
|
||||
"""执行期间向观察者发出的不可变事件。
|
||||
|
||||
传递给 :func:`pyflowx.run` 的 ``on_event`` 回调,让调用者无需耦合
|
||||
执行器内部即可构建进度条、指标或结构化日志。
|
||||
"""
|
||||
"""执行期间向观察者发出的不可变事件。"""
|
||||
|
||||
task: str
|
||||
status: TaskStatus
|
||||
attempts: int = 0
|
||||
error: Optional[str] = None
|
||||
duration: Optional[float] = None
|
||||
reason: Optional[str] = None # 跳过原因,如 "条件不满足"、"上游任务被跳过"、"缓存"
|
||||
error: str | None = None
|
||||
duration: float | None = None
|
||||
reason: str | None = None
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
"""系统操作任务模块.
|
||||
|
||||
提供常用的系统操作任务封装, 包括清屏、环境变量设置、命令查找等.
|
||||
遵循实用主义原则, 仅提供核心功能, 无过度设计.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = [
|
||||
"clr",
|
||||
"reset_icon_cache",
|
||||
"setenv",
|
||||
"setenv_group",
|
||||
"which",
|
||||
"write_file",
|
||||
]
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx import BuiltinConditions
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
def clr():
|
||||
"""清屏任务."""
|
||||
cmd = ["cls"] if Constants.IS_WINDOWS else ["clear"]
|
||||
return px.TaskSpec("clear_screen", fn=lambda: subprocess.run(cmd, check=False))
|
||||
|
||||
|
||||
def reset_icon_cache() -> list[px.TaskSpec]:
|
||||
"""重置图标缓存任务."""
|
||||
if not Constants.IS_WINDOWS:
|
||||
print("reset_icon_cache: 仅在 Windows 上支持")
|
||||
return []
|
||||
|
||||
local_app_data = os.environ.get("LOCALAPPDATA", "")
|
||||
icon_cache_db = Path(local_app_data) / "IconCache.db"
|
||||
explorer_cache_dir = Path(local_app_data) / "Microsoft" / "Windows" / "Explorer"
|
||||
|
||||
return [
|
||||
px.TaskSpec(
|
||||
"kill_explorer",
|
||||
cmd=["taskkill", "/f", "/im", "explorer.exe"],
|
||||
conditions=(BuiltinConditions.IS_RUNNING("explorer.exe"),),
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
"delete_icon_cache",
|
||||
cmd=["cmd", "/c", "del", "/a", "/q", str(icon_cache_db)],
|
||||
conditions=(BuiltinConditions.DIR_EXISTS(icon_cache_db),),
|
||||
depends_on=("kill_explorer",),
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
"delete_icon_cache_all",
|
||||
cmd=["cmd", "/c", "del", "/a", "/q", str(explorer_cache_dir / "iconcache*")],
|
||||
conditions=(BuiltinConditions.DIR_EXISTS(explorer_cache_dir),),
|
||||
depends_on=("kill_explorer",),
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
"restart_explorer",
|
||||
cmd=["cmd", "/c", "start", "explorer.exe"],
|
||||
conditions=(
|
||||
BuiltinConditions.HAS_INSTALLED("explorer.exe"),
|
||||
BuiltinConditions.NOT(BuiltinConditions.IS_RUNNING("explorer.exe")),
|
||||
),
|
||||
depends_on=("delete_icon_cache", "delete_icon_cache_all"),
|
||||
allow_upstream_skip=True,
|
||||
verbose=True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def setenv(name: str, value: str, default: bool = False) -> px.TaskSpec:
|
||||
"""设置环境变量任务."""
|
||||
|
||||
def set_env():
|
||||
if default:
|
||||
os.environ.setdefault(name, value)
|
||||
else:
|
||||
os.environ[name] = value
|
||||
|
||||
return px.TaskSpec(f"setenv_{name.lower()}", fn=set_env, verbose=True)
|
||||
|
||||
|
||||
def setenv_group(envs: dict[str, str], default: bool = False) -> list[px.TaskSpec]:
|
||||
"""设置环境变量组任务."""
|
||||
return [setenv(name, value, default) for name, value in envs.items()]
|
||||
|
||||
|
||||
def which(cmd: str) -> px.TaskSpec:
|
||||
"""查找命令路径任务."""
|
||||
which_cmd = "where" if Constants.IS_WINDOWS else "which"
|
||||
|
||||
def find_command():
|
||||
result = subprocess.run([which_cmd, cmd], capture_output=True, text=True, check=False)
|
||||
|
||||
if result.returncode == 0:
|
||||
# Windows 的 where 可能返回多行, 取第一个
|
||||
path = result.stdout.strip().split("\n")[0].strip()
|
||||
print(f"{cmd} -> {path}")
|
||||
else:
|
||||
print(f"{cmd} -> 未找到")
|
||||
|
||||
return px.TaskSpec(f"which_{cmd}", fn=find_command)
|
||||
|
||||
|
||||
def write_file(path: str, content: str, encoding: str = "utf-8") -> px.TaskSpec:
|
||||
"""写入文件任务."""
|
||||
|
||||
def write():
|
||||
p = Path(path)
|
||||
p.write_text(content, encoding=encoding)
|
||||
|
||||
return px.TaskSpec(f"write_file_{path}", fn=write, verbose=True)
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user