Compare commits
112 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 08149d990d | |||
| 54aef98f3a | |||
| e31646e281 | |||
| f7fb95af83 | |||
| 6da42ec5ff | |||
| c55a37173a | |||
| 960b8672f4 | |||
| 4fd1d70b58 | |||
| 6fb9223066 | |||
| 1f7127357e | |||
| 58ee84ded6 | |||
| 9a96e5d052 | |||
| c9c7529c58 | |||
| c498d9b1c9 | |||
| b36e279f92 | |||
| 58d6f1faad | |||
| d93da0d8b4 | |||
| 701c455c42 | |||
| e174b64495 | |||
| 3afb25bb5e | |||
| fbd17536fd | |||
| 32ca8c1208 | |||
| a7ff68d279 | |||
| de368ea810 | |||
| 6a3e3a57cd | |||
| 7089944306 | |||
| ec5e348694 | |||
| 12d9f2f647 | |||
| 6ffcbecade | |||
| e76d93187b | |||
| 52e20e3f93 | |||
| 3f966a230e | |||
| 5d0b211a44 | |||
| 6931f36fd1 | |||
| db02443463 | |||
| eb8e1402bc | |||
| c93f45dcb8 | |||
| a0b1814024 | |||
| 3a2826d3f9 | |||
| dbd30689ab | |||
| 5eb59b8a66 | |||
| 8e7b866de2 | |||
| 1b4f9bfa6a | |||
| 2d39272330 | |||
| f699bb9167 | |||
| 35f07e96e1 | |||
| 1f274fe828 | |||
| 85793ff9d5 | |||
| 37ac4b8025 | |||
| 0edeadb846 | |||
| f63db6c71a | |||
| 4d397606e6 | |||
| f24388b151 | |||
| 535b7cba31 | |||
| 3f68bed3fd | |||
| 2e2ca812a1 | |||
| 8de565d0cb | |||
| 5480c48e67 | |||
| c6653d5117 | |||
| d194a991a0 | |||
| 4446658170 | |||
| 1d26f9d3e7 | |||
| d9644ca5d1 | |||
| d3c2d53449 | |||
| 9cfcfb38e4 | |||
| 69db241611 | |||
| 66e6295a24 | |||
| aebb4fce68 | |||
| 7784c8ff86 | |||
| 77918a5568 | |||
| 7e4c615dc7 | |||
| ac5082523e | |||
| 0df6f7c8ac | |||
| 4b66176ce6 | |||
| cf6b6fd059 | |||
| 6f93e6eb6d | |||
| 43e1aad1fe | |||
| 467634f8c7 | |||
| ce31f60441 | |||
| 3d6d769685 | |||
| 3f9c52e6f1 | |||
| 8fadf6edd8 | |||
| abc1152538 | |||
| 5e561b4b3a | |||
| 40f641611b | |||
| 232e7293d9 | |||
| a1bae58e56 | |||
| cbc7cc0a75 | |||
| d0ff7d7b4d | |||
| d154f67ce0 | |||
| 9999071119 | |||
| bdd70e9c43 | |||
| c15b38516a | |||
| 7d4e8a40ce | |||
| 1b2d6d6a2c | |||
| df890f0f16 | |||
| b62a544569 | |||
| d58fc5536e | |||
| c3b86b603d | |||
| 327bd6e069 | |||
| 22f8d2110d | |||
| 2a1f2f7175 | |||
| 9d033e1c0b | |||
| 336f7b7292 | |||
| 65dcbcbf62 | |||
| 7fa97a01e3 | |||
| 83da5135d0 | |||
| 7463a60649 | |||
| 87dd010342 | |||
| bdfee7bee4 | |||
| b954fb1622 | |||
| a7b7a82dff |
@@ -0,0 +1,46 @@
|
||||
# 版本控制
|
||||
.git
|
||||
.gitignore
|
||||
.github
|
||||
|
||||
# Python 缓存与构建产物
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
*.egg-info
|
||||
*.egg
|
||||
dist
|
||||
build
|
||||
.eggs
|
||||
|
||||
# 测试与覆盖率
|
||||
.pytest_cache
|
||||
.coverage
|
||||
htmlcov
|
||||
.tox
|
||||
coverage.xml
|
||||
|
||||
# 虚拟环境
|
||||
.venv
|
||||
venv
|
||||
env
|
||||
|
||||
# 工具缓存
|
||||
.uv-cache
|
||||
.ruff_cache
|
||||
.pyrefly_cache
|
||||
.mypy_cache
|
||||
|
||||
# IDE 与编辑器
|
||||
.idea
|
||||
.vscode
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# 文档(按需保留)
|
||||
docs
|
||||
|
||||
# 系统文件
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
+18
-34
@@ -9,42 +9,26 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
lint-and-typecheck:
|
||||
name: Lint & Typecheck
|
||||
ci:
|
||||
name: Lint, Typecheck & Test
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: pyflowx-ci:latest
|
||||
env:
|
||||
UV_LINK_MODE: copy
|
||||
# ---- 国内源 ----
|
||||
PIP_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
PIP_TRUSTED_HOST: pypi.tuna.tsinghua.edu.cn
|
||||
UV_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
UV_TRUSTED_HOST: pypi.tuna.tsinghua.edu.cn
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: http://gitea:3000/zhou/checkout.git@main
|
||||
|
||||
- uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
- name: Sync dependencies
|
||||
run: uv sync --frozen
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- name: Ruff check
|
||||
run: ruff check src tests
|
||||
|
||||
- run: uv sync
|
||||
- run: uv run ruff check src tests
|
||||
- run: uv run pyrefly check .
|
||||
|
||||
test:
|
||||
name: Test (${{ matrix.os }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: |
|
||||
3.8
|
||||
3.13
|
||||
|
||||
- run: uvx tox run -e py38,py313
|
||||
- name: Tox test (py38, py313)
|
||||
run: uvx tox run -e py38,py313
|
||||
|
||||
@@ -6,56 +6,53 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
|
||||
- run: uv build
|
||||
|
||||
- id: version
|
||||
run: echo "version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: dist
|
||||
path: dist/
|
||||
|
||||
publish-pypi:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
environment: pypi
|
||||
steps:
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: dist
|
||||
path: dist
|
||||
|
||||
- uses: pypa/gh-action-pypi-publish@release/v1
|
||||
|
||||
release:
|
||||
needs: [build, publish-pypi]
|
||||
name: Build, Publish & Release
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: pyflowx-ci:latest
|
||||
env:
|
||||
UV_LINK_MODE: copy
|
||||
# ---- 国内源 ----
|
||||
PIP_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
PIP_TRUSTED_HOST: pypi.tuna.tsinghua.edu.cn
|
||||
UV_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
UV_TRUSTED_HOST: pypi.tuna.tsinghua.edu.cn
|
||||
steps:
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: dist
|
||||
path: dist
|
||||
- uses: http://gitea:3000/zhou/checkout.git@v4
|
||||
|
||||
- uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: dist/*
|
||||
generate_release_notes: true
|
||||
- name: Build distributions
|
||||
run: uv build
|
||||
|
||||
- name: Publish to pypi
|
||||
run: uv publish --token '${{ secrets.PYPI_TOKEN }}'
|
||||
|
||||
- name: Create Gitea Release & Upload Assets
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TAG_NAME: ${{ github.ref_name }}
|
||||
REPO: ${{ github.repository }}
|
||||
GITEA_URL: http://172.17.0.1:3000
|
||||
run: |
|
||||
set -e
|
||||
# 1. 创建 Release
|
||||
RELEASE_ID=$(curl -sS -X POST "$GITEA_URL/api/v1/repos/$REPO/releases" \
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"tag_name\":\"$TAG_NAME\",\"name\":\"Release $TAG_NAME\",\"body\":\"Automated release from CI\",\"draft\":false,\"prerelease\":false}" \
|
||||
| python3 -c "import sys,json;print(json.load(sys.stdin)['id'])")
|
||||
|
||||
echo "Created release id=$RELEASE_ID"
|
||||
|
||||
# 2. 上传 dist/ 下所有文件作为附件
|
||||
for f in dist/*; do
|
||||
echo "Uploading $f ..."
|
||||
curl -sS -X POST "$GITEA_URL/api/v1/repos/$REPO/releases/$RELEASE_ID/assets?name=$(basename $f)" \
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
-H "Content-Type: application/octet-stream" \
|
||||
--data-binary "@$f"
|
||||
done
|
||||
|
||||
|
||||
|
||||
@@ -10,3 +10,8 @@ wheels/
|
||||
.venv
|
||||
.coverage
|
||||
.idea
|
||||
*_profile.html
|
||||
|
||||
# Sphinx 文档构建输出
|
||||
docs/_build/
|
||||
.trae/refs
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# ReadTheDocs 配置
|
||||
# https://docs.readthedocs.io/en/stable/config-file/v2.html
|
||||
version: 2
|
||||
|
||||
# 构建配置
|
||||
build:
|
||||
os: ubuntu-24.04
|
||||
tools:
|
||||
python: "3.11"
|
||||
|
||||
# Python 依赖与构建命令
|
||||
python:
|
||||
install:
|
||||
- method: pip
|
||||
path: .
|
||||
extra_requirements:
|
||||
- docs
|
||||
|
||||
# Sphinx 构建
|
||||
sphinx:
|
||||
configuration: docs/conf.py
|
||||
builder: html
|
||||
fail_on_warning: false
|
||||
@@ -0,0 +1,15 @@
|
||||
# PYTHON
|
||||
.coverage
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
.tox/
|
||||
.venv/
|
||||
__pycache__/
|
||||
|
||||
# NODEJS
|
||||
node_modules/
|
||||
|
||||
# IDE
|
||||
.idea
|
||||
.trae
|
||||
.vscode
|
||||
@@ -0,0 +1,108 @@
|
||||
# 文档整理与 Sphinx 文档搭建计划
|
||||
|
||||
## Context
|
||||
|
||||
最近完成 CLI 重构:新增 `pf` 统一入口,13 个工具迁移到 YAML 配置并删除了对应 .py 入口脚本,`run()` 的 verbose 统一应用到 spec。但文档未同步:README 仍引用旧命令(`yamlrun`、`python build.py`),模块结构表缺漏;`runner.py` 的 `_apply_verbose_to_graph` 成为死代码;项目缺少可发布的 Sphinx 文档。本次任务整理这些遗留,并搭建 ReadTheDocs 文档站。
|
||||
|
||||
## 任务范围
|
||||
|
||||
### 1. 清理死代码
|
||||
- 删除 `src/pyflowx/runner.py` 的 `_apply_verbose_to_graph` 函数(line 38-68),功能已移入 `executors.py` 的 `run()`。
|
||||
- 删除 `tests/test_runner.py` 中对应测试(line 610-636,`TestApplyVerboseToGraph` 类)。
|
||||
- 清理 `runner.py` 顶部 `from dataclasses import replace` 若变为未使用。
|
||||
|
||||
### 2. 修复版本不一致
|
||||
- `src/pyflowx/__init__.py:105` 硬编码 `__version__ = "0.4.5"`,`pyproject.toml:25` 为 `0.3.5`。
|
||||
- 统一为 `0.4.5`(`__init__.py` 为准,pyproject.toml 是源但 bumpversion 工具应同时更新两者)。
|
||||
|
||||
### 3. 更新 README.md
|
||||
- L304-308:`python build.py clean/build/test` → `pf pymake clean/build/test`。
|
||||
- L335-351、L435:`yamlrun pipeline.yaml ...` → `pf yamlrun pipeline.yaml ...`(6 处)。
|
||||
- L311:`verbose=True(默认)` 描述保留,但 CLI 示例改为 `pf`。
|
||||
- L558-574 模块结构表:补充 `cli/pf.py`(统一入口)、`cli/configs/`(YAML 工具配置)、`cli/_ops/`(工具函数)、`profiling.py`、`registry.py`。
|
||||
- 顶部增加「文档」徽章链接到 ReadTheDocs。
|
||||
|
||||
### 4. 搭建 Sphinx 文档结构
|
||||
新建 `docs/` 目录:
|
||||
|
||||
```
|
||||
docs/
|
||||
├── conf.py # Sphinx 配置
|
||||
├── index.rst # 首页与目录
|
||||
├── installation.rst # 安装
|
||||
├── quickstart.rst # 快速上手(从 README 提炼)
|
||||
├── guide/
|
||||
│ ├── task.rst # TaskSpec 任务描述
|
||||
│ ├── graph.rst # Graph DAG 构建
|
||||
│ ├── execution.rst # 执行策略与 run()
|
||||
│ ├── yaml.rst # YAML 任务编排
|
||||
│ └── cli.rst # pf 统一入口与工具列表
|
||||
├── api.rst # API 参考(automodule 自动生成)
|
||||
└── changelog.rst # 变更日志摘要
|
||||
```
|
||||
|
||||
**conf.py 要点**:
|
||||
- 扩展:`sphinx.ext.autodoc`、`sphinx.ext.napoleon`(支持 Google/NumPy docstring)、`sphinx.ext.viewcode`、`myst_parser`(支持 Markdown)
|
||||
- 主题:`sphinx_rtd_theme`
|
||||
- 项目版本从 `pyflowx.__version__` 动态读取
|
||||
- `autodoc_default_options`:`members: True, undoc-members: True, show-inheritance: True`
|
||||
|
||||
**api.rst**:用 `automodule:: pyflowx` 抓取 `__all__` 的 56 个公共符号。
|
||||
|
||||
### 5. ReadTheDocs 配置
|
||||
- 新建 `.readthedocs.yaml`:Python 3.11,`pip install -e .[docs]`,`sphinx -b html docs/ docs/_build/`。
|
||||
- `.gitignore` 增加 `docs/_build/`。
|
||||
|
||||
### 6. pyproject.toml 补充 docs 依赖
|
||||
```toml
|
||||
docs = [
|
||||
"sphinx>=7.0",
|
||||
"sphinx-rtd-theme>=2.0",
|
||||
"myst-parser>=3.0",
|
||||
]
|
||||
```
|
||||
并在 `[dependency-groups]` 的 dev 中加入 `pyflowx[docs]`。
|
||||
|
||||
## 关键文件
|
||||
|
||||
| 文件 | 操作 |
|
||||
|------|------|
|
||||
| `src/pyflowx/runner.py` | 删除 `_apply_verbose_to_graph` |
|
||||
| `tests/test_runner.py` | 删除 `TestApplyVerboseToGraph` |
|
||||
| `src/pyflowx/__init__.py` | 版本统一(已 0.4.5,确认) |
|
||||
| `pyproject.toml` | 版本 → 0.4.5;加 docs 依赖 |
|
||||
| `README.md` | 更新 CLI 示例与模块结构表 |
|
||||
| `docs/conf.py` | 新建 |
|
||||
| `docs/*.rst` | 新建 |
|
||||
| `.readthedocs.yaml` | 新建 |
|
||||
| `.gitignore` | 加 docs/_build/ |
|
||||
|
||||
## 验证
|
||||
|
||||
1. **测试与 lint**:
|
||||
```bash
|
||||
uv run pytest tests/ -q
|
||||
uv run ruff check src/ tests/ docs/conf.py
|
||||
uv run pyrefly check src/pyflowx/runner.py
|
||||
```
|
||||
|
||||
2. **Sphinx 构建本地验证**:
|
||||
```bash
|
||||
uv sync --extra docs
|
||||
uv run sphinx-build -b html docs/ docs/_build/
|
||||
```
|
||||
确认无 warning,打开 `docs/_build/index.html` 检查页面。
|
||||
|
||||
3. **pf 功能回归**:
|
||||
```bash
|
||||
pf gitt c
|
||||
pf pymake b --dry-run
|
||||
```
|
||||
|
||||
4. **RTD 配置校验**:`.readthedocs.yaml` 语法正确,`docs/conf.py` 能独立构建。
|
||||
|
||||
## 不在范围
|
||||
|
||||
- 不统一各模块 docstring 风格(napoleon 兼容 Google/NumPy,够用)。
|
||||
- 不重构现有 CLI 工具 YAML。
|
||||
- 不新增中文文档翻译(文档用中文撰写,与项目既有风格一致)。
|
||||
@@ -0,0 +1,171 @@
|
||||
# 方案C:移除 YAML,统一 @px.tool 装饰器配置
|
||||
|
||||
## Context
|
||||
|
||||
现状:YAML 配置(22 个文件)+ `register_fn` 函数(79 个)分裂严重。fn 模式下 YAML 沦为冗余参数表——参数要三处重复声明(`variables`/`cli.options`/`jobs.args`),表达力不足(跨平台逻辑被迫下沉 `register_fn`),失类型检查/IDE 支持,函数签名与 YAML `args:` 顺序远端耦合。**矩阵 0 个实际使用、`if:` 0 个实际使用**,YAML 的声明式 DAG 优势在 fn 工具集场景完全没发挥。
|
||||
|
||||
目标:完全移除 YAML 支持,统一用 `@px.tool` 装饰器——函数签名驱动 CLI 生成,函数体即逻辑,DAG 编排通过装饰器参数(`needs`/`strategy`)表达。用户无感知迁移(别名/子命令/参数名全保持)。验收:全部门禁通过(ruff/pyrefly/pytest)且覆盖率 ≥95%。
|
||||
|
||||
## 设计要点
|
||||
|
||||
### 新模块 `src/pyflowx/tools.py`
|
||||
|
||||
职责:`@px.tool` 装饰器 + `ToolSpec`(frozen dataclass)+ 全局注册表 + CLI 生成 + 执行入口。**不扩展 `registry.py`**(职责不同,@px.tool 直接持有函数对象)。
|
||||
|
||||
### `@px.tool` API
|
||||
|
||||
```python
|
||||
@px.tool("pdftool", subcommand="m", help="合并 PDF",
|
||||
needs=["setup"], strategy="thread", # DAG 编排参数
|
||||
cmd=["..."], # 可选:有 cmd 执行命令,无 cmd 执行函数体
|
||||
hidden=False, allow_upstream_skip=False)
|
||||
def pdf_merge(
|
||||
inputs: list[Path], # positional(无默认值)→ nargs="+"
|
||||
output: Path = Path("merged.pdf"), # option(有默认值)→ --output
|
||||
password: str = "", # --password
|
||||
dry_run: bool = False, # bool 默认 False → --dry-run store_true
|
||||
level: int = 5, # --level type=int
|
||||
mode: Literal["a","b"] = "a", # --mode choices=["a","b"]
|
||||
) -> None:
|
||||
"""函数体即逻辑(无 cmd 时执行;有 cmd 时签名仅驱动 CLI)."""
|
||||
...
|
||||
```
|
||||
|
||||
**装饰器关键字参数**:`subcommand`、`help`、`description`、`cmd`、`needs`、`strategy`、`cwd`、`allow_upstream_skip`、`hidden`、`env`/`retry`/`timeout`(透传 TaskSpec)。
|
||||
|
||||
**CLI 生成规则**(inspect.signature + typing.get_type_hints):
|
||||
- 无默认值 → positional;有默认值 → `--kebab-case` 选项
|
||||
- 类型映射:`list[X]`→`nargs="+"`、`Path`→`type=Path`、`bool`默认 False→`store_true`、`Literal`→`choices`、`int`/`float`→对应 type
|
||||
- 全局选项 `--dry-run`/`--quiet`/`--strategy`/`--list` 注入每个 subparser
|
||||
|
||||
**执行分支**:
|
||||
- 有 `cmd`:TaskSpec 用 `cmd=cmd`,函数体不执行(签名仅驱动 CLI);cwd/env 从装饰器 + CLI kwargs 合并
|
||||
- 无 `cmd`:TaskSpec 用 `fn=func`,kwargs 透传
|
||||
- 聚合任务(有 needs 无 cmd 无函数逻辑):fn=noop
|
||||
|
||||
**needs 传递依赖**:从 `_TOOL_REGISTRY[tool_name]` BFS 收集(复用 `_collect_with_deps` 思路),每个依赖生成 TaskSpec,构建子图 → `px.run(graph, strategy)`。
|
||||
|
||||
**动态 cmd 改 fn 模式**:`piptool i`/`gittool clean` 等原 `cmd+${VAR}` 改为函数体内 `subprocess.run([...])`,消除 `${VAR}` 列表展开,模型更纯。
|
||||
|
||||
### pf.py 路由改造
|
||||
|
||||
- `_TOOL_ALIASES`(57 个)保留全量,值改为 tool_name(去掉 .yaml 含义)
|
||||
- `_run_yaml` → `_run_tool`:`importlib.import_module(f"pyflowx.ops.{tool_name}")` 触发注册 → 从 `_TOOL_REGISTRY[tool_name]` 取 subcommand → `run_tool(tool, subcommand, argv)` → 退出码 0/1/130
|
||||
- `_LEGACY_TOOLS` 保留 emlmanager/profiler;**删除 yamlrun**
|
||||
- `_list_tools` 改为遍历 `_TOOL_REGISTRY` 取 description
|
||||
|
||||
### ops 模块重组
|
||||
|
||||
从 6 个功能模块(`files/dev/media/system/llm/bumpversion`)重组为 **22 个工具模块**:`ops/pdftool.py`、`ops/pymake.py`、`ops/envdev.py` 等,一文件一工具,文件内多个 `@px.tool` = 多 subcommand。
|
||||
|
||||
- 跨工具复用函数(如 `add_date_prefix` 被 filedate 用)放 `ops/_common.py`
|
||||
- 未被 YAML 引用的约 20 个 `register_fn`(辅助函数如 `get_file_timestamp`/`has_files`)去掉装饰器,降级为模块级私有函数
|
||||
- `register_fn`/`get_fn`/`has_fn` 在阶段5 一并移除(YAML 配套,失去用途)
|
||||
|
||||
## 迁移路径(6 阶段)
|
||||
|
||||
### 阶段1:核心实现 + 单元测试
|
||||
- 新增 `src/pyflowx/tools.py`、`tests/test_tools.py`
|
||||
- 覆盖:注册/重复检测/CLI 生成(各类型)/单命令执行/cmd 分支/fn 分支/聚合/DAG/hidden/全局选项/退出码三态
|
||||
- 验证:`tools.py` 单元测试通过,覆盖率 ≥95%
|
||||
|
||||
### 阶段2:迁移简单单命令工具
|
||||
- 工具:clr/which/taskkill/reseticoncache/screenshot/dockercmd/lscalc
|
||||
- 新增 `ops/<tool>.py`,每个用 `@px.tool` 重写;删除对应 `configs/*.yaml`
|
||||
- 改 `tests/cli/test_*.py` 的 import 路径(函数体不变,测试基本不动)
|
||||
- 验证:`pf clr`/`pf which python` 端到端 + 测试通过
|
||||
|
||||
### 阶段3:迁移多 subcommand fn 工具
|
||||
- 工具:pdftool(14子命令)/piptool/packtool/filedate/filelevel/folderback/folderzip/sshcopyid/autofmt/bumpversion
|
||||
- pdftool:函数签名已有完整类型注解,迁移最直接
|
||||
- piptool:`i`/`up` 的 cmd 改 fn 模式(`subprocess.run`)
|
||||
- bumpversion:positional `PART` 默认 "patch" + `--no-tag` store_true
|
||||
- 验证:每工具 `pf <tool> --list` + 至少一个 subcommand 端到端
|
||||
|
||||
### 阶段4:迁移复杂工具
|
||||
- pymake:12 cmd job + 6 聚合(ba/bump/cov/tc/p/pb)+ 4 内部 job(`hidden=True`:pyrefly_check/git_add_all/git_push/test_coverage)
|
||||
- envdev:9 job,4 处 `allow_upstream_skip`;`install_rust` 用 needs + allow_upstream_skip
|
||||
- gittool:`a`/`i`/`isub` 是 fn,`c`/`p`/`pl` 是 cmd,`clean` 是 hidden cmd
|
||||
- sglang/msdownload:跨平台 + 多参数
|
||||
- 验证:`pf pymake tc`/`pf pymake b`/`pf envdev` 端到端
|
||||
|
||||
### 阶段5:清理 YAML
|
||||
- 删除:`yaml_loader.py`、`configs/*.yaml`(22 个)、`cli/yamlrun.py`、`tests/test_yaml_loader.py`
|
||||
- 删除:`ops/{files,dev,media,system,llm,bumpversion}.py`(被 `ops/<tool>.py` 替代后)
|
||||
- 删除:`registry.py` 的 `register_fn`/`get_fn`/`has_fn`/`FnRegistry`(YAML 配套)
|
||||
- `__init__.py`:移除 `YamlLoadError`/`build_cli_parser`/`load_yaml`/`parse_yaml_string`/`run_cli`/`run_yaml`/`Graph.from_yaml`/`register_fn`/`get_fn`/`has_fn`/`FnRegistry`;新增 `tool`/`ToolSpec`/`run_tool`
|
||||
- `graph.py`:删 `from_yaml` 类方法
|
||||
- `pyproject.toml`:删 `yamlrun` 脚本入口、`pyyaml` 运行时依赖、`types-PyYAML` dev 依赖
|
||||
- 验证:全局无 `import yaml`、无 `load_yaml`/`run_cli` 残留引用
|
||||
|
||||
### 阶段6:补充测试,覆盖率达标
|
||||
- `tests/test_tools.py` 扩展:覆盖所有装饰器分支
|
||||
- `tests/cli/test_*.py`(16 个):改为通过 `PfApp([tool, subcommand, ...]).run()` 或 `run_tool(...)` 测试
|
||||
- 删除 `tests/test_yaml_loader.py`
|
||||
- 验证:`pf pymake cov` ≥95% branch
|
||||
|
||||
## 关键文件
|
||||
|
||||
**新增**:
|
||||
- `src/pyflowx/tools.py`(核心,@px.tool 装饰器)
|
||||
- `src/pyflowx/ops/<tool>.py`(22 个工具模块)
|
||||
- `src/pyflowx/ops/_common.py`(跨工具复用函数)
|
||||
- `tests/test_tools.py`
|
||||
|
||||
**修改**:
|
||||
- `src/pyflowx/cli/pf.py`(_run_yaml → _run_tool,加载 ops/<tool>.py)
|
||||
- `src/pyflowx/__init__.py`(API 导出调整)
|
||||
- `pyproject.toml`(删 yamlrun/pyyaml)
|
||||
- `tests/cli/test_*.py`(import 路径 + 测试入口)
|
||||
|
||||
**删除**:
|
||||
- `src/pyflowx/yaml_loader.py`
|
||||
- `src/pyflowx/configs/*.yaml`(22 个)
|
||||
- `src/pyflowx/cli/yamlrun.py`
|
||||
- `tests/test_yaml_loader.py`
|
||||
- `src/pyflowx/ops/{files,dev,media,system,llm,bumpversion}.py`(被替代后)
|
||||
- `src/pyflowx/registry.py`(YAML 配套)
|
||||
|
||||
## 可复用的现有实现
|
||||
|
||||
- `_collect_with_deps`(yaml_loader.py:287):needs 传递依赖收集,迁到 tools.py
|
||||
- `_add_global_options`(yaml_loader.py:1017):全局选项注入,思路复用
|
||||
- `CliExitCode`(runner.py:30):退出码三态,直接复用
|
||||
- `px.Graph.from_specs` + `px.run`:DAG 构建与执行,直接复用
|
||||
- `TaskSpec`(task.py):任务描述符,@px.tool 内部构建它
|
||||
|
||||
## CLI 兼容性
|
||||
|
||||
- 57 个 pf 别名不变
|
||||
- 子命令名不变(pdftool m/s/c/...、pymake b/ba/tc/...)
|
||||
- 参数名不变(`--output`/`--password`/...,靠 snake_case→kebab-case 转换)
|
||||
- `--dry-run`/`--quiet`/`--strategy`/`--list` 全局选项保留
|
||||
- 退出码 0/1/130 三态保留
|
||||
|
||||
## 风险与权衡
|
||||
|
||||
- **动态 cmd 改 fn**:失去 cmd 任务的 verbose 输出(命令打印/returncode),但模型更统一;verbose 可在 fn 内手动 print
|
||||
- **Graph.from_yaml 移除**:破坏性公共 API 变更,changelog 说明;yamlrun 用户迁移到 pf
|
||||
- **ops 重组工作量大**:22 个工具模块从 6 个功能模块拆出,跨工具函数需识别;分阶段(2-4)渐进,每阶段可独立验证
|
||||
- **register_fn 移除**:20 个未被引用的函数降级私有,被 @px.tool 间接调用的内联或放 `_common.py`
|
||||
|
||||
## 验证
|
||||
|
||||
每阶段结束运行:
|
||||
```bash
|
||||
uv run ruff check . && uv run ruff format --check .
|
||||
uv run pyrefly check .
|
||||
uv run pytest --cov=pyflowx --cov-branch --cov-report=term-missing
|
||||
```
|
||||
|
||||
端到端验证(阶段4 后):
|
||||
```bash
|
||||
pf clr # 单命令
|
||||
pf which python # 单命令 + positional
|
||||
pf pdftool m a.pdf b.pdf --output merged.pdf # 多 subcommand + positional nargs+
|
||||
pf pymake tc # DAG + 聚合 + 内部 job
|
||||
pf envdev --python-mirror aliyun # 跨平台 fn
|
||||
pf # 列出所有工具
|
||||
```
|
||||
|
||||
最终:`pf pymake cov` 通过,覆盖率 ≥95%。
|
||||
@@ -0,0 +1,228 @@
|
||||
# 方案C 续接执行计划
|
||||
|
||||
## Summary
|
||||
|
||||
承接已批准的 `.trae/documents/方案C-迁移执行计划.md`。前序会话已完成**阶段1**(tools.py 测试 + 公共 API 导出)与**阶段2 的大部分工作**(7 个新 ops 模块创建 + pf.py 双路由 + 4 个测试文件 import 更新),但因上下文中断尚未收尾。本计划从当前状态起,完成阶段2 收尾,并继续执行阶段3-6,最终完全移除 YAML 支持,覆盖率 ≥95%。
|
||||
|
||||
## Current State Analysis
|
||||
|
||||
### 阶段1(已完成)
|
||||
- `src/pyflowx/tools.py`:完整实现 @px.tool 装饰器、ToolSpec、_TOOL_REGISTRY、CLI 生成、依赖收集、run_tool、三态退出码
|
||||
- `tests/test_tools.py`:110 测试全通过,覆盖率 96.91%
|
||||
- `src/pyflowx/__init__.py`:已导出 `tool`/`ToolSpec`/`run_tool`/`list_tools`/`list_subcommands`(同时保留 YAML API 供过渡)
|
||||
- `pyproject.toml`:已加 `[tool.ruff.lint.per-file-ignores]` 中 `src/pyflowx/tools.py = ["PLR0911"]`
|
||||
|
||||
### 阶段2(进行中,待收尾)
|
||||
**已完成**:
|
||||
- 7 个新 ops 模块创建完成,均使用 `@px.tool` 装饰器:
|
||||
- `ops/clr.py`(`clear_screen_run`)
|
||||
- `ops/which.py`(`which_run`)
|
||||
- `ops/taskkill.py`(`taskkill_run`)
|
||||
- `ops/reseticoncache.py`(`reset_icon_cache_run`)
|
||||
- `ops/screenshot.py`(`take_screenshot_full`/`take_screenshot_area`,双 subcommand)
|
||||
- `ops/dockercmd.py`(`docker_login_tencent`,subcommand="login")
|
||||
- `ops/lscalc.py`(`run_ls_dyna`/`run_ls_dyna_mpi`/`check_ls_dyna_status`,三 subcommand)
|
||||
- `pf.py` 已改双路由:`_run_yaml` → `_run_tool_or_yaml`(@px.tool 优先,YAML 回退)
|
||||
- 4 个测试文件 import 路径已更新:`test_reseticoncache.py`、`test_screenshot.py`、`test_lscalc.py`、`test_system_run.py`
|
||||
- 旧功能模块(`ops/system.py`、`ops/media.py`、`ops/dev.py`)中的原函数仍保留(`@px.register_fn`),供未迁移的 YAML 工具使用
|
||||
|
||||
**待完成**:
|
||||
1. **test_envdev.py 无需改动**:该文件测试 `ops/dev.py` 中的 `docker_login_tencent` 等 10 个函数,这些函数在阶段4 前仍在 `dev.py` 中(`@px.register_fn` 装饰),`dockercmd.py` 中的 `@px.tool` 版本是独立副本,两者共存无冲突
|
||||
2. **删除 7 个 YAML 配置文件**:`configs/{clr,which,taskkill,reseticoncache,screenshot,dockercmd,lscalc}.yaml`
|
||||
3. **验证**:运行 ruff/pyrefly/pytest,确认阶段2 通过
|
||||
|
||||
### 阶段3-6(未开始)
|
||||
- 阶段3:迁移 10 个多 subcommand fn 工具(pdftool/piptool/packtool/filedate/filelevel/folderback/folderzip/sshcopyid/autofmt/bumpversion)
|
||||
- 阶段4:迁移 5 个复杂工具(pymake/envdev/gittool/sglang/msdownload)
|
||||
- 阶段5:清理 YAML 与配套代码(yaml_loader.py/registry.py/configs 目录/旧 ops 功能模块)
|
||||
- 阶段6:补充测试,覆盖率达标 95%
|
||||
|
||||
## Proposed Changes
|
||||
|
||||
### 阶段2 收尾:删除 YAML + 验证
|
||||
|
||||
**删除文件**(7 个):
|
||||
- `src/pyflowx/configs/clr.yaml`
|
||||
- `src/pyflowx/configs/which.yaml`
|
||||
- `src/pyflowx/configs/taskkill.yaml`
|
||||
- `src/pyflowx/configs/reseticoncache.yaml`
|
||||
- `src/pyflowx/configs/screenshot.yaml`
|
||||
- `src/pyflowx/configs/dockercmd.yaml`
|
||||
- `src/pyflowx/configs/lscalc.yaml`
|
||||
|
||||
**验证**:
|
||||
```bash
|
||||
uv run ruff check . && uv run ruff format --check .
|
||||
uv run pyrefly check .
|
||||
uv run pytest tests/cli/test_{reseticoncache,screenshot,lscalc,system_run}.py -v
|
||||
uv run pytest tests/test_tools.py -v
|
||||
```
|
||||
|
||||
端到端冒烟(可选,若环境允许):
|
||||
```bash
|
||||
pf clr
|
||||
pf which python
|
||||
pf lscalc --list
|
||||
```
|
||||
|
||||
### 阶段3:迁移多 subcommand fn 工具(10 个)
|
||||
|
||||
**工具清单**:`pdftool`(14 子命令)/`piptool`/`packtool`/`filedate`/`filelevel`/`folderback`/`folderzip`/`sshcopyid`/`autofmt`/`bumpversion`
|
||||
|
||||
**新建文件**(10 个,`ops/<tool>.py`,一文件一工具):
|
||||
- `ops/pdftool.py`:14 个 `@px.tool("pdftool", subcommand="...")`,函数体来自 `ops/media.py`(`pdf_merge`/`pdf_split`/`pdf_compress`/`pdf_encrypt`/`pdf_decrypt`/`pdf_extract_text`/`pdf_extract_images`/`pdf_add_watermark`/`pdf_rotate`/`pdf_crop`/`pdf_info`/`pdf_ocr`/`pdf_reorder`/`pdf_to_images`/`pdf_repair`)
|
||||
- `ops/piptool.py`:多 subcommand(`i`/`up`/`un`/`re`/`fr`/`dl`),`i`/`up` 原 cmd 模式改 fn 模式
|
||||
- `ops/packtool.py`:来自 `ops/system.py`(`pack_source`/`pack_wheel`/`pack_dependencies`/`clean_build_dir`)
|
||||
- `ops/filedate.py`/`ops/filelevel.py`/`ops/folderback.py`/`ops/folderzip.py`:来自 `ops/files.py`
|
||||
- `ops/sshcopyid.py`:来自 `ops/system.py`(`ssh_copy_id`)
|
||||
- `ops/autofmt.py`:4 subcommand(`fmt`/`lint`/`doc`/`sync`),`fmt`/`lint` 原 cmd 改 fn 模式
|
||||
- `ops/bumpversion.py`:positional `part: str = "patch"` + `no_tag: bool = False`,直接调用 `bump_project_version`
|
||||
|
||||
**关键决策**:
|
||||
- 函数签名保持不变(测试兼容),仅装饰器 `@px.register_fn` → `@px.tool(name, subcommand=...)` 变化
|
||||
- 参数名 snake_case,CLI 自动转 kebab-case,**保持 CLI 兼容**
|
||||
- `piptool i`/`up`:原 `cmd: ["pip","install","${PKG}"]` 改为 `def i(packages: list[str]) -> None: subprocess.run(["pip","install",*packages], check=True)`
|
||||
- `autofmt fmt`/`lint`:原 `cmd: ["ruff","format","${TARGET}"]` 改为 `def fmt(target: str = ".") -> None: subprocess.run(["ruff","format",target], check=True)`
|
||||
- `bumpversion`:原 `cmd: ["pf","bumpversion","patch"]` 改为直接调用 `bump_project_version` 函数(避免递归调用 pf)
|
||||
|
||||
**测试更新**(10 个):
|
||||
- `tests/cli/test_pdftool.py`、`test_piptool.py`、`test_packtool.py`、`test_filedate.py`、`test_filelevel.py`、`test_folderback.py`、`test_folderzip.py`、`test_sshcopyid.py`、`test_autofmt.py`、`test_bumpversion.py`
|
||||
- import 路径:`from pyflowx.ops import <tool>` 或 `from pyflowx.ops.<tool> import <func>`
|
||||
- 函数调用不变(函数体未变)
|
||||
|
||||
**删除文件**(10 个):
|
||||
- `configs/{pdftool,piptool,packtool,filedate,filelevel,folderback,folderzip,sshcopyid,autofmt,bumpversion}.yaml`
|
||||
|
||||
**验证**:每工具 `pf <tool> --list` + 对应 cli 测试通过
|
||||
|
||||
### 阶段4:迁移复杂工具(5 个)
|
||||
|
||||
**工具清单**:`pymake`/`envdev`/`gittool`/`sglang`/`msdownload`
|
||||
|
||||
**新建文件**(5 个):
|
||||
- `ops/pymake.py`:
|
||||
- 12 个 cmd subcommand(`b`/`bc`/`sync`/`c`/`t`/`tf`/`bumpversion`/`bumpmi`/`doc`/`lint`/`tox`):`@px.tool("pymake", subcommand="b", cmd=["uv","build"])` + 函数体 `pass` 或 docstring
|
||||
- hidden 内部 job(`test_coverage`/`pyrefly_check`/`git_add_all`/`git_push`/`git_push_tags`/`twine_publish`/`publish_python`):`hidden=True`,`cmd=[...]`
|
||||
- 6 个聚合(`ba`/`bump`/`cov`/`tc`/`p`/`pb`):`needs=[...]` + 函数体 `pass`
|
||||
- `cwd: Path = Path(".")` 作为函数签名参数,CLI `--cwd` 覆盖
|
||||
- `ops/envdev.py`:9 个 fn subcommand,来自 `ops/dev.py`;`install_rust` 用 `needs=["setup_rust","download_rustup"]` + `allow_upstream_skip=True`
|
||||
- `ops/gittool.py`:`a`/`i`/`isub` 是 fn,`c`/`p`/`pl` 是 cmd,`clean` 是 `hidden=True` cmd
|
||||
- `ops/sglang.py`/`ops/msdownload.py`:跨平台 fn,来自 `ops/llm.py`,`Constants.IS_LINUX`/`IS_WINDOWS` 判断在函数体内
|
||||
|
||||
**关键决策**:
|
||||
- pymake 的 `cwd` 参数:`TaskSpec` 的 `cwd` 由 `_build_task_spec` 从函数签名参数 `cwd: Path = Path(".")` 取,CLI `--cwd` 透传
|
||||
- pymake `bumpversion` 子命令:保持 cmd 模式 `cmd=["pf","bumpversion","patch"]`(调用外部 pf 命令)
|
||||
- envdev 跨平台逻辑保留在函数体内,仅装饰器迁移
|
||||
- gittool `c`(clean)是 hidden,被 `tc`/`p` 通过 needs 引用
|
||||
|
||||
**测试更新**(5 个):
|
||||
- `tests/cli/test_envdev.py`:`docker_login_tencent` import 从 `pyflowx.ops.dev` 改 `pyflowx.ops.dockercmd`;其余 envdev 函数从 `pyflowx.ops.envdev` 导入
|
||||
- `tests/cli/test_gittool.py`、`test_pymake.py`(若存在)、`test_llm.py`:import 路径改 `from pyflowx.ops import <tool>`
|
||||
|
||||
**删除文件**(5 个):
|
||||
- `configs/{pymake,envdev,gittool,sglang,msdownload}.yaml`
|
||||
|
||||
**验证**:`pf pymake tc`、`pf envdev --python-mirror aliyun`、`pf gittool a -m "msg"` 端到端;cli 测试通过
|
||||
|
||||
### 阶段5:清理 YAML 与配套代码
|
||||
|
||||
**删除文件**:
|
||||
- `src/pyflowx/yaml_loader.py`
|
||||
- `src/pyflowx/cli/yamlrun.py`
|
||||
- `src/pyflowx/registry.py`
|
||||
- `src/pyflowx/configs/`(整个目录,22 个 .yaml 已在前阶段删完,删目录本身)
|
||||
- `src/pyflowx/ops/{files,dev,media,system,llm,bumpversion}.py`(6 个功能模块,被 22 个工具模块替代)
|
||||
- `tests/test_yaml_loader.py`
|
||||
- `tests/test_registry.py`
|
||||
|
||||
**修改文件**:
|
||||
- `src/pyflowx/__init__.py`:
|
||||
- 删导入:`YamlLoadError`/`build_cli_parser`/`load_yaml`/`parse_yaml_string`/`run_cli`/`run_yaml`/`FnRegistry`/`get_fn`/`has_fn`/`register_fn`
|
||||
- 删 `__all__` 对应条目
|
||||
- `src/pyflowx/graph.py`:删 `from_yaml` 类方法(若存在)
|
||||
- `src/pyflowx/cli/pf.py`:
|
||||
- `_run_tool_or_yaml` → `_run_tool`:仅走 `importlib.import_module` + `run_tool`,删除 YAML 回退分支
|
||||
- `_LEGACY_TOOLS` 删 `yamlrun`(保留 `emlman`/`profiler`/`pxp`)
|
||||
- `_list_tools` 改为遍历 `_TOOL_REGISTRY` 取 description(不再读 YAML)
|
||||
- `_tool_description` 删除(不再读 YAML)
|
||||
- 模块 docstring 更新(去掉"YAML 配置"描述)
|
||||
- `src/pyflowx/ops/__init__.py`:从导入 6 个功能模块改为导入 22 个工具模块
|
||||
- `pyproject.toml`:
|
||||
- `[project.scripts]` 删 `yamlrun`
|
||||
- `[project] dependencies` 删 `pyyaml>=6.0.1`
|
||||
- `[project.optional-dependencies] dev` 删 `types-PyYAML>=6.0.12`
|
||||
|
||||
**全局检查**:
|
||||
```bash
|
||||
# 无残留 YAML 引用
|
||||
grep -rE "import yaml|load_yaml|run_cli|register_fn|YamlLoadError|build_cli_parser|parse_yaml_string|from_yaml|yamlrun" src/ tests/ || echo "OK"
|
||||
```
|
||||
|
||||
### 阶段6:补充测试 + 覆盖率达标
|
||||
|
||||
**扩展 `tests/test_tools.py`**:
|
||||
- 补齐阶段2-4 迁移中发现的边界(如 `cwd` 从 CLI 透传、`env`/`retry`/`timeout` 透传 TaskSpec、`_has_function_logic` 对 async 函数/类方法、`_unwrap_optional` 对 `X | None` 等)
|
||||
- 若 `tools.py` 覆盖率未达 95%,补齐
|
||||
|
||||
**新增 `tests/cli/test_pf_routing.py`**:
|
||||
- 测试 `PfApp` 路由:已知别名 → `_run_tool`、legacy → `_run_legacy`、未知 → 退出码 1
|
||||
- 测试 `_list_tools` 输出格式
|
||||
|
||||
**更新 `tests/cli/test_*.py`**:
|
||||
- 阶段2-4 已更新 import 路径,本阶段补充通过 `run_tool(...)` 或 `PfApp([tool, subcommand]).run()` 的端到端测试
|
||||
|
||||
**覆盖率验证**:
|
||||
```bash
|
||||
uv run pytest --cov=pyflowx --cov-branch --cov-report=term-missing
|
||||
```
|
||||
- 目标:`pyflowx.tools` ≥95%、整体 ≥95%(branch)
|
||||
- 若 `ops/<tool>.py` 因跨平台分支未覆盖,补 mock 测试(`Constants.IS_LINUX`/`IS_WINDOWS`)
|
||||
|
||||
**最终门禁**:
|
||||
```bash
|
||||
uv run ruff check . && uv run ruff format --check .
|
||||
uv run pyrefly check .
|
||||
uv run pytest --cov=pyflowx --cov-branch --cov-report=term-missing
|
||||
```
|
||||
全部门禁通过,覆盖率 ≥95%。
|
||||
|
||||
## Assumptions & Decisions
|
||||
|
||||
1. **延续已批准计划**:本计划是 `.trae/documents/方案C-迁移执行计划.md` 的续接,不改变其架构决策,仅推进执行
|
||||
2. **test_envdev.py 阶段2 不改**:该文件测试 `ops/dev.py` 函数,dev.py 在阶段4 前仍存在,无需改动;阶段4 迁移 envdev 时再统一更新
|
||||
3. **函数体复制策略**:阶段3-4 从旧功能模块复制函数体到新工具模块,旧模块在阶段5 统一删除,避免提前破坏未迁移工具
|
||||
4. **CLI 兼容**:57 个 pf 别名不变、子命令名不变、参数名(snake_case → kebab-case)不变、全局选项保留、退出码三态保留
|
||||
5. **cmd 任务改 fn 模式**:`piptool i`/`autofmt fmt` 等原 `cmd+${VAR}` 改为函数体内 `subprocess.run`,verbose 由函数内 `print` 补偿
|
||||
6. **聚合任务判定**:`_has_function_logic` 用 ast 分析函数体(pass/.../docstring 视为无逻辑),迁移时聚合任务的函数体写 `pass`
|
||||
7. **每阶段独立验证**:每阶段结束运行 ruff + pyrefly + pytest,确保不累积错误
|
||||
8. **自动提交**:每阶段完成且门禁通过后自动 git commit + push(分支已跟踪远程),遵循中文提交信息风格
|
||||
|
||||
## Verification Steps
|
||||
|
||||
每阶段结束运行:
|
||||
```bash
|
||||
uv run ruff check . && uv run ruff format --check .
|
||||
uv run pyrefly check .
|
||||
uv run pytest tests/ --cov=pyflowx --cov-branch --cov-report=term-missing -x
|
||||
```
|
||||
|
||||
端到端验证(阶段4 后):
|
||||
```bash
|
||||
pf clr # 单命令
|
||||
pf which python # 单命令 + positional
|
||||
pf pdftool m a.pdf b.pdf --output merged.pdf # 多 subcommand + positional nargs+
|
||||
pf pdftool --list # --list 输出
|
||||
pf pymake tc # DAG + 聚合 + 内部 job(hidden)
|
||||
pf pymake b --cwd /tmp # cwd CLI 透传
|
||||
pf envdev --python-mirror aliyun # 跨平台 fn
|
||||
pf gittool a -m "msg" # fn subcommand
|
||||
pf # 列出所有工具(从 _TOOL_REGISTRY)
|
||||
pf unknown_tool # 退出码 1
|
||||
```
|
||||
|
||||
阶段5 后全局清理验证:
|
||||
```bash
|
||||
grep -rE "import yaml|load_yaml|run_cli|register_fn|YamlLoadError|build_cli_parser|parse_yaml_string|from_yaml|yamlrun" src/ tests/ || echo "OK"
|
||||
```
|
||||
|
||||
阶段6 最终:`pf pymake cov` 通过,覆盖率 ≥95%(branch)。
|
||||
@@ -0,0 +1,251 @@
|
||||
# 方案C 迁移执行计划
|
||||
|
||||
## Summary
|
||||
|
||||
承接已批准的高层设计 `.trae/documents/方案C-移除YAML-统一px-tool装饰器.md`。当前 `src/pyflowx/tools.py` 已完整实现(@px.tool 装饰器 + ToolSpec + 注册表 + CLI 生成 + 依赖收集 + run_tool 入口),其余所有代码仍处于旧 YAML 模式。本计划从当前状态起,执行 6 个阶段,完全移除 YAML 支持,统一用 `@px.tool` 装饰器,覆盖率 ≥95%。
|
||||
|
||||
## Current State Analysis
|
||||
|
||||
**已完成**:
|
||||
- `src/pyflowx/tools.py`(607 行):ToolSpec/`@tool`/注册表/`_add_func_args_to_parser`/`_collect_with_deps`/`_build_task_spec`/`run_tool`,完整实现三态退出码(0/1/130)
|
||||
|
||||
**未完成(本计划覆盖)**:
|
||||
1. `tests/test_tools.py` 不存在
|
||||
2. `__init__.py` 仍导出 YAML API(`load_yaml`/`run_cli`/`YamlLoadError`/`build_cli_parser`/`parse_yaml_string`/`run_yaml`)+ `register_fn`/`get_fn`/`has_fn`/`FnRegistry`,未导出 `tool`/`ToolSpec`/`run_tool`/`list_tools`/`list_subcommands`
|
||||
3. `pf.py` 仍走 `_run_yaml` → `px.run_cli(config_path, argv)`,57 个别名指向 .yaml 文件;`_LEGACY_TOOLS` 含 `yamlrun`
|
||||
4. `ops/` 仍是 6 个功能模块(`files/dev/media/system/llm/bumpversion`),共 79 个 `@px.register_fn`;无 22 个工具模块
|
||||
5. `configs/` 仍有 22 个 .yaml 文件
|
||||
6. `tests/cli/test_*.py`(16 个)+ `tests/test_yaml_loader.py`(49KB)+ `tests/test_registry.py` 仍依赖旧结构
|
||||
7. `pyproject.toml` 仍含 `yamlrun` 脚本入口、`pyyaml` 运行时依赖、`types-PyYAML` dev 依赖
|
||||
8. `src/pyflowx/yaml_loader.py`、`src/pyflowx/cli/yamlrun.py`、`src/pyflowx/registry.py` 仍存在
|
||||
9. `graph.py` 仍含 `from_yaml` 类方法(若存在)
|
||||
|
||||
**关键依赖关系**:
|
||||
- `register_fn` 函数被 YAML `fn:` 字段引用,迁移后改用 `@px.tool` 直接持有函数对象,不再需要 `register_fn`/`get_fn`/`has_fn`
|
||||
- 现有 `tests/cli/test_*.py` 直接调用 `media.pdf_merge(...)`/`dev._get_installed_packages()` 等函数体,迁移后函数体不变,仅 import 路径与装饰器变化
|
||||
- `tools.py` 中 `run_tool` 依赖 `px.Graph.from_specs` + `px.run`,与 YAML 无耦合,可直接使用
|
||||
|
||||
## Proposed Changes
|
||||
|
||||
### 阶段1:tools.py 单元测试 + 公共 API 导出
|
||||
|
||||
**文件**:
|
||||
- 新建 `tests/test_tools.py`
|
||||
- 修改 `src/pyflowx/__init__.py`
|
||||
|
||||
**test_tools.py 覆盖点**(对应 `tools.py` 各分支):
|
||||
1. 注册:`@tool` 装饰器注册到 `_TOOL_REGISTRY`,`list_tools`/`list_subcommands`/`get_tool`/`clear_tool_registry`
|
||||
2. 重复检测:同名同 subcommand 二次注册抛 `ValueError`
|
||||
3. CLI 生成各类型:
|
||||
- positional(无默认值)/ option(有默认值)
|
||||
- `list[Path]` → `nargs="+"`
|
||||
- `Literal["a","b"]` → `choices`
|
||||
- `int`/`float`/`Path`/`str` → `type=`
|
||||
- `bool` 默认 False → `--flag store_true`;默认 True → `--no-flag store_false`
|
||||
- `Optional[X]` → `_unwrap_optional`
|
||||
4. 单命令工具执行:`run_tool` 返回 0(成功)/1(失败)/130(KeyboardInterrupt)
|
||||
5. cmd 分支:`cmd=["..."]` 时构建 TaskSpec 用 cmd,函数体不执行
|
||||
6. fn 分支:无 cmd 时执行函数体,kwargs 透传
|
||||
7. 聚合任务:`_is_aggregate` 判定 pass/.../docstring,fn=`_noop`
|
||||
8. DAG:`_collect_with_deps` BFS 拓扑顺序(依赖在前)
|
||||
9. hidden:`list_subcommands(include_hidden=False)` 排除,`_build_parser` 不暴露
|
||||
10. 全局选项:`--dry-run`/`--quiet`/`--strategy`/`--list`
|
||||
11. 退出码三态:成功 0、`TaskFailedError` → 1、`PyFlowXError` → 1、`KeyboardInterrupt` → 130
|
||||
12. 未注册工具:`run_tool("unknown")` → 1,stderr 提示
|
||||
13. `_handle_list` 输出格式
|
||||
14. `_snake_to_kebab` 转换
|
||||
|
||||
**__init__.py 改动**:
|
||||
- 新增导入:`from .tools import ToolSpec, list_subcommands, list_tools, run_tool, tool`
|
||||
- 新增 `__all__` 条目:`ToolSpec`/`list_subcommands`/`list_tools`/`run_tool`/`tool`
|
||||
- 暂时**保留** YAML/register_fn 导出(阶段5 才删),避免提前破坏
|
||||
|
||||
**验证**:`uv run pytest tests/test_tools.py --cov=pyflowx.tools --cov-branch` 通过,`tools.py` 覆盖率 ≥95%
|
||||
|
||||
### 阶段2:迁移简单单命令工具(7 个)
|
||||
|
||||
**工具**:`clr`/`which`/`taskkill`/`reseticoncache`/`screenshot`/`dockercmd`/`lscalc`
|
||||
|
||||
**新建文件**(`ops/<tool>.py`,一文件一工具):
|
||||
- `ops/clr.py`:`@px.tool("clr")` 包装 `clear_screen_run`(来自 `ops/system.py`)
|
||||
- `ops/which.py`:`@px.tool("which")` 包装 `which_run`
|
||||
- `ops/taskkill.py`:`@px.tool("taskkill")` 包装 `taskkill_run`
|
||||
- `ops/reseticoncache.py`:`@px.tool("reseticoncache")` 包装 `reset_icon_cache_run`
|
||||
- `ops/screenshot.py`:`@px.tool("screenshot")` 包装 `take_screenshot_full`/`take_screenshot_area`(两个 subcommand)
|
||||
- `ops/dockercmd.py`:`@px.tool("dockercmd")` 包装 `docker_login_tencent`
|
||||
- `ops/lscalc.py`:`@px.tool("lscalc")` 包装 `run_ls_dyna`/`run_ls_dyna_mpi`/`check_ls_dyna_status`(多 subcommand)
|
||||
|
||||
**函数迁移策略**:
|
||||
- 从 `ops/system.py` 把对应函数体复制到新 `ops/<tool>.py`,改 `@px.register_fn` 为 `@px.tool(...)`
|
||||
- 函数签名保持不变(测试兼容),仅装饰器变化
|
||||
- 跨工具复用的常量(`LS_DYNA_COMMANDS` 等)若被多工具引用,放 `ops/_common.py`;否则就地保留
|
||||
|
||||
**测试更新**:
|
||||
- `tests/cli/test_reseticoncache.py`、`test_screenshot.py`、`test_lscalc.py`、`test_system_run.py`:import 路径从 `from pyflowx.ops import system` 改为 `from pyflowx.ops import <tool>`
|
||||
- 函数调用不变(`system.clear_screen_run()` → `clr.clear_screen_run()` 或类似)
|
||||
|
||||
**删除**:`configs/{clr,which,taskkill,reseticoncache,screenshot,dockercmd,lscalc}.yaml`(7 个)
|
||||
|
||||
**验证**:`pf clr`、`pf which python`、`pf lscalc --list` 端到端可用;`uv run pytest tests/cli/test_{reseticoncache,screenshot,lscalc,system_run}.py` 通过
|
||||
|
||||
### 阶段3:迁移多 subcommand fn 工具(10 个)
|
||||
|
||||
**工具**:`pdftool`(14 子命令)/`piptool`/`packtool`/`filedate`/`filelevel`/`folderback`/`folderzip`/`sshcopyid`/`autofmt`/`bumpversion`
|
||||
|
||||
**新建文件**:
|
||||
- `ops/pdftool.py`:14 个 `@px.tool("pdftool", subcommand="m"/"s"/"c"/...)`,函数来自 `ops/media.py`
|
||||
- `ops/piptool.py`:多 subcommand(`i`/`up`/`un`/`re`/`fr`/`dl`),`i`/`up` 原 cmd 模式改 fn 模式(`subprocess.run` 在函数体内)
|
||||
- `ops/packtool.py`:多 subcommand,来自 `ops/system.py`(`pack_source`/`pack_wheel`/`pack_dependencies`/`clean_build_dir`)
|
||||
- `ops/filedate.py`/`filelevel.py`/`folderback.py`/`folderzip.py`:来自 `ops/files.py`
|
||||
- `ops/sshcopyid.py`:来自 `ops/system.py`(`ssh_copy_id`)
|
||||
- `ops/autofmt.py`:4 subcommand(`fmt`/`lint`/`doc`/`sync`),`fmt`/`lint` 原 cmd 改 fn 模式
|
||||
- `ops/bumpversion.py`:positional `PART` 默认 "patch" + `--no-tag store_true`,来自 `ops/bumpversion.py`
|
||||
|
||||
**关键决策**:
|
||||
- pdftool 函数签名已有完整类型注解,迁移最直接;`pdf_merge(input_paths: list[Path], output_path: Path)` → `@px.tool("pdftool", subcommand="m")` + `inputs: list[Path]`(positional nargs="+") + `output: Path = Path("merged.pdf")`
|
||||
- 参数名映射:YAML 的 `INPUTS`/`OUTPUT`/`PASSWORD` → snake_case `inputs`/`output`/`password`,CLI 自动转 kebab-case `--output`/`--password`,**保持 CLI 兼容**
|
||||
- `piptool i`/`up`:原 `cmd: ["pip", "install", "${PKG}"]` 改为 `def i(packages: list[str]) -> None: subprocess.run(["pip", "install", *packages], check=True)`
|
||||
- `autofmt fmt`/`lint`:原 `cmd: ["ruff", "format", "${TARGET}"]` 改为 `def fmt(target: str = ".") -> None: subprocess.run(["ruff", "format", target], check=True)`
|
||||
- bumpversion:positional `part: str = "patch"` + `no_tag: bool = False`(store_true),原 `cmd: ["pf", "bumpversion", "patch"]` 改为直接调用 `bump_project_version` 函数
|
||||
|
||||
**测试更新**:
|
||||
- `tests/cli/test_pdftool.py`、`test_piptool.py`、`test_packtool.py`、`test_filedate.py`、`test_filelevel.py`、`test_folderback.py`、`test_folderzip.py`、`test_sshcopyid.py`、`test_autofmt.py`、`test_bumpversion.py`:import 路径改 `from pyflowx.ops import <tool>`
|
||||
- 函数调用不变(函数体未变)
|
||||
|
||||
**删除**:`configs/{pdftool,piptool,packtool,filedate,filelevel,folderback,folderzip,sshcopyid,autofmt,bumpversion}.yaml`(10 个)
|
||||
|
||||
**验证**:每工具 `pf <tool> --list` + 至少一个 subcommand 端到端;对应 cli 测试通过
|
||||
|
||||
### 阶段4:迁移复杂工具(5 个)
|
||||
|
||||
**工具**:`pymake`/`envdev`/`gittool`/`sglang`/`msdownload`
|
||||
|
||||
**新建文件**:
|
||||
- `ops/pymake.py`:
|
||||
- 12 个 cmd subcommand(`b`/`bc`/`sync`/`c`/`t`/`tf`/`bumpversion`/`bumpmi`/`doc`/`lint`/`tox`):`@px.tool("pymake", subcommand="b", cmd=["uv","build"])` + `def b(cwd: Path = Path(".")) -> None: pass`
|
||||
- 4 个 hidden 内部 job(`test_coverage`/`pyrefly_check`/`git_add_all`/`git_push`/`git_push_tags`/`twine_publish`/`publish_python`):`hidden=True`,`cmd=[...]`
|
||||
- 6 个聚合(`ba`/`bump`/`cov`/`tc`/`p`/`pb`):`needs=[...]` + 函数体 `pass`
|
||||
- `cwd: Path = Path(".")` 作为函数签名参数,CLI `--cwd` 覆盖
|
||||
- `ops/envdev.py`:9 个 fn subcommand,4 处 `allow_upstream_skip=True`;`install_rust` 用 `needs=["setup_rust","download_rustup"]` + `allow_upstream_skip=True`
|
||||
- `ops/gittool.py`:`a`/`i`/`isub` 是 fn,`c`/`p`/`pl` 是 cmd,`clean` 是 `hidden=True` cmd
|
||||
- `ops/sglang.py`/`ops/msdownload.py`:跨平台 fn(来自 `ops/llm.py`),`Constants.IS_LINUX`/`IS_WINDOWS` 判断在函数体内
|
||||
|
||||
**关键决策**:
|
||||
- pymake 的 `cwd` 参数:`TaskSpec` 的 `cwd` 由 `_build_task_spec` 从 `variables["cwd"]` 取,CLI `--cwd` 透传;装饰器级 `cwd=None`
|
||||
- pymake `bumpversion` 子命令:原 `cmd: ["pf","bumpversion","patch"]` 改为直接 `cmd=["pf","bumpversion","patch"]`(保持 cmd 模式,因调用外部 pf 命令)
|
||||
- envdev 跨平台逻辑保留在函数体内(已实现),仅装饰器迁移
|
||||
- gittool `c`(clean)是 hidden,被 `tc`/`p` 通过 needs 引用
|
||||
|
||||
**测试更新**:
|
||||
- `tests/cli/test_envdev.py`、`test_gittool.py`、`test_llm.py`:import 路径改 `from pyflowx.ops import <tool>`
|
||||
- `test_pdftool.py` 等(若引用 gittool/pymake 的函数)同步改
|
||||
|
||||
**删除**:`configs/{pymake,envdev,gittool,sglang,msdownload}.yaml`(5 个)
|
||||
|
||||
**验证**:`pf pymake tc`、`pf pymake b`、`pf envdev --python-mirror aliyun`、`pf gittool a -m "msg"` 端到端;cli 测试通过
|
||||
|
||||
### 阶段5:清理 YAML 与配套代码
|
||||
|
||||
**删除文件**:
|
||||
- `src/pyflowx/yaml_loader.py`
|
||||
- `src/pyflowx/cli/yamlrun.py`
|
||||
- `src/pyflowx/registry.py`
|
||||
- `src/pyflowx/configs/`(整个目录,22 个 .yaml 已在前阶段删完,删目录本身)
|
||||
- `src/pyflowx/ops/{files,dev,media,system,llm,bumpversion}.py`(6 个功能模块,被 22 个工具模块替代)
|
||||
- `tests/test_yaml_loader.py`
|
||||
- `tests/test_registry.py`(若专为 register_fn 写;若含其他内容则保留并更新)
|
||||
|
||||
**修改文件**:
|
||||
- `src/pyflowx/__init__.py`:
|
||||
- 删导入:`YamlLoadError`/`build_cli_parser`/`load_yaml`/`parse_yaml_string`/`run_cli`/`run_yaml`/`FnRegistry`/`get_fn`/`has_fn`/`register_fn`
|
||||
- 删 `__all__` 对应条目
|
||||
- `src/pyflowx/graph.py`:删 `from_yaml` 类方法(若存在)
|
||||
- `src/pyflowx/cli/pf.py`:
|
||||
- `_TOOL_ALIASES` 保留 57 个别名(值已是工具名,无 .yaml 含义)
|
||||
- `_run_yaml` → `_run_tool`:`importlib.import_module(f"pyflowx.ops.{tool_name}")` 触发 `@px.tool` 注册 → `from pyflowx.tools import run_tool; return run_tool(tool_name, argv)`
|
||||
- `_LEGACY_TOOLS` 删 `yamlrun`(保留 `emlman`/`profiler`/`pxp`)
|
||||
- `_list_tools` 改为遍历 `_TOOL_REGISTRY` 取 description
|
||||
- `_tool_description` 删除(不再读 YAML)
|
||||
- `src/pyflowx/ops/__init__.py`:从导入 6 个功能模块改为导入 22 个工具模块
|
||||
- `pyproject.toml`:
|
||||
- `[project.scripts]` 删 `yamlrun`
|
||||
- `[project] dependencies` 删 `pyyaml>=6.0.1`
|
||||
- `[project.optional-dependencies] dev` 删 `types-PyYAML>=6.0.12`
|
||||
- `src/pyflowx/yaml_loader.py` 内 `import pyflowx.ops.*` 触发注册的副作用:迁移后由 `pf.py` 的 `importlib.import_module` 显式触发
|
||||
|
||||
**全局检查**:
|
||||
- `Grep` 全局 `import yaml`、`load_yaml`、`run_cli`、`run_yaml`、`register_fn`、`get_fn`、`has_fn`、`FnRegistry`、`YamlLoadError`、`build_cli_parser`、`parse_yaml_string`、`from_yaml`、`yamlrun`,确认无残留引用
|
||||
- `uv run ruff check .` + `uv run pyrefly check .` 通过
|
||||
|
||||
### 阶段6:补充测试 + 覆盖率达标
|
||||
|
||||
**扩展 `tests/test_tools.py`**:
|
||||
- 阶段1 已覆盖核心分支,本阶段补齐阶段2-4 迁移中发现的边界(如 `cwd` 从 CLI 透传、`env`/`retry`/`timeout` 透传 TaskSpec、`_has_function_logic` 对 async 函数/类方法、`_unwrap_optional` 对 `X | None`(3.10+) 等)
|
||||
- 阶段1 后若 `tools.py` 覆盖率未达 95%,本阶段补齐
|
||||
|
||||
**新增 `tests/cli/test_pf_routing.py`**:
|
||||
- 测试 `PfApp` 路由:已知别名 → `_run_tool`、legacy → `_run_legacy`、未知 → 退出码 1
|
||||
- 测试 `_list_tools` 输出格式
|
||||
|
||||
**更新 `tests/cli/test_*.py`**(16 个):
|
||||
- 阶段2-4 已更新 import 路径,本阶段补充通过 `run_tool(...)` 或 `PfApp([tool, subcommand]).run()` 的端到端测试
|
||||
- 删除 `tests/test_yaml_loader.py`、`tests/test_registry.py`(若纯 register_fn 测试)
|
||||
|
||||
**覆盖率验证**:
|
||||
```bash
|
||||
uv run pytest --cov=pyflowx --cov-branch --cov-report=term-missing
|
||||
```
|
||||
- 目标:`pyflowx.tools` ≥95%、整体 ≥95%(branch)
|
||||
- 若 `ops/<tool>.py` 因跨平台分支未覆盖,补 mock 测试(`Constants.IS_LINUX`/`IS_WINDOWS`)
|
||||
|
||||
**最终门禁**:
|
||||
```bash
|
||||
uv run ruff check . && uv run ruff format --check .
|
||||
uv run pyrefly check .
|
||||
uv run pytest --cov=pyflowx --cov-branch --cov-report=term-missing
|
||||
```
|
||||
全部门禁通过,覆盖率 ≥95%。
|
||||
|
||||
## Assumptions & Decisions
|
||||
|
||||
1. **tools.py 实现不重写**:已存在的 `tools.py` 经审查实现完整,本计划仅补测试,不改实现(除非测试发现 bug)
|
||||
2. **函数体保持不变**:迁移仅改装饰器(`@px.register_fn` → `@px.tool`)与文件位置,函数体逻辑、签名、类型注解保持不变,最大化测试兼容
|
||||
3. **CLI 兼容**:57 个 pf 别名不变、子命令名不变、参数名(snake_case → kebab-case)不变、全局选项保留、退出码三态保留
|
||||
4. **cmd 任务改 fn 模式**:`piptool i`/`autofmt fmt` 等原 `cmd+${VAR}` 改为函数体内 `subprocess.run`,失去 cmd 任务的 verbose 输出,但模型更统一;verbose 由函数内 `print` 补偿
|
||||
5. **聚合任务判定**:`_has_function_logic` 用 ast 分析函数体(pass/.../docstring 视为无逻辑),迁移时聚合任务的函数体写 `pass`
|
||||
6. **ops 重组**:6 功能模块 → 22 工具模块,跨工具复用函数(如 `add_date_prefix` 被 filedate 用)放 `ops/_common.py`;未被 YAML 引用的约 20 个 `register_fn` 辅助函数(如 `get_file_timestamp`/`has_files`)去掉装饰器降级为模块级私有
|
||||
7. **register_fn 移除时机**:阶段5 统一移除,避免阶段2-4 迁移中破坏未迁移工具
|
||||
8. **pyyaml 依赖移除**:阶段5 从 `pyproject.toml` 删除,需确认无其他模块隐式依赖(全局 grep `import yaml` 验证)
|
||||
9. **测试更新策略**:阶段2-4 每阶段更新对应 cli 测试 import,阶段6 补充端到端测试;`tests/test_yaml_loader.py` 阶段5 删除
|
||||
|
||||
## Verification Steps
|
||||
|
||||
每阶段结束运行:
|
||||
```bash
|
||||
uv run ruff check . && uv run ruff format --check .
|
||||
uv run pyrefly check .
|
||||
uv run pytest tests/ --cov=pyflowx --cov-branch --cov-report=term-missing -x
|
||||
```
|
||||
|
||||
端到端验证(阶段4 后):
|
||||
```bash
|
||||
pf clr # 单命令
|
||||
pf which python # 单命令 + positional
|
||||
pf pdftool m a.pdf b.pdf --output merged.pdf # 多 subcommand + positional nargs+
|
||||
pf pdftool --list # --list 输出
|
||||
pf pymake tc # DAG + 聚合 + 内部 job(hidden)
|
||||
pf pymake b --cwd /tmp # cwd CLI 透传
|
||||
pf envdev --python-mirror aliyun # 跨平台 fn
|
||||
pf gittool a -m "msg" # fn subcommand
|
||||
pf # 列出所有工具(从 _TOOL_REGISTRY)
|
||||
pf unknown_tool # 退出码 1
|
||||
```
|
||||
|
||||
阶段5 后全局清理验证:
|
||||
```bash
|
||||
# 无残留 YAML 引用
|
||||
grep -r "import yaml" src/ tests/ || echo "OK"
|
||||
grep -r "load_yaml\|run_cli\|register_fn\|YamlLoadError" src/ tests/ || echo "OK"
|
||||
```
|
||||
|
||||
阶段6 最终:`pf pymake cov` 通过,覆盖率 ≥95%(branch)。
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
alwaysApply: true
|
||||
scene: git_message
|
||||
---
|
||||
|
||||
在此处编写规则,自定义 AI 生成提交信息的风格。
|
||||
|
||||
## 提交信息格式
|
||||
- 提交信息必须使用中文。
|
||||
- 提交信息必须包含变更的类型(例如 "fix"、"feat"、"refactor" 等)。
|
||||
- 提交信息必须尽简洁明了,不要超过一段落。
|
||||
@@ -0,0 +1,157 @@
|
||||
# Python 开发规范
|
||||
|
||||
本规范结合 Python 最佳实践,作为编写与审查 Python 代码的统一标准。
|
||||
详细操作指南见 `.agents/skills/` 下相应技能。
|
||||
|
||||
## 工具链(以 pyproject.toml 为准)
|
||||
|
||||
| 工具 | 用途 | 配置要点 |
|
||||
|------|------|---------|
|
||||
| **ruff** | lint + format | `line-length=120`,`target-version="py38"` |
|
||||
| **pyrefly** | 类型检查 | `preset="strict"`,`python-version="3.8"` |
|
||||
| **pytest** | 测试 | `asyncio_default_fixture_loop_scope="function"`,marker `slow` |
|
||||
| **coverage** | 覆盖率 | `branch=true`,`fail_under=95`,`concurrency=["thread"]` |
|
||||
| **pre-commit** | 提交前检查 | ruff `--fix` + trailing-whitespace + end-of-file-fixer |
|
||||
|
||||
验证(每次修改后必做):
|
||||
|
||||
```bash
|
||||
uvx --from pyflowx pymake tc
|
||||
uvx --from pyflowx pymake cov
|
||||
```
|
||||
|
||||
## 兼容性
|
||||
|
||||
- **最低 Python 3.8**:用 `from __future__ import annotations` 延迟注解求值;
|
||||
按版本用 `typing.List`(3.8) → 内置泛型(3.9) → `X | Y`(3.10) → `typing.override`(3.12)。
|
||||
- **版本守卫**:`if sys.version_info >= (3, X):` 引入高版本 API;低版本回退分支加 `# pragma: no cover`。
|
||||
- **零运行时依赖**:仅依赖标准库(3.8 需 `graphlib_backport`、`typing-extensions`)。
|
||||
新增依赖须审慎,优先用标准库。
|
||||
|
||||
## 类型注解
|
||||
|
||||
- **公共 API 必须有完整类型注解**,包括返回类型;私有函数也应有注解。
|
||||
- 泛型用 `TypeVar`;PEP 696 `default=` 仅 3.13+ 标准库支持,3.8–3.12 用 `typing_extensions.TypeVar`。
|
||||
- `Mapping`/`Sequence` 用于只读参数,`dict`/`list` 用于可变返回。
|
||||
- `Any` 仅用于真正动态场景(如 `Context` 跨任务异构映射);任务内部类型必须完全静态。
|
||||
- 禁用裸 `# type: ignore`;确需时加具体规则码(如 `# type: ignore[union-attr]`)。
|
||||
- **`TYPE_CHECKING` 守卫**:仅类型检查需要的导入放 `if TYPE_CHECKING:` 块内,避免循环依赖。
|
||||
- **类型收窄**:用 `assert isinstance(x, Y)` 辅助 pyrefly 推断;`cast()` 仅用于类型系统无法表达的场景。
|
||||
|
||||
## 数据结构
|
||||
|
||||
- **不可变优先**:配置/描述类用 `@dataclass(frozen=True)`;可变类属性标注 `RUF012` 豁免。
|
||||
- **缓存**:实例级用 `functools.cached_property`,按参数键控用 `functools.lru_cache`;
|
||||
不可哈希参数需 try/except 回退。修改被缓存数据源后必须手动清空缓存。
|
||||
- **抽象基类**:接口用 `abc.ABC` + `@abstractmethod`(如 `StateBackend`)。
|
||||
- **枚举**:状态/标志值用 `enum.Enum`(如 `TaskStatus`),禁止裸字符串/魔术数字;枚举值用 `UPPER_SNAKE`。
|
||||
- **`__repr__`**:可变类实现 `__repr__`(含关键字段);`frozen=True` dataclass 自动生成。
|
||||
|
||||
## 模块与导入
|
||||
|
||||
- **单一职责**:每模块只做一件事(`task.py` 数据结构、`executors.py` 执行、`command.py` 命令、`compose.py` 组合)。禁止跨职责边界。
|
||||
- **导入顺序**(ruff isort):`__future__` → 标准库 → 第三方 → 本地,各组间空行。
|
||||
- **惰性导入**:仅为打破循环依赖时使用,函数体内导入并注释说明;顶层导入是默认。
|
||||
- **`__all__`**:定义 `__all__` 显式声明导出符号,位置仅次于 `__future__` 之后。
|
||||
- **禁用 star imports**:`from x import *` 污染命名空间、破坏类型检查(`__init__.py` 聚合经 `__all__` 控制为例外)。
|
||||
- **避免 `utils.py`/`helpers.py`**:按职责归入对应模块。
|
||||
|
||||
## 函数设计
|
||||
|
||||
- **模块级函数优于 Mixin**:共享逻辑用模块级函数,类只持有状态与薄方法。
|
||||
- **静态方法慎用**:纯函数直接放模块级。
|
||||
- **参数 ≤ 5 个**为宜;超出用 dataclass 封装参数对象。
|
||||
- **单一职责**:一个函数做一件事;过长函数考虑拆分。
|
||||
- **异常范围要窄**:只捕获预期异常(如 `(TypeError, ValueError, KeyError, AttributeError)`),
|
||||
**禁止** `except Exception` 掩盖 bug;捕获后至少 `logger.warning` 记录。
|
||||
- **可变默认参数**:`def f(x=[])` 是经典坑;用 `None` 哨兵或 `field(default_factory=list)`。
|
||||
|
||||
## 异常处理
|
||||
|
||||
- **自定义异常家族**:继承公共基类(如 `PyFlowXError`),按错误场景分类。
|
||||
- **异常包装**:`raise NewError(...) from exc` 保留因果链。
|
||||
- **不要吞异常**:捕获后必须处理(记录/包装/重抛),禁止空 `except: pass`。
|
||||
- **钩子/回调异常**:第三方回调异常仅记录,不影响主流程。
|
||||
|
||||
## 并发与线程安全
|
||||
|
||||
- **进程全局状态**(`os.environ`/`os.chdir`)在并发场景下必须用全局锁(`threading.RLock`)序列化。
|
||||
- **条件评估不可有可变状态**:组合条件(NOT/AND/OR)不得修改共享 `_reason`,避免竞态。
|
||||
- **批量 I/O**:循环内多次写盘改为批量一次(`contextmanager` 包裹延迟落盘)。
|
||||
- **信号量限流**:`concurrency_key` + `Semaphore` 按组限流。
|
||||
|
||||
## 测试
|
||||
|
||||
详细操作指南见 `.agents/skills/pyflowx-testing` 技能。硬约束:
|
||||
|
||||
- **覆盖率 ≥ 95%**(branch coverage),不得下降。
|
||||
- **公共 API 优先测试**:用公共接口(`has`/`get`),不访问私有方法;
|
||||
故障注入等场景可临时访问私有属性,docstring 注明原因。
|
||||
- **命名**:`test_<被测对象>_<场景>`。
|
||||
- **断言**:原生 `assert x == 1`,禁用 `self.assertEqual`;`pytest.raises` 必填 `match=`。
|
||||
- **Mock 优先级**:`monkeypatch` > 内联 stub > `unittest.mock` > `pytest-mock`。
|
||||
禁用 `@patch` 装饰器、`mock.patch.object` 上下文、`pytest-mock` 的 `mocker` fixture。
|
||||
- **fixture**:`tmp_path`/`monkeypatch`/`capsys` 优先;autouse 仅全局必需时用。
|
||||
- **slow 标记**:耗时测试加 `@pytest.mark.slow`,CI 可 `-m "not slow"` 跳过。
|
||||
- **测试代码也跑 ruff**:`tests/**` 忽略 `ARG001`/`ARG002`。
|
||||
|
||||
## 代码风格
|
||||
|
||||
- **行宽 120**(ruff formatter 处理)。
|
||||
- **docstring**:公共 API 必须有;中文叙述 + 中文注释是本项目既有风格。
|
||||
- **打印和日志**:使用中文打印和日志,避免使用英文。
|
||||
- **命名**:`snake_case` 函数/变量,`PascalCase` 类,`UPPER_SNAKE` 常量,`_leading_underscore` 私有。
|
||||
- **字符串引号**:ruff 默认双引号。
|
||||
- **末尾单 `\n`**、**无尾随空格**(pre-commit 强制)。
|
||||
- **不用 emoji**:除非用户明确要求。
|
||||
|
||||
## Pythonic 风格
|
||||
|
||||
- **`is` 比较 `None`/`True`/`False`**:单例用 `is`,值用 `==`(PEP 8 E711/E712)。
|
||||
- **EAFP 优于 LBYL**:先尝试再处理异常,而非先检查再执行(避免竞态窗口)。
|
||||
- **truthiness**:`if items:` 优于 `if len(items) > 0:`。
|
||||
- **字符串格式化**:首选 f-string;`%` 仅用于 `logging` 延迟格式化。
|
||||
- **推导式**优于 `map`+`filter`;> 2 层拆为显式循环。
|
||||
- **`enumerate`** 替代 `range(len())`;**`zip`** 并行迭代(3.10+ 用 `strict=True`)。
|
||||
- **解包** `a, b = pair` 优于索引访问;忽略值用 `_`。
|
||||
- **海象运算符 `:=`**(3.8+):赋值+判断合一,但不滥用。
|
||||
|
||||
## 日志
|
||||
|
||||
- **`logging.getLogger(__name__)`**:每模块独立 logger,禁用 `print` 调试残留。
|
||||
- **结构化上下文**:`extra={...}` 传字段;`logger.warning("task %r failed: %s", name, exc)` 优于 f-string(延迟格式化)。
|
||||
- **日志级别**:`DEBUG` 诊断 / `INFO` 关键流程 / `WARNING` 可恢复异常 / `ERROR` 需人工介入。
|
||||
- **禁止日志密码/密钥**:脱敏后再记录。
|
||||
|
||||
## 路径与资源
|
||||
|
||||
- **优先 `pathlib.Path`**:`Path("a") / "b"` 而非 `os.path.join`(ruff `PTH` 强制);
|
||||
禁止字符串拼接路径。类型注解用 `Path`,边界 `str` 立即包装。
|
||||
- **`with` 语句**:文件、锁、连接、临时目录一律用 `with` 或 `contextlib.contextmanager`;
|
||||
多资源用 `contextlib.ExitStack`。
|
||||
- **显式关闭**:长生命周期对象(连接池、线程池)实现 `close()`,但优先 `with`。
|
||||
- **批量操作**:循环内多次 acquire/release 改为批量一次。
|
||||
|
||||
## 安全
|
||||
|
||||
- **禁用 `eval`/`exec`**:处理不可信输入时绝不使用;用 `ast.literal_eval` 或专用解析器。
|
||||
- **`subprocess`**:禁用 `shell=True` 除非命令完全可信;优先 `list[str]` 形式。
|
||||
- **凭证不入仓**:密钥/token/密码放 `.env` 或环境变量,`.gitignore` 必须包含 `.env`。
|
||||
- **日志脱敏**:记录请求/响应时移除 `Authorization`、`password` 等字段。
|
||||
- **依赖审计**:`uv lock` 后审阅新增依赖,避免引入已知 CVE 的包。
|
||||
|
||||
## 性能要点
|
||||
|
||||
- **避免重复计算**:循环内查询应缓存或预构建映射(如 `{name: spec}`)。
|
||||
- **避免双重查找**:`has(k)` + `get(k)` 改为单次 `get(k)` + `KeyError` 回退。
|
||||
- **统一校验**:入口校验一次,下游路径不重复(如 `run()` 统一 `validate()`,`layers()` 不再重复)。
|
||||
- **事件 emit**:任务生命周期必须 emit `RUNNING` → `SUCCESS`/`FAILED`/`SKIPPED`,
|
||||
不要留死分支(`# pragma: no cover` 是清理信号,应激活或删除)。
|
||||
|
||||
## Git 与提交
|
||||
|
||||
- **自动提交**:任务完成后自动 `git add`(按文件名)+ `git commit` + `git push`(仅当分支已跟踪远程时执行 push;新分支跳过 push 并在总结中说明)。
|
||||
- **不修改 git config**。
|
||||
- **不运行破坏性命令**(`push --force`/`reset --hard`/`clean -f`)除非用户明确要求。
|
||||
- **staging**:按文件名添加,不用 `git add -A`/`git add .`,避免误加敏感文件。
|
||||
- **commit message**:简洁,聚焦"为什么"而非"是什么";遵循仓库既有风格。
|
||||
@@ -0,0 +1,134 @@
|
||||
---
|
||||
alwaysApply: true
|
||||
---
|
||||
|
||||
# 自驱动开发规则
|
||||
|
||||
本规则定义一种"目标驱动、闭环执行"的工作模式:仅在任务开始时与用户确认一次目标与边界,后续由 Agent 自主完成"计划 → 编码 → 测试 → 文档 → 验证"的迭代循环,直到用户目标达成。
|
||||
|
||||
## 核心原则
|
||||
|
||||
- **目标导向**:始终以用户最终目标为准绳,所有阶段产出都应服务于该目标。
|
||||
- **闭环执行**:每个子任务必须走完"计划 → 实现 → 测试 → 文档 → 验证"五步;禁止跳步留半成品。
|
||||
- **自主决策**:初始确认之后,实现路径、API 形态、重构范围、文件命名、测试组织、错误修复策略等由 Agent 自行决断,不再逐项请示。**可逆操作(编辑文件、运行测试、修复 lint、调整实现)直接执行,不询问**;只有不可逆/高风险操作才暂停。
|
||||
- **透明沟通**:每个阶段开始前用一句话说明意图;关键节点(完成、阻塞、转向)给简短更新;不复述内部思考,**不在收尾时停下询问"是否继续"或"是否提交"**——直接输出总结并结束。
|
||||
- **安全边界**:仅在高风险、不可逆操作或真正阻塞时才暂停找用户。
|
||||
|
||||
## 初始确认(一次性,仅在最开始)
|
||||
|
||||
任务启动时,用 `AskUserQuestion` 一次性确认以下信息(已由项目规范覆盖的不必重复确认):
|
||||
|
||||
1. **目标与范围**:要解决什么问题?交付物是什么?显式列出不在范围内的内容。
|
||||
2. **验收标准**:怎样算"完成"?可观测的判定条件(功能、性能、覆盖率阈值)。
|
||||
3. **特殊约束**:除 `python-standards.md` 之外的约束(兼容性、依赖限制、API 兼容策略等)。
|
||||
4. **测试要求**:覆盖率门槛(项目默认 ≥95%,branch);是否需要新增 `slow` 标记。
|
||||
|
||||
**git commit/push 不在确认范围内**:任务完成后自动 commit + push(仅当分支已跟踪远程时执行 push;新分支跳过 push 并在总结中说明),遵循 `.trae/rules/git-commit-message.md` 风格。仅 force-push、reset --hard、clean -f、修改 git config 等真正破坏性操作才需暂停确认。
|
||||
|
||||
确认后,将目标与验收标准固化进 `TaskCreate` 任务列表,后续不再就同一信息反复询问。
|
||||
|
||||
## 迭代循环
|
||||
|
||||
下列五个阶段构成一个完整闭环。未达验收标准时,回到「计划」开启下一轮;达标准时,进入「收尾」。
|
||||
|
||||
### 1. 计划(Plan)
|
||||
|
||||
- 用 Explore/Glob/Grep 研究相关代码与既有模式,避免凭空设计。
|
||||
- 用 `TaskCreate` 把目标拆为可独立验证的子任务;每完成一项立即 `TaskUpdate` 为 completed。
|
||||
- 优先复用现有抽象;不为本轮假想需求设计接口。
|
||||
- 不过早抽象:三处相似才考虑提取,否则就地写。
|
||||
|
||||
### 2. 实现(Code)
|
||||
|
||||
- 严格遵守 `.trae/rules/python-standards.md` 与既有代码风格。
|
||||
- 优先 Edit 现有文件;新增文件需有明确职责边界。
|
||||
- 不引入运行时依赖(项目零依赖原则);确需引入须在计划阶段说明。
|
||||
- 公共 API 必须有完整类型注解与中文 docstring。
|
||||
- 不写未被要求的功能、不为未来场景预留扩展点。
|
||||
|
||||
### 3. 测试(Test)
|
||||
|
||||
- 新增/修改的公共 API 必须配套测试;优先通过公共接口测试,故障注入可访问私有属性并在 docstring 注明。
|
||||
- Mock 优先级:`monkeypatch` > 内联 stub > `unittest.mock` > `pytest-mock`;禁用 `@patch` 装饰器。
|
||||
- 必跑校验(每次修改后):
|
||||
|
||||
```bash
|
||||
uvx --from pyflowx pymake tc
|
||||
uvx --from pyflowx pymake cov
|
||||
```
|
||||
|
||||
- 测试失败时定位根因再修复,不通过放宽断言或 `# pragma: no cover` 绕过。
|
||||
- 覆盖率不得低于上一次的值(项目门槛 95%,branch)。
|
||||
|
||||
### 4. 文档(Docs)
|
||||
|
||||
- 同步更新 docstring、README、模块结构说明。
|
||||
- 行为变更须同步更新 `.agents/skills/pyflowx-development/SKILL.md` 中的对应章节。
|
||||
- 跨会话有价值的设计决策、约束、陷阱,追加到 memory(`project_memory.md` 或对应 `topics.md`)。
|
||||
- 不主动新建 `*.md` 文档;除非用户明确要求。
|
||||
|
||||
### 5. 验证(Verify)
|
||||
|
||||
- 逐条对照初始确认的「验收标准」核验;未满足则回到「计划」继续下一轮。
|
||||
- 全套门禁通过:ruff、pyrefly、pytest、coverage。
|
||||
- 给出本轮变更清单(改了哪些文件、为什么)。
|
||||
|
||||
## 暂停条件(仅在以下情况中断自驱动找用户)
|
||||
|
||||
1. **歧义无法自决**:需求存在多种合理解读且无既有约定可循。
|
||||
2. **高风险/不可逆操作**:删除非临时文件、`git push --force`、`reset --hard`、删表、修改 CI 配置、修改 git config、卸载依赖等。**普通 `git commit`/`push` 不属于此类**(任务完成后自动执行)。
|
||||
3. **不可恢复的失败**:根因不在本仓库、需外部环境/权限配合、或经两轮尝试仍无法定位。
|
||||
4. **超出初始确认范围**:用户目标在执行中发现需要显著扩大范围或改变方向。
|
||||
5. **用户主动询问**:用户在对话中提出新问题或要求澄清。
|
||||
|
||||
**注意**:"目标已达成"**不是**暂停条件——验收标准全部满足后直接进入收尾并结束任务,不询问"是否扩展范围"或"是否提交"。
|
||||
|
||||
非以上情况,一律继续自驱动,不要为"求确认"而暂停。
|
||||
|
||||
## 决策判据:该问还是自决
|
||||
|
||||
遇到不确定时,按以下顺序判断:
|
||||
|
||||
1. **是否不可逆/高风险?** 是 → 暂停确认(如删除文件、`push --force`、修改 CI 配置、卸载依赖)。否 → 继续。
|
||||
2. **是否在初始确认范围内?** 是 → 按确认执行,不询问。否 → 视为"超出初始确认范围",暂停。
|
||||
3. **是否有既有约定可循?** 是 → 按约定执行(参考 `python-standards.md`、`project_memory.md`)。否 → 视为"歧义无法自决",暂停。
|
||||
4. **是否可逆?** 是 → 直接执行,即使结果可能不完美(可在后续迭代修正)。否 → 暂停。
|
||||
|
||||
**可直接自决(不询问)的典型情况**:
|
||||
|
||||
- 测试失败、覆盖率不达标、lint/类型检查报错 → 定位根因并修复。
|
||||
- 代码风格选择(命名、模块划分、参数顺序)→ 自决。
|
||||
- 文件编辑、运行测试、运行校验命令 → 直接执行。
|
||||
- 任务完成后输出收尾总结 → 直接输出,不询问下一步。
|
||||
- 显式指定 `name` 参数以保持测试兼容性 → 自决。
|
||||
- 重命名局部变量以避免遮蔽 → 自决。
|
||||
|
||||
**必须暂停询问的典型情况**:
|
||||
|
||||
- 删除非临时文件、重命名公共模块/包。
|
||||
- `git push --force`、`reset --hard`、`clean -f`、修改 git config(普通 commit/push 自动执行,无需询问)。
|
||||
- 引入新的运行时依赖(违反项目零依赖原则)。
|
||||
- 修改 CI 配置、pre-commit 钩子、pyproject.toml 的工具链配置。
|
||||
- 卸载或降级既有依赖。
|
||||
|
||||
## 沟通风格
|
||||
|
||||
- 阶段切换时一句话说明即可;不要把内部推理写给用户看。
|
||||
- 完成子任务后用一两句总结改了什么、下一步做什么。
|
||||
- 遇到阻塞时直接说明:卡在哪、试了什么、需要用户做什么。
|
||||
- **不在收尾时询问"是否需要提交"或"是否扩展范围"**——直接输出总结并结束。用户后续若有新需求,由用户主动提出。
|
||||
- 不使用 emoji,除非用户明确要求。
|
||||
|
||||
## 工具使用
|
||||
|
||||
- 独立操作尽量并行调用(多个 Read/Grep/Glob 一批发出)。
|
||||
- 用 `TaskCreate`/`TaskUpdate` 维护进度,不批量推迟标记。
|
||||
- 长命令用后台运行(`run_in_background`),完成会自动通知。
|
||||
- 文件操作一律用专用工具:Read/Edit/Write/Glob/Grep,不用 `cat`/`sed`/`grep`/`find`。
|
||||
|
||||
## 收尾
|
||||
|
||||
- 验收标准全部满足后,**直接输出最终总结并结束任务**:交付物、关键决策、遗留事项。
|
||||
- **自动提交**:收尾时自动 `git add`(按文件名)+ `git commit`(遵循 `.trae/rules/git-commit-message.md` 风格)+ `git push`(仅当分支已跟踪远程时执行;新分支跳过 push 并在总结中说明);**不询问**"是否需要提交"或"是否扩展范围"。
|
||||
- 若验收标准未全部满足,回到「计划」继续下一轮,不停下询问。
|
||||
- 将本次会话的关键产出与决策更新到 memory,便于后续会话续接。
|
||||
@@ -0,0 +1,135 @@
|
||||
---
|
||||
name: "pyflowx-testing"
|
||||
description: "PyFlowX 项目的测试编写规范与 mock 使用指南。在编写或审查测试、选择 mock 工具、设计 fixture、处理 asyncio 测试时调用。"
|
||||
---
|
||||
|
||||
# PyFlowX 测试规范
|
||||
|
||||
本技能是 `.trae/rules/python-standards.md` 测试章节的详细展开。
|
||||
规则文件仅保留硬约束指针,本文件提供完整操作指南。
|
||||
|
||||
## 总则
|
||||
|
||||
- **覆盖率 ≥ 95%**(branch coverage),不得下降。
|
||||
- **公共 API 优先测试**:测试用公共接口(`has`/`get`),不访问私有方法
|
||||
(如 `_expired`)。兼容旧测试的私有方法应删除并迁移测试。
|
||||
例外:`_store`/`_flush` 等内部状态在无法用公共 API 触发时(如模拟过期、
|
||||
故障注入),可临时访问私有属性,并在 docstring 注明原因。
|
||||
- **命名**:`test_<被测对象>_<场景>`,如 `test_storage_key_cache_key_exception_returns_name`。
|
||||
- **每个测试一个断言重点**;多个断言要语义相关。
|
||||
- **slow 标记**:耗时测试加 `@pytest.mark.slow`,CI 可 `-m "not slow"` 跳过。
|
||||
- **测试代码也跑 ruff**:`tests/**` 忽略 `ARG001`/`ARG002`(未用 fixture 参数)。
|
||||
- **断言风格**:用原生 `assert` + 比较运算符(`assert x == 1`),
|
||||
不用 `self.assertEqual`;pytest 会生成更清晰的 diff。
|
||||
|
||||
## Mock 工具选择(强制)
|
||||
|
||||
**优先级**:`monkeypatch` > 内联 stub > `unittest.mock` > `pytest-mock`。
|
||||
|
||||
| 场景 | 工具 | 示例 |
|
||||
|------|------|------|
|
||||
| 替换模块属性 / 环境变量 / 工作目录 | `monkeypatch` | `monkeypatch.setattr(subprocess, "run", fake_run)` |
|
||||
| `os.environ["KEY"]` 临时设置 | `monkeypatch.setenv` | `monkeypatch.setenv("LOCALAPPDATA", "C:\\...")` |
|
||||
| 切换 cwd | `monkeypatch.chdir` | `monkeypatch.chdir(tmp_path)` |
|
||||
| 一次性 stub 函数 | 内联 lambda / 闭包 | `ran = []; monkeypatch.setattr(subprocess, "run", lambda *c, **__: ran.append(c))` |
|
||||
| 复杂 spy(记录调用次数/参数/返回序列) | `unittest.mock.MagicMock` | 仅当 lambda 不足以表达时 |
|
||||
| `with patch(...)` 上下文 | **禁用**(用 monkeypatch) | monkeypatch 自动 teardown 更安全 |
|
||||
|
||||
**禁止**:
|
||||
- 不用 `pytest-mock` 的 `mocker` fixture(项目虽在 dev 依赖声明,但实际
|
||||
测试代码未使用;为保持风格统一,新代码继续用 `monkeypatch`)。
|
||||
- 不用 `unittest.mock.patch` 装饰器(`@patch("x.y")`),它隐藏依赖且
|
||||
与 pytest fixture 模式不兼容;用 `monkeypatch.setattr` 替代。
|
||||
- 不用 `mock.patch.object` 作为上下文管理器,除非被测代码本身就是
|
||||
contextmanager(此时用 `monkeypatch.setattr` 仍更简单)。
|
||||
|
||||
## monkeypatch 使用规范
|
||||
|
||||
- **类型注解**:fixture 参数标注 `monkeypatch: pytest.MonkeyPatch`。
|
||||
- **作用域**:monkeypatch 自动在测试结束时撤销,**禁止**手动
|
||||
`monkeypatch.setattr(x, "y", original)` 恢复(多余且容易遗漏)。
|
||||
例外:在单个测试内需要中途恢复时,用 `monkeypatch.undo()` 全量撤销。
|
||||
- **替换目标**:替换"被测代码看到的对象",而非全局对象本身。
|
||||
- 错误:`monkeypatch.setattr("os.path.exists", fake)` —— 替换全局,影响其他模块。
|
||||
- 正确:`monkeypatch.setattr(pyflowx.command.shutil, "which", fake)` ——
|
||||
替换被测模块引用的 `shutil.which`。
|
||||
- **属性 vs 字符串路径**:优先属性访问形式 `monkeypatch.setattr(obj, "attr", val)`
|
||||
而非字符串路径 `monkeypatch.setattr("pkg.mod.obj.attr", val)`,
|
||||
前者有 IDE 跳转与重构支持。
|
||||
- **记录调用**:用闭包 `ran: list[tuple] = []` + `lambda *a, **k: ran.append((a, k))`
|
||||
替代 `MagicMock`,可读性更好且无需导入。
|
||||
|
||||
## Stub 与 Spy 模式
|
||||
|
||||
- **轻量 stub**:内联定义 `class MockResult: returncode = 0; stdout = ""`,
|
||||
替代 `MagicMock(return_value=...)`,类型明确且不引入 mock 依赖。
|
||||
- **状态收集**:闭包 + list 比 `mock.call_args_list` 更易断言:
|
||||
```python
|
||||
calls: list[list[str]] = []
|
||||
|
||||
|
||||
def fake_run(cmd: list[str], **_: Any) -> MockResult:
|
||||
calls.append(cmd)
|
||||
return MockResult()
|
||||
|
||||
|
||||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||||
assert calls == [["clear"]]
|
||||
```
|
||||
- **副作用序列**:需要按调用次数返回不同值时,用 `itertools.cycle` 或
|
||||
手动计数器,而非 `side_effect=[...]`(mock 专有 API)。
|
||||
- **异常注入**:`def raise_oserror(*a, **k): raise OSError("...")`,
|
||||
用 `pytest.raises(OSError)` 验证,而非 `side_effect=OSError`。
|
||||
|
||||
## 异常断言
|
||||
|
||||
- **`pytest.raises`**:必填 `match=` 正则(除非异常消息完全不可预测),
|
||||
避免误捕获同类异常:
|
||||
```python
|
||||
with pytest.raises(StorageError, match="cannot write"):
|
||||
b.save("a", 1)
|
||||
```
|
||||
- **异常链**:验证 `__cause__` 时用 `exc_info.value.__cause__`,
|
||||
确认 `raise X from Y` 因果链完整。
|
||||
- **禁止** `try/except + assert False`:用 `pytest.raises` 替代。
|
||||
|
||||
## Fixture 规范
|
||||
|
||||
- **`tmp_path`**:处理临时文件,自动清理,禁止 `tempfile.mkdtemp()` 手动管理。
|
||||
- **`monkeypatch`**:环境变量、cwd、模块属性 mock(见上)。
|
||||
- **`capsys`/`capfd`**:捕获 stdout/stderr,验证日志或命令输出。
|
||||
- **autouse fixture**:仅在全局必需时用(如 `conftest.py` 的
|
||||
`packtool_tmp_workdir` 自动切到 tmp_path);否则显式声明参数。
|
||||
- **fixture 命名**:`snake_case`,描述"提供什么"而非"测试什么"
|
||||
(`sample_graph` 优于 `test_data`)。
|
||||
- **fixture 作用域**:默认 `function`;`module`/`session` 仅当构造昂贵且
|
||||
只读时,并加注释说明无副作用。
|
||||
|
||||
## asyncio 测试
|
||||
|
||||
- **fixture `loop_scope="function"`**(pyproject 已配置默认值)。
|
||||
- **async 测试**:`async def test_x():`,pytest-asyncio 自动驱动。
|
||||
- **await 检查**:测试异步函数必须 `await` 结果,禁止仅验证返回 coroutine 对象。
|
||||
- **异步 mock**:用 `AsyncMock`(3.8+ 在 `unittest.mock`)或
|
||||
`async def fake(): return value`,禁用 `MagicMock(return_value=coro)`。
|
||||
|
||||
## 参数化
|
||||
|
||||
- **`@pytest.mark.parametrize`**:用 `ids` 参数提供可读标识:
|
||||
```python
|
||||
@pytest.mark.parametrize(
|
||||
("strategy", "expected_workers"),
|
||||
[("sequential", 1), ("thread", 8), ("async", 1)],
|
||||
ids=["seq", "thread-8", "async"],
|
||||
)
|
||||
```
|
||||
- **参数命名**:参数元组用有意义名称,而非 `("a", "b")`。
|
||||
- **组合爆炸**:参数组合 > 20 时拆分测试,避免单个测试函数臃肿。
|
||||
|
||||
## 测试组织
|
||||
|
||||
- **文件命名**:`test_<被测模块>.py`(`test_storage.py` 对应 `storage.py`)。
|
||||
- **类分组**:仅在测试逻辑强相关时用 `class TestXxx:` 分组;默认用模块级函数。
|
||||
- **docstring**:每个测试函数一句话说明"测试什么场景",复杂场景补充"为什么"。
|
||||
- **setup/teardown**:优先 fixture;`setup_method`/`teardown_method` 仅在
|
||||
无法用 fixture 表达时(罕见)。
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
# 使用国内镜像源拉取基础镜像
|
||||
# 备选镜像源前缀:docker.1ms.run / dockerpull.com / docker.xuanyuan.me
|
||||
FROM docker.m.daocloud.io/python:3.13-slim
|
||||
|
||||
# 国内镜像源(清华)
|
||||
ENV PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_TRUSTED_HOST=pypi.tuna.tsinghua.edu.cn
|
||||
ENV UV_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV UV_TRUSTED_HOST=pypi.tuna.tsinghua.edu.cn
|
||||
|
||||
# 环境变量:非交互 + 路径配置
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
LANG=C.UTF-8 \
|
||||
LC_ALL=C.UTF-8 \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_CACHE_DIR=/uv-cache \
|
||||
UV_PYTHON_INSTALL_DIR=/uv-python \
|
||||
UV_PROJECT_ENVIRONMENT=/opt/venv \
|
||||
PATH="/opt/venv/bin:${PATH}"
|
||||
|
||||
# 配置 apt 国内镜像(阿里云)并安装系统依赖
|
||||
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git \
|
||||
jq \
|
||||
build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 配置 pip 国内镜像(阿里云)
|
||||
RUN mkdir -p /etc/pip \
|
||||
&& printf '[global]\nindex-url = https://mirrors.aliyun.com/pypi/simple/\ntrusted-host = mirrors.aliyun.com\n' \
|
||||
> /etc/pip/pip.conf \
|
||||
&& mkdir -p /root/.config/pip \
|
||||
&& ln -sf /etc/pip/pip.conf /root/.config/pip/pip.conf
|
||||
|
||||
# 安装 uv 并预装 Python 3.8 / 3.13
|
||||
RUN pip install --no-cache-dir uv -i https://mirrors.aliyun.com/pypi/simple/ \
|
||||
&& uv python install 3.8 3.13
|
||||
|
||||
# 安装 Node.js 20.x(actions/checkout 需要)
|
||||
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
|
||||
apt-get install -y nodejs && \
|
||||
node --version
|
||||
|
||||
# 预装项目 dev 依赖(仅复制依赖描述文件,利用 Docker 层缓存)
|
||||
WORKDIR /workspace
|
||||
COPY pyproject.toml tox.ini README.md ./
|
||||
COPY src/ ./src/
|
||||
|
||||
# 同步依赖到 /opt/venv(CI 时直接复用)
|
||||
RUN uv sync --frozen --no-install-project 2>/dev/null || uv sync --no-install-project
|
||||
|
||||
# 预装 tox 环境(py38 + py313)
|
||||
RUN uvx tox run -e py38,py313 --notest 2>/dev/null || true
|
||||
|
||||
# 持久化 uv 缓存目录(CI 可挂载到宿主机加速)
|
||||
VOLUME ["/uv-cache"]
|
||||
|
||||
# 默认入口
|
||||
CMD ["/bin/bash"]
|
||||
@@ -5,6 +5,7 @@
|
||||
[](https://github.com/gookeryoung/pyflowx/actions/workflows/ci.yml)
|
||||
[](https://pypi.org/project/pyflowx/)
|
||||
[](https://pypi.org/project/pyflowx/)
|
||||
[](https://pyflowx.readthedocs.io/zh/latest/)
|
||||
[](https://github.com/gookeryoung/pyflowx)
|
||||
[](https://github.com/gookeryoung/pyflowx/blob/main/LICENSE)
|
||||
|
||||
@@ -14,18 +15,26 @@ PyFlowX 把"任务依赖"这件事做到极致简单:**参数名就是依赖
|
||||
## 特性
|
||||
|
||||
- **零样板** —— 参数名即依赖,框架自动注入上游结果
|
||||
- **三种执行策略** —— `sequential`(调试)/ `thread`(I/O 密集同步)/ `async`(I/O 密集异步)
|
||||
- **四种执行策略** —— `sequential`(调试)/ `thread`(I/O 密集同步)/ `async`(I/O 密集异步)/ `dependency`(依赖驱动,最大化并行)
|
||||
- **类型安全** —— `TaskSpec[T]` 把返回类型一路传到 `RunReport`,mypy strict 通过
|
||||
- **DAG 校验** —— 构建时即时校验重名、缺失依赖、环
|
||||
- **自动分层** —— Kahn 算法分组,同层任务可并行
|
||||
- **重试与超时** —— 每个任务独立配置 `retries` 与 `timeout`
|
||||
- **断点续跑** —— `MemoryBackend` / `JSONBackend`,成功结果可缓存复用
|
||||
- **重试与超时** —— 每个任务独立配置 `RetryPolicy`(max_attempts/delay/backoff/jitter/retry_on)与 `timeout`
|
||||
- **软依赖** —— `soft_depends_on` 仅用于上下文注入,不参与拓扑分层
|
||||
- **并发限制** —— `concurrency_key` + `concurrency_limits` 按组限流
|
||||
- **任务钩子** —— `TaskHooks`(pre_run/post_run/on_failure)生命周期回调
|
||||
- **断点续跑** —— `MemoryBackend` / `JSONBackend`,成功结果可缓存复用;`batch()` 批量落盘
|
||||
- **缓存键** —— `cache_key` 函数基于输入计算稳定键,使不同输入产生独立缓存
|
||||
- **命令任务** —— `cmd` 参数直接执行外部命令,支持列表/shell/可调用对象
|
||||
- **条件执行** —— `conditions` 参数按平台、环境变量、应用安装等条件跳过任务
|
||||
- **图组合** —— `compose` / `GraphComposer` 编程式展开多图字符串引用
|
||||
- **任务模板** —— `task_template` 工厂批量生成相似 TaskSpec
|
||||
- **图级默认值** —— `GraphDefaults` 统一配置 retry/timeout/concurrency 等
|
||||
- **CLI 运行器** —— `CliRunner` 把多个图映射为命令行子命令,替代 Makefile
|
||||
- **可观测** —— `on_event` 回调、`dry_run` 预览、`verbose` 生命周期日志、Mermaid 可视化
|
||||
- **零运行时依赖** —— 仅依赖标准库(3.8 需 `graphlib_backport`)
|
||||
- **95% 测试覆盖** —— 分支覆盖率>= 95%
|
||||
- **可观测** —— `on_event` 回调(RUNNING/SUCCESS/FAILED/SKIPPED)、`dry_run` 预览、`verbose` 生命周期日志、Mermaid 可视化
|
||||
- **YAML 任务编排** —— GitHub Actions 风格的声明式任务图,支持 `jobs`/`needs`/`strategy.matrix`/`if` 等 CI/CD 概念,从 YAML 文件直接加载执行
|
||||
- **最小依赖** —— 仅依赖标准库 + PyYAML(3.8 需 `graphlib_backport`、`typing-extensions`)
|
||||
- **97% 测试覆盖** —— 分支覆盖率 >= 95%
|
||||
|
||||
## 安装
|
||||
|
||||
@@ -67,23 +76,31 @@ print(report["double"]) # [2, 4, 6]
|
||||
|
||||
### TaskSpec —— 任务描述
|
||||
|
||||
`TaskSpec` 是不可变的任务描述符,是唯一需要配置的东西:
|
||||
`TaskSpec` 是不可变的任务描述符(`Generic[T]`,返回类型一路传到 `RunReport`),是唯一需要配置的东西:
|
||||
|
||||
```python
|
||||
px.TaskSpec(
|
||||
name="fetch_user", # 唯一标识
|
||||
fn=fetch_user, # 同步或异步函数
|
||||
cmd=["curl", "..."], # 或: 执行命令(覆盖 fn)
|
||||
depends_on=("auth",), # 依赖的任务名
|
||||
depends_on=("auth",), # 硬依赖(参与拓扑分层)
|
||||
soft_depends_on=("cache",), # 软依赖(仅注入,不参与分层)
|
||||
args=(uid,), # 静态位置参数(追加在注入参数后)
|
||||
kwargs={"timeout": 30}, # 静态关键字参数
|
||||
retries=3, # 失败重试次数(0 = 仅一次)
|
||||
retry=px.RetryPolicy(max_attempts=3, delay=1.0, backoff=2.0), # 重试策略
|
||||
timeout=30.0, # 超时秒数(None = 不限制)
|
||||
tags=("api", "user"), # 自由标签,用于子图过滤
|
||||
conditions=(is_prod,), # 条件函数列表(全部为 True 才执行)
|
||||
priority=10, # 同层内优先级(高优先执行,默认 0)
|
||||
concurrency_key="db", # 并发分组键(配合 concurrency_limits 限流)
|
||||
cache_key=lambda ctx: str(ctx.get("uid")), # 缓存键函数(不同输入独立缓存)
|
||||
hooks=px.TaskHooks(pre_run=..., post_run=..., on_failure=...), # 生命周期钩子
|
||||
cwd=Path("/tmp"), # 命令工作目录(仅 cmd 模式)
|
||||
env={"DEBUG": "1"}, # 环境变量覆盖(fn 与 cmd 模式均生效)
|
||||
verbose=True, # 打印命令输出(仅 cmd 模式)
|
||||
skip_if_missing=True, # 命令不存在时自动跳过(仅 list[str] cmd)
|
||||
allow_upstream_skip=False, # 上游 SKIPPED/FAILED 时是否仍执行
|
||||
continue_on_error=False, # 本任务失败是否不中断整体
|
||||
)
|
||||
```
|
||||
|
||||
@@ -97,18 +114,54 @@ px.TaskSpec(
|
||||
### Graph —— DAG 构建
|
||||
|
||||
```python
|
||||
graph = px.Graph.from_specs([...]) # 整批校验(推荐)
|
||||
# 图级默认值:TaskSpec 字段为 None 时回退
|
||||
defaults = px.GraphDefaults(retry=px.RetryPolicy(max_attempts=2), timeout=60.0)
|
||||
|
||||
graph = px.Graph.from_specs([...], defaults=defaults) # 整批校验(推荐)
|
||||
# 或增量构建
|
||||
graph = px.Graph()
|
||||
graph = px.Graph(defaults=defaults)
|
||||
graph.add(px.TaskSpec("a", fn_a))
|
||||
graph.add(px.TaskSpec("b", fn_b, ("a",)))
|
||||
|
||||
graph.validate() # 显式校验(环检测)
|
||||
graph.layers() # 拓扑分层
|
||||
graph.layers() # 拓扑分层(run() 入口已统一校验,直接调用需自行先 validate)
|
||||
graph.to_mermaid() # Mermaid 可视化
|
||||
graph.describe() # 人类可读摘要
|
||||
graph.subgraph(("api",)) # 按标签切片
|
||||
graph.subgraph_by_names(("a", "b")) # 按名称切片
|
||||
graph.map("fetch", [1, 2, 3], lambda i: TaskSpec(f"fetch_{i}", ...)) # 批量 fan-out
|
||||
```
|
||||
|
||||
### 图组合 —— compose
|
||||
|
||||
`compose` / `GraphComposer` 把带字符串引用的多个图展开为纯 `Graph`:
|
||||
|
||||
```python
|
||||
graphs = {
|
||||
"build": px.Graph.from_specs([px.TaskSpec("b", cmd=["echo", "b"])]),
|
||||
"all": px.Graph.from_specs(["build", px.TaskSpec("t", cmd=["echo", "t"])]),
|
||||
}
|
||||
resolved = px.compose(graphs) # "all" 图中的 "build" 引用被展开
|
||||
```
|
||||
|
||||
引用格式:`"command_name"`(整个图)或 `"command_name.task_name"`(特定任务)。
|
||||
`CliRunner` 内部自动调用 `compose`。
|
||||
|
||||
### 任务模板 —— task_template
|
||||
|
||||
`task_template` 工厂批量生成相似 TaskSpec:
|
||||
|
||||
```python
|
||||
fetch = px.task_template(
|
||||
fn=fetch_url,
|
||||
retry=px.RetryPolicy(max_attempts=5),
|
||||
timeout=30.0,
|
||||
tags=("api",),
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
fetch("users", url="https://api.example.com/users"),
|
||||
fetch("posts", url="https://api.example.com/posts"),
|
||||
])
|
||||
```
|
||||
|
||||
### run —— 执行
|
||||
@@ -116,12 +169,14 @@ graph.subgraph_by_names(("a", "b")) # 按名称切片
|
||||
```python
|
||||
report = px.run(
|
||||
graph,
|
||||
strategy="async", # sequential | thread | async
|
||||
strategy="async", # sequential | thread | async | dependency
|
||||
max_workers=8, # thread 策略的线程池大小
|
||||
concurrency_limits={"db": 2}, # 按 concurrency_key 限流
|
||||
dry_run=False, # True = 仅打印计划
|
||||
verbose=False, # True = 打印任务生命周期日志
|
||||
on_event=callback, # 状态转换回调
|
||||
on_event=callback, # 状态转换回调(RUNNING/SUCCESS/FAILED/SKIPPED)
|
||||
state=px.JSONBackend("state.json"), # 断点续跑后端
|
||||
continue_on_error=False, # True = 单任务失败不中断整体
|
||||
)
|
||||
```
|
||||
|
||||
@@ -141,7 +196,7 @@ report.describe() # 人类可读报告
|
||||
按顺序求值:
|
||||
|
||||
1. **标注为 `Context`** 的参数 → 接收完整上游结果映射
|
||||
2. **名称匹配依赖** 的参数 → 接收该依赖的结果
|
||||
2. **名称匹配依赖** 的参数 → 接收该依赖的结果(含软依赖,缺失时注入默认值)
|
||||
3. **`**kwargs`** 参数 → 接收所有依赖结果(dict)
|
||||
4. **`TaskSpec.args` / `kwargs`** → 为非依赖参数提供静态值
|
||||
|
||||
@@ -170,8 +225,11 @@ def fetch_user(uid: int) -> dict: # uid 来自 TaskSpec.args
|
||||
| `sequential` | 串行 | 调试、CPU 密集 | 直接调用 | 事件循环 |
|
||||
| `thread` | 线程池 | I/O 密集同步 | 线程池 | 不支持 |
|
||||
| `async` | 事件循环 | I/O 密集异步 | 卸载到线程池 | 事件循环 |
|
||||
| `dependency` | 依赖驱动 | 最大化并行度 | 卸载到线程池 | 事件循环 |
|
||||
|
||||
所有策略都遵循 `retries`、`timeout`、上下文注入、状态后端,并发出 `TaskEvent`。
|
||||
所有策略都遵循 `RetryPolicy`、`timeout`、上下文注入、状态后端、`concurrency_limits`,
|
||||
并发出 `TaskEvent`(RUNNING/SUCCESS/FAILED/SKIPPED)。`dependency` 策略无层屏障:
|
||||
任务在其所有硬依赖完成后立即启动。
|
||||
|
||||
## 命令任务
|
||||
|
||||
@@ -244,43 +302,148 @@ runner.run_cli() # 解析 sys.argv 并执行
|
||||
命令行用法:
|
||||
|
||||
```bash
|
||||
python build.py clean # 执行 clean 图
|
||||
python build.py build --strategy thread # 覆盖执行策略
|
||||
python build.py test --dry-run # 仅打印执行计划
|
||||
python build.py --list # 列出所有命令
|
||||
python build.py --quiet # 静默模式
|
||||
pf pymake clean # 执行 clean 图
|
||||
pf pymake build --strategy thread # 覆盖执行策略
|
||||
pf pymake test --dry-run # 仅打印执行计划
|
||||
pf pymake --list # 列出所有命令
|
||||
pf pymake --quiet # 静默模式
|
||||
```
|
||||
|
||||
`verbose=True`(默认)时打印任务生命周期(开始/成功/失败/跳过)与命令输出;`--quiet` 关闭。
|
||||
|
||||
## 示例
|
||||
## YAML 任务编排
|
||||
|
||||
仓库 `examples/` 目录包含完整示例:
|
||||
PyFlowX 支持 GitHub Actions 风格的声明式 YAML 任务编排,从 YAML 文件直接加载任务图。
|
||||
|
||||
- [`etl_pipeline.py`](examples/etl_pipeline.py) —— ETL 流水线(sequential)
|
||||
- [`parallel_run.py`](examples/parallel_run.py) —— 并行执行对比(thread vs sequential)
|
||||
- [`async_aggregation.py`](examples/async_aggregation.py) —— 异步聚合 + Context 注入
|
||||
### 编程式 API
|
||||
|
||||
运行:
|
||||
```python
|
||||
import pyflowx as px
|
||||
|
||||
# 从 YAML 文件加载任务图
|
||||
graph = px.Graph.from_yaml("pipeline.yaml")
|
||||
report = px.run(graph, strategy="thread")
|
||||
|
||||
# 或用函数式 API
|
||||
graph = px.load_yaml("pipeline.yaml")
|
||||
graph = px.parse_yaml_string("""
|
||||
jobs:
|
||||
hello:
|
||||
cmd: ["echo", "hello"]
|
||||
""")
|
||||
```
|
||||
|
||||
### CLI 入口
|
||||
|
||||
通过 `pf` 统一入口调用(详见 [pf 工具](#cli-工具) 章节):
|
||||
|
||||
```bash
|
||||
python examples/etl_pipeline.py
|
||||
python examples/parallel_run.py
|
||||
python examples/async_aggregation.py
|
||||
# 执行 YAML 任务图
|
||||
pf yamlrun pipeline.yaml
|
||||
|
||||
# 指定执行策略
|
||||
pf yamlrun pipeline.yaml --strategy thread
|
||||
|
||||
# 仅打印任务分层,不执行
|
||||
pf yamlrun pipeline.yaml --dry-run
|
||||
|
||||
# 列出所有任务名
|
||||
pf yamlrun pipeline.yaml --list
|
||||
|
||||
# 静默模式
|
||||
pf yamlrun pipeline.yaml --quiet
|
||||
```
|
||||
|
||||
### YAML Schema(GitHub Actions 风格)
|
||||
|
||||
```yaml
|
||||
strategy: thread # 图级默认策略
|
||||
defaults: # 图级默认值
|
||||
retry: {max_attempts: 3}
|
||||
verbose: true
|
||||
env: {CI: "true"}
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
cmd: ["git", "clone", "..."]
|
||||
runs-on: linux
|
||||
|
||||
build:
|
||||
needs: [setup] # 依赖列表
|
||||
cmd: ["python", "-m", "build"]
|
||||
timeout: 300
|
||||
retry: {max_attempts: 2, delay: 1.0}
|
||||
|
||||
test:
|
||||
needs: [build]
|
||||
cmd: ["python${{ matrix.version }}", "-m", "pytest"] # 矩阵占位符
|
||||
strategy:
|
||||
matrix: # 笛卡尔积展开为 6 个任务
|
||||
version: ["3.8", "3.9", "3.10"]
|
||||
os: ["linux", "macos"]
|
||||
if: "env.CI" # 条件: 环境变量存在
|
||||
|
||||
lint:
|
||||
needs: [build]
|
||||
cmd: ["ruff", "check"]
|
||||
if: "env.CI == 'true'" # 条件: 环境变量等于
|
||||
|
||||
deploy:
|
||||
needs: [test, lint] # 矩阵依赖自动展开
|
||||
cmd: ["twine", "upload"]
|
||||
if: "env.DEPLOY_TOKEN != ''"
|
||||
allow-upstream-skip: true
|
||||
concurrency-key: deploy_lock
|
||||
```
|
||||
|
||||
### 字段映射
|
||||
|
||||
| YAML 字段 | TaskSpec 字段 | 说明 |
|
||||
|-----------|---------------|------|
|
||||
| `jobs.<id>` | `name` | job ID 作为任务名 |
|
||||
| `cmd` / `run` | `cmd` | `cmd` 为列表形式,`run` 为 shell 字符串 |
|
||||
| `needs` | `depends_on` | 依赖列表(矩阵任务自动展开) |
|
||||
| `if` | `conditions` | `success()` / `always()` / `env.VAR` / `env.VAR == 'x'` |
|
||||
| `strategy.matrix` | 矩阵扇出 | 笛卡尔积展开为多个任务 |
|
||||
| `${{ matrix.key }}` | 占位符 | 在 cmd/run/cwd/env 中替换 |
|
||||
| `timeout` | `timeout` | 超时秒数 |
|
||||
| `retry` | `retry` | `{max_attempts, delay, backoff, jitter}` |
|
||||
| `cwd` | `cwd` | 工作目录 |
|
||||
| `env` | `env` | 环境变量 |
|
||||
| `verbose` | `verbose` | 详细输出 |
|
||||
| `continue-on-error` | `continue_on_error` | 失败不中止整图 |
|
||||
| `skip-if-missing` | `skip_if_missing` | 命令不存在时跳过 |
|
||||
| `allow-upstream-skip` | `allow_upstream_skip` | 上游跳过时仍执行 |
|
||||
| `priority` | `priority` | 同层优先级 |
|
||||
| `concurrency-key` | `concurrency_key` | 并发限制键 |
|
||||
| `tags` | `tags` | 自由标签 |
|
||||
| `runs-on` | `tags`(追加) | 运行环境标签 |
|
||||
|
||||
## 断点续跑
|
||||
|
||||
```python
|
||||
from pyflowx import JSONBackend
|
||||
|
||||
# 第一次运行:成功结果写入 state.json
|
||||
backend = JSONBackend("state.json")
|
||||
backend = JSONBackend("state.json", ttl=3600) # ttl 秒数,过期条目自动忽略
|
||||
report = px.run(graph, strategy="sequential", state=backend)
|
||||
|
||||
# 第二次运行:已缓存任务自动跳过
|
||||
# 第二次运行:已缓存任务自动跳过(状态为 SKIPPED)
|
||||
report = px.run(graph, strategy="sequential", state=backend)
|
||||
# report.results 中缓存任务状态为 SKIPPED
|
||||
```
|
||||
|
||||
`run()` 内部以 `backend.batch()` 包裹整个执行:所有 `save` 延迟到运行结束时统一落盘一次
|
||||
(`JSONBackend` 从 O(N²) 降为 O(N) 磁盘写入;`MemoryBackend` 为 no-op)。
|
||||
|
||||
**缓存键**:默认存储键为任务名。配置 `cache_key` 函数后,键为 `"name:cache_key_value"`,
|
||||
使不同输入产生独立缓存条目:
|
||||
|
||||
```python
|
||||
px.TaskSpec(
|
||||
"fetch_user",
|
||||
fn=fetch_user,
|
||||
cache_key=lambda ctx: str(ctx.get("uid")), # 不同 uid 独立缓存
|
||||
)
|
||||
```
|
||||
|
||||
## 错误处理
|
||||
@@ -321,23 +484,90 @@ except px.PyFlowXError:
|
||||
|
||||
PyFlowX 专注于**单机 DAG 调度**的极致简洁,适合 ETL、数据处理、CI 流水线等场景。
|
||||
|
||||
## 高级特性
|
||||
|
||||
### 并发限制
|
||||
|
||||
按 `concurrency_key` 分组限流,避免压垮下游资源:
|
||||
|
||||
```python
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("q1", fn=query_db, concurrency_key="db"),
|
||||
px.TaskSpec("q2", fn=query_db, concurrency_key="db"),
|
||||
px.TaskSpec("q3", fn=query_db, concurrency_key="db"),
|
||||
])
|
||||
# 同一时刻最多 2 个 "db" 组任务运行
|
||||
px.run(graph, strategy="async", concurrency_limits={"db": 2})
|
||||
```
|
||||
|
||||
### 任务钩子
|
||||
|
||||
`TaskHooks` 在任务生命周期触发(异常仅记录,不影响任务状态):
|
||||
|
||||
```python
|
||||
hooks = px.TaskHooks(
|
||||
pre_run=lambda spec: print(f"start {spec.name}"),
|
||||
post_run=lambda spec, value: print(f"done {spec.name}"),
|
||||
on_failure=lambda spec, exc: alert(spec.name, exc),
|
||||
)
|
||||
px.TaskSpec("task", fn=work, hooks=hooks)
|
||||
```
|
||||
|
||||
### 优先级
|
||||
|
||||
同层内按 `priority` 降序执行(稳定排序):
|
||||
|
||||
```python
|
||||
px.TaskSpec("low", fn=work, priority=0)
|
||||
px.TaskSpec("high", fn=work, priority=10) # 同层内先执行
|
||||
```
|
||||
|
||||
## 开发
|
||||
|
||||
```bash
|
||||
# 安装开发依赖
|
||||
uv sync --extra dev
|
||||
|
||||
# 运行测试(含覆盖率)
|
||||
uv run pytest --cov=pyflowx --cov-fail-under=100
|
||||
# 运行测试(含覆盖率,阈值 95%)
|
||||
uv run pytest --cov=pyflowx --cov-fail-under=95
|
||||
|
||||
# 类型检查
|
||||
uv run mypy
|
||||
|
||||
# 代码风格
|
||||
uv run ruff check src tests examples
|
||||
uv run ruff format --check src tests examples
|
||||
uv run ruff check src tests
|
||||
uv run ruff format --check src tests
|
||||
```
|
||||
|
||||
## 模块结构
|
||||
|
||||
### 核心
|
||||
|
||||
| 模块 | 职责 |
|
||||
|------|------|
|
||||
| `task.py` | 纯数据结构:`TaskSpec`、`RetryPolicy`、`TaskHooks`、`TaskStatus` |
|
||||
| `graph.py` | DAG 构建、校验、分层、可视化 |
|
||||
| `compose.py` | 多图组合:`GraphComposer` / `compose` |
|
||||
| `context.py` | 上下文注入:参数名→依赖解析 |
|
||||
| `command.py` | 命令执行:`run_command`(list/shell/Callable) |
|
||||
| `conditions.py` | 条件执行:内置条件与组合器 |
|
||||
| `executors.py` | 执行器与 `run` 入口:四种策略共享模块级辅助;verbose 统一应用到 spec |
|
||||
| `storage.py` | 状态后端:`MemoryBackend` / `JSONBackend`(batch flush) |
|
||||
| `runner.py` | CLI 运行器:`CliRunner` |
|
||||
| `report.py` | 运行结果:`RunReport` / `TaskResult` |
|
||||
| `tools.py` | `@px.tool` 装饰器:`ToolSpec` / `run_tool` / `list_tools` / `list_subcommands` |
|
||||
| `profiling.py` | 性能分析:`Profiler` 任务耗时统计 |
|
||||
| `errors.py` | 错误家族:`PyFlowXError` 子类 |
|
||||
| `ops/` | CLI 工具模块集合(每个子模块用 `@px.tool` 注册一个工具) |
|
||||
|
||||
### CLI 工具
|
||||
|
||||
| 模块 | 职责 |
|
||||
|------|------|
|
||||
| `cli/pf.py` | 统一入口:`pf <tool> [command]`,按需导入 `ops/<tool>.py` 触发 `@px.tool` 注册 |
|
||||
| `cli/profiler.py` | 性能分析 CLI |
|
||||
| `cli/emlmanager.py` | 邮件管理 CLI |
|
||||
|
||||
## 许可证
|
||||
|
||||
MIT
|
||||
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
API 参考
|
||||
========
|
||||
|
||||
任务描述
|
||||
--------
|
||||
|
||||
.. autoclass:: pyflowx.TaskSpec
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
:exclude-members: args, kwargs
|
||||
|
||||
.. autoclass:: pyflowx.RetryPolicy
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.TaskHooks
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.TaskStatus
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
图构建
|
||||
------
|
||||
|
||||
.. autoclass:: pyflowx.Graph
|
||||
:members:
|
||||
:undoc-members:
|
||||
:exclude-members: from_specs, from_yaml
|
||||
|
||||
.. autoclass:: pyflowx.GraphDefaults
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autofunction:: pyflowx.compose
|
||||
.. autofunction:: pyflowx.task_template
|
||||
|
||||
执行
|
||||
----
|
||||
|
||||
.. autofunction:: pyflowx.run
|
||||
|
||||
.. autoclass:: pyflowx.RunReport
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.TaskResult
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
@px.tool 工具
|
||||
-------------
|
||||
|
||||
.. autoclass:: pyflowx.ToolSpec
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autofunction:: pyflowx.tool
|
||||
.. autofunction:: pyflowx.run_tool
|
||||
.. autofunction:: pyflowx.list_tools
|
||||
.. autofunction:: pyflowx.list_subcommands
|
||||
|
||||
命令执行
|
||||
--------
|
||||
|
||||
.. autofunction:: pyflowx.run_command
|
||||
|
||||
CLI 运行器
|
||||
----------
|
||||
|
||||
.. autoclass:: pyflowx.CliRunner
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
状态后端
|
||||
--------
|
||||
|
||||
.. autoclass:: pyflowx.StateBackend
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.MemoryBackend
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
.. autoclass:: pyflowx.JSONBackend
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
错误家族
|
||||
--------
|
||||
|
||||
.. autoexception:: pyflowx.PyFlowXError
|
||||
.. autoexception:: pyflowx.DuplicateTaskError
|
||||
.. autoexception:: pyflowx.MissingDependencyError
|
||||
.. autoexception:: pyflowx.CycleError
|
||||
.. autoexception:: pyflowx.TaskFailedError
|
||||
.. autoexception:: pyflowx.TaskTimeoutError
|
||||
.. autoexception:: pyflowx.InjectionError
|
||||
.. autoexception:: pyflowx.StorageError
|
||||
@@ -0,0 +1,45 @@
|
||||
变更日志
|
||||
========
|
||||
|
||||
0.4.5
|
||||
-----
|
||||
|
||||
CLI 重构
|
||||
~~~~~~~~
|
||||
|
||||
- 新增 ``pf`` 统一入口:通过 ``pf <tool> [command] [options]`` 调用所有工具
|
||||
- 13 个工具迁移到 YAML 配置(filedate/filelevel/folderback/folderzip/screenshot/sshcopyid/lscalc/bumpversion/autofmt/piptool/packtool/pdftool/gittool)
|
||||
- YAML 配置支持 ``cli:`` 段声明命令行参数 schema,由 ``build_cli_parser`` 自动生成 argparse
|
||||
- 删除 13 个冗余 ``.py`` 入口脚本,统一通过 ``pf`` 调用
|
||||
- ``run()`` 在 ``verbose=True`` 时自动把 verbose 标记应用到所有 spec
|
||||
- 全局选项 ``--verbose`` 改为 ``--quiet``(默认显示执行过程)
|
||||
- ``cmd`` 任务成功时打印 stdout(此前被静默丢弃)
|
||||
- ``gittool`` 用 ``CLEAN_EXCLUDES`` 数组变量配置 ``git clean -e`` 参数
|
||||
|
||||
YAML 任务编排
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- 支持 ``variables`` 变量定义,``${VAR}`` 在 cmd/env/cwd 中替换
|
||||
- 列表变量展开为 cmd 数组多个元素
|
||||
- ``cli:`` 段支持 subcommands/positional/options 三级 schema
|
||||
- 支持 ``type: path`` 自动转为 ``pathlib.Path``
|
||||
|
||||
文档
|
||||
~~~~
|
||||
|
||||
- 搭建 Sphinx 文档,发布到 ReadTheDocs
|
||||
- 更新 README:CLI 示例改为 ``pf`` 统一入口,模块结构表补全
|
||||
|
||||
0.3.x
|
||||
-----
|
||||
|
||||
- 新增 YAML 任务编排(GitHub Actions 风格 schema)
|
||||
- 新增 ``fn:`` 函数引用与 ``register_fn`` / ``get_fn`` 注册中心
|
||||
- 新增 ``compose`` / ``GraphComposer`` 多图组合
|
||||
- 新增 ``task_template`` 任务模板工厂
|
||||
- 新增 ``concurrency_key`` + ``concurrency_limits`` 并发限制
|
||||
- 新增 ``JSONBackend`` 断点续跑与 ``batch()`` 批量落盘
|
||||
- 新增 ``cache_key`` 缓存键函数
|
||||
- 新增条件执行(``IS_WINDOWS`` / ``HAS_INSTALLED`` / ``ENV_VAR_EQUALS`` 等)
|
||||
- 四种执行策略:``sequential`` / ``thread`` / ``async`` / ``dependency``
|
||||
- 参数名即依赖的上下文注入机制
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Sphinx 配置.
|
||||
|
||||
ReadTheDocs 构建 PyFlowX 文档站。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 确保 src/ 在 sys.path 中, autodoc 能导入 pyflowx
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
|
||||
|
||||
from pyflowx import __version__
|
||||
|
||||
# -- 项目信息 --------------------------------------------------------------
|
||||
project = "PyFlowX"
|
||||
author = "pyflowx"
|
||||
copyright = "2024, pyflowx"
|
||||
release = __version__
|
||||
version = __version__
|
||||
|
||||
# -- Sphinx 配置 -----------------------------------------------------------
|
||||
extensions = [
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.napoleon",
|
||||
"sphinx.ext.viewcode",
|
||||
"sphinx.ext.intersphinx",
|
||||
"myst_parser",
|
||||
]
|
||||
|
||||
# -- 主题 ------------------------------------------------------------------
|
||||
html_theme = "sphinx_rtd_theme"
|
||||
html_static_path = ["_static"]
|
||||
|
||||
# -- autodoc 配置 ----------------------------------------------------------
|
||||
autodoc_default_options = {
|
||||
"members": True,
|
||||
"undoc-members": True,
|
||||
"show-inheritance": True,
|
||||
"member-order": "bysource",
|
||||
}
|
||||
autodoc_type_hints = "description"
|
||||
autodoc_typehints_format = "short"
|
||||
|
||||
# -- napoleon 配置 (Google/NumPy docstring 兼容) --------------------------
|
||||
napoleon_google_docstring = True
|
||||
napoleon_numpy_docstring = True
|
||||
napoleon_include_init_with_doc = False
|
||||
napoleon_include_private_with_doc = False
|
||||
napoleon_include_special_with_doc = True
|
||||
|
||||
# -- intersphinx -----------------------------------------------------------
|
||||
intersphinx_mapping = {
|
||||
"python": ("https://docs.python.org/3", None),
|
||||
}
|
||||
|
||||
# -- 全局选项 ---------------------------------------------------------------
|
||||
language = "zh_CN"
|
||||
master_doc = "index"
|
||||
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
|
||||
source_suffix = {
|
||||
".rst": "restructuredtext",
|
||||
".md": "markdown",
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
pf 统一 CLI 入口
|
||||
================
|
||||
|
||||
所有工具通过 ``pf <tool> [command] [options]`` 调用。工具定义在 ``cli/configs/`` 目录下的 YAML 文件中。
|
||||
|
||||
基本用法
|
||||
--------
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pf # 列出所有可用工具
|
||||
pf filedate # 查看 filedate 工具帮助
|
||||
pf filedate add a.txt # 调用 filedate 的 add 子命令
|
||||
pf gitt c # 调用 gittool 的 c 子命令
|
||||
pf pymake b # 调用 pymake 的 b 别名
|
||||
|
||||
全局选项
|
||||
--------
|
||||
|
||||
所有 YAML 工具支持以下全局选项:
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 25 75
|
||||
|
||||
* - 选项
|
||||
- 说明
|
||||
* - ``--dry-run``
|
||||
- 仅打印执行计划,不执行
|
||||
* - ``--quiet`` / ``-q``
|
||||
- 减少输出,不显示执行过程
|
||||
* - ``--strategy``
|
||||
- 执行策略(``sequential`` / ``thread`` / ``async`` / ``dependency``)
|
||||
* - ``--list``
|
||||
- 列出所有任务名后退出
|
||||
|
||||
默认 ``verbose`` 开启,显示执行过程(任务开始/命令/返回码/任务成功)。``--quiet`` 关闭。
|
||||
|
||||
YAML 配置工具
|
||||
--------------
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 20 15 65
|
||||
|
||||
* - 工具
|
||||
- 别名
|
||||
- 说明
|
||||
* - ``filedate``
|
||||
- ``fd``
|
||||
- 文件日期处理
|
||||
* - ``filelevel``
|
||||
- ``fl``
|
||||
- 文件等级重命名
|
||||
* - ``folderback``
|
||||
- ``fb``
|
||||
- 文件夹备份
|
||||
* - ``folderzip``
|
||||
- ``fz``
|
||||
- 文件夹压缩
|
||||
* - ``gittool``
|
||||
- ``gitt``
|
||||
- Git 执行工具
|
||||
* - ``lscalc``
|
||||
- ``ls``
|
||||
- LS-DYNA 计算工具
|
||||
* - ``packtool``
|
||||
- ``pack``
|
||||
- Python 打包工具
|
||||
* - ``pdftool``
|
||||
- ``pdf``
|
||||
- PDF 文件工具集
|
||||
* - ``piptool``
|
||||
- ``pip``
|
||||
- pip 包管理工具
|
||||
* - ``screenshot``
|
||||
- ``ss``
|
||||
- 截图工具
|
||||
* - ``sshcopyid``
|
||||
- ``ssh``
|
||||
- SSH 密钥部署工具
|
||||
* - ``autofmt``
|
||||
- ``af``
|
||||
- 自动格式化工具
|
||||
* - ``bumpversion``
|
||||
- ``bump``
|
||||
- 版本号自动管理工具
|
||||
|
||||
传统工具
|
||||
--------
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 20 80
|
||||
|
||||
* - 工具
|
||||
- 说明
|
||||
* - ``pymake``
|
||||
- 构建工具(替代 Makefile),如 ``pf pymake b`` 构建
|
||||
* - ``yamlrun``
|
||||
- YAML pipeline 执行器,``pf yamlrun pipeline.yaml``
|
||||
* - ``profiler``
|
||||
- 性能分析
|
||||
* - ``emlman``
|
||||
- 邮件管理
|
||||
* - ``reseticon``
|
||||
- 重置图标缓存
|
||||
|
||||
自定义工具
|
||||
----------
|
||||
|
||||
在 ``cli/configs/`` 目录新建 ``<tool>.yaml`` 即可被 ``pf`` 自动发现:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
# cli/configs/mytool.yaml
|
||||
strategy: sequential
|
||||
variables:
|
||||
MSG: "hello"
|
||||
cli:
|
||||
description: "我的工具"
|
||||
usage: "pf mytool [command]"
|
||||
subcommands:
|
||||
greet:
|
||||
help: "打招呼"
|
||||
jobs:
|
||||
greet:
|
||||
cmd: ["echo", "${MSG}"]
|
||||
|
||||
执行::
|
||||
|
||||
pf mytool greet
|
||||
|
||||
CliRunner(编程式)
|
||||
-------------------
|
||||
|
||||
``CliRunner`` 把多个 Graph 映射为命令行子命令,适合构建项目专属构建工具:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
runner = px.CliRunner(
|
||||
strategy="sequential",
|
||||
description="My Build Tool",
|
||||
graphs={
|
||||
"clean": clean_graph,
|
||||
"build": build_graph,
|
||||
"test": test_graph,
|
||||
},
|
||||
)
|
||||
runner.run_cli() # 解析 sys.argv 并执行
|
||||
|
||||
命令行::
|
||||
|
||||
pf pymake clean
|
||||
pf pymake build --strategy thread
|
||||
pf pymake test --dry-run
|
||||
pf pymake --list
|
||||
pf pymake --quiet
|
||||
@@ -0,0 +1,93 @@
|
||||
执行策略与 run()
|
||||
=================
|
||||
|
||||
``run()`` 是执行入口,支持四种策略:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
report = px.run(
|
||||
graph,
|
||||
strategy="async", # sequential | thread | async | dependency
|
||||
max_workers=8, # thread 策略的线程池大小
|
||||
concurrency_limits={"db": 2}, # 按 concurrency_key 限流
|
||||
dry_run=False, # True = 仅打印计划
|
||||
verbose=True, # True = 打印执行过程
|
||||
on_event=callback, # 状态转换回调
|
||||
state=px.JSONBackend("state.json"), # 断点续跑后端
|
||||
continue_on_error=False, # True = 单任务失败不中断整体
|
||||
)
|
||||
|
||||
策略对比
|
||||
--------
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 18 18 30 16 18
|
||||
|
||||
* - 策略
|
||||
- 并发模型
|
||||
- 适用场景
|
||||
- 同步任务
|
||||
- 异步任务
|
||||
* - ``sequential``
|
||||
- 串行
|
||||
- 调试、CPU 密集
|
||||
- 直接调用
|
||||
- 事件循环
|
||||
* - ``thread``
|
||||
- 线程池
|
||||
- I/O 密集同步
|
||||
- 线程池
|
||||
- 不支持
|
||||
* - ``async``
|
||||
- 事件循环
|
||||
- I/O 密集异步
|
||||
- 卸载到线程池
|
||||
- 事件循环
|
||||
* - ``dependency``
|
||||
- 依赖驱动
|
||||
- 最大化并行度
|
||||
- 卸载到线程池
|
||||
- 事件循环
|
||||
|
||||
所有策略都遵循 ``RetryPolicy``、``timeout``、上下文注入、状态后端、``concurrency_limits``,
|
||||
并发出 ``TaskEvent``(RUNNING/SUCCESS/FAILED/SKIPPED)。``dependency`` 策略无层屏障:
|
||||
任务在其所有硬依赖完成后立即启动。
|
||||
|
||||
上下文注入规则
|
||||
--------------
|
||||
|
||||
按顺序求值:
|
||||
|
||||
1. **标注为 ``Context``** 的参数 → 接收完整上游结果映射
|
||||
2. **名称匹配依赖** 的参数 → 接收该依赖的结果(含软依赖,缺失时注入默认值)
|
||||
3. **``**kwargs``** 参数 → 接收所有依赖结果(dict)
|
||||
4. **``TaskSpec.args`` / ``kwargs``** → 为非依赖参数提供静态值
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
def aggregate(ctx: px.Context) -> Dict[str, Any]:
|
||||
"""ctx 包含所有 depends_on 任务的返回值。"""
|
||||
return dict(ctx)
|
||||
|
||||
def merge(fetch_a: str, fetch_b: str) -> str:
|
||||
"""fetch_a / fetch_b 自动注入。"""
|
||||
return fetch_a + fetch_b
|
||||
|
||||
断点续跑
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from pyflowx import JSONBackend
|
||||
|
||||
backend = JSONBackend("state.json", ttl=3600)
|
||||
report = px.run(graph, strategy="sequential", state=backend)
|
||||
|
||||
``run()`` 内部以 ``backend.batch()`` 包裹整个执行:所有 ``save`` 延迟到运行结束时统一落盘一次。
|
||||
|
||||
缓存键:默认存储键为任务名。配置 ``cache_key`` 函数后,键为 ``"name:cache_key_value"``。
|
||||
|
||||
完整 API 说明详见 :doc:`/api`。
|
||||
@@ -0,0 +1,50 @@
|
||||
Graph —— DAG 构建
|
||||
=================
|
||||
|
||||
``Graph`` 管理任务集合,提供建构建、校验、分层、可视化能力。
|
||||
|
||||
构建方式
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# 图级默认值:TaskSpec 字段为 None 时回退
|
||||
defaults = px.GraphDefaults(retry=px.RetryPolicy(max_attempts=2), timeout=60.0)
|
||||
|
||||
graph = px.Graph.from_specs([...], defaults=defaults) # 整批校验(推荐)
|
||||
|
||||
# 或增量构建
|
||||
graph = px.Graph(defaults=defaults)
|
||||
graph.add(px.TaskSpec("a", fn_a))
|
||||
graph.add(px.TaskSpec("b", fn_b, ("a",)))
|
||||
|
||||
常用方法
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
graph.validate() # 显式校验(环检测)
|
||||
graph.layers() # 拓扑分层(Kahn 算法)
|
||||
graph.to_mermaid() # Mermaid 可视化
|
||||
graph.describe() # 人类可读摘要
|
||||
graph.subgraph(("api",)) # 按标签切片
|
||||
graph.subgraph_by_names(("a", "b")) # 按名称切片
|
||||
graph.map("fetch", [1, 2, 3], lambda i: TaskSpec(f"fetch_{i}", ...)) # 批量 fan-out
|
||||
|
||||
图组合
|
||||
------
|
||||
|
||||
``compose`` / ``GraphComposer`` 把带字符串引用的多个图展开为纯 ``Graph``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
graphs = {
|
||||
"build": px.Graph.from_specs([px.TaskSpec("b", cmd=["echo", "b"])]),
|
||||
"all": px.Graph.from_specs(["build", px.TaskSpec("t", cmd=["echo", "t"])]),
|
||||
}
|
||||
resolved = px.compose(graphs) # "all" 图中的 "build" 引用被展开
|
||||
|
||||
引用格式:``"command_name"``(整个图)或 ``"command_name.task_name"``(特定任务)。
|
||||
``CliRunner`` 内部自动调用 ``compose``。
|
||||
|
||||
完整方法说明详见 :doc:`/api`。
|
||||
@@ -0,0 +1,89 @@
|
||||
TaskSpec —— 任务描述
|
||||
=====================
|
||||
|
||||
``TaskSpec`` 是不可变的任务描述符(``Generic[T]``,返回类型一路传到 ``RunReport``),是唯一需要配置的东西。
|
||||
|
||||
主要参数说明:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
px.TaskSpec(
|
||||
name="fetch_user", # 唯一标识
|
||||
fn=fetch_user, # 同步或异步函数
|
||||
cmd=["curl", "..."], # 或: 执行命令(覆盖 fn)
|
||||
depends_on=("auth",), # 硬依赖(参与拓扑分层)
|
||||
soft_depends_on=("cache",), # 软依赖(仅注入,不参与分层)
|
||||
args=(uid,), # 静态位置参数(追加在注入参数后)
|
||||
kwargs={"timeout": 30}, # 静态关键字参数
|
||||
retry=px.RetryPolicy(max_attempts=3, delay=1.0, backoff=2.0),
|
||||
timeout=30.0, # 超时秒数(None = 不限制)
|
||||
tags=("api", "user"), # 自由标签,用于子图过滤
|
||||
conditions=(is_prod,), # 条件函数列表(全部为 True 才执行)
|
||||
priority=10, # 同层内优先级(高优先执行,默认 0)
|
||||
concurrency_key="db", # 并发分组键(配合 concurrency_limits 限流)
|
||||
cache_key=lambda ctx: str(ctx.get("uid")), # 缓存键函数
|
||||
hooks=px.TaskHooks(pre_run=..., post_run=..., on_failure=...),
|
||||
cwd=Path("/tmp"), # 命令工作目录(仅 cmd 模式)
|
||||
env={"DEBUG": "1"}, # 环境变量覆盖
|
||||
verbose=True, # 打印命令输出(仅 cmd 模式)
|
||||
skip_if_missing=True, # 命令不存在时自动跳过(仅 list[str] cmd)
|
||||
allow_upstream_skip=False, # 上游 SKIPPED/FAILED 时是否仍执行
|
||||
continue_on_error=False, # 本任务失败是否不中断整体
|
||||
)
|
||||
|
||||
两种任务形态
|
||||
------------
|
||||
|
||||
- **函数任务**(``fn``):普通 Python 函数,参数名驱动自动注入
|
||||
- **命令任务**(``cmd``):执行外部命令,支持 ``list[str]``、``str``(shell)、``Callable`` 三种形态
|
||||
|
||||
``skip_if_missing=True`` 时,``list[str]`` 类型的 ``cmd`` 会通过 ``shutil.which`` 检查命令是否存在,不存在则跳过任务(标记为 ``SKIPPED``)而非失败。
|
||||
|
||||
重试策略
|
||||
--------
|
||||
|
||||
``RetryPolicy`` 配置重试次数、延迟、退避:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
retry = px.RetryPolicy(
|
||||
max_attempts=3, # 最大尝试次数
|
||||
delay=1.0, # 初始延迟秒数
|
||||
backoff=2.0, # 退避倍数
|
||||
jitter=0.1, # 随机抖动(避免惊群)
|
||||
retry_on=(ConnectionError,), # 仅对这些异常重试
|
||||
)
|
||||
|
||||
任务钩子
|
||||
--------
|
||||
|
||||
``TaskHooks`` 在任务生命周期触发(异常仅记录,不影响任务状态):
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
hooks = px.TaskHooks(
|
||||
pre_run=lambda spec: print(f"start {spec.name}"),
|
||||
post_run=lambda spec, value: print(f"done {spec.name}"),
|
||||
on_failure=lambda spec, exc: alert(spec.name, exc),
|
||||
)
|
||||
px.TaskSpec("task", fn=work, hooks=hooks)
|
||||
|
||||
任务模板
|
||||
--------
|
||||
|
||||
``task_template`` 工厂批量生成相似 TaskSpec:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
fetch = px.task_template(
|
||||
fn=fetch_url,
|
||||
retry=px.RetryPolicy(max_attempts=5),
|
||||
timeout=30.0,
|
||||
tags=("api",),
|
||||
)
|
||||
graph = px.Graph.from_specs([
|
||||
fetch("users", url="https://api.example.com/users"),
|
||||
fetch("posts", url="https://api.example.com/posts"),
|
||||
])
|
||||
|
||||
完整字段说明详见 :doc:`/api`。
|
||||
@@ -0,0 +1,164 @@
|
||||
YAML 任务编排
|
||||
=============
|
||||
|
||||
PyFlowX 支持 GitHub Actions 风格的声明式 YAML 任务编排,从 YAML 文件直接加载任务图。
|
||||
|
||||
编程式 API
|
||||
----------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# 从 YAML 文件加载任务图
|
||||
graph = px.Graph.from_yaml("pipeline.yaml")
|
||||
report = px.run(graph, strategy="thread")
|
||||
|
||||
# 或用函数式 API
|
||||
graph = px.load_yaml("pipeline.yaml")
|
||||
|
||||
# 从字符串解析
|
||||
graph = px.parse_yaml_string("""
|
||||
jobs:
|
||||
hello:
|
||||
cmd: ["echo", "hello"]
|
||||
""")
|
||||
|
||||
YAML Schema
|
||||
-----------
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
strategy: thread # 图级默认策略
|
||||
defaults: # 图级默认值
|
||||
retry: {max_attempts: 3}
|
||||
verbose: true
|
||||
env: {CI: "true"}
|
||||
|
||||
variables: # 变量定义 (可在 cmd/env 中 ${VAR} 引用)
|
||||
OUTPUT: "dist"
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
cmd: ["git", "clone", "..."]
|
||||
runs-on: linux
|
||||
|
||||
build:
|
||||
needs: [setup] # 依赖列表
|
||||
cmd: ["python", "-m", "build"]
|
||||
timeout: 300
|
||||
retry: {max_attempts: 2, delay: 1.0}
|
||||
|
||||
test:
|
||||
needs: [build]
|
||||
cmd: ["python${{ matrix.version }}", "-m", "pytest"]
|
||||
strategy:
|
||||
matrix: # 笛卡尔积展开为 6 个任务
|
||||
version: ["3.8", "3.9", "3.10"]
|
||||
os: ["linux", "macos"]
|
||||
if: "env.CI" # 条件: 环境变量存在
|
||||
|
||||
lint:
|
||||
needs: [build]
|
||||
cmd: ["ruff", "check"]
|
||||
if: "env.CI == 'true'"
|
||||
|
||||
deploy:
|
||||
needs: [test, lint] # 矩阵依赖自动展开
|
||||
cmd: ["twine", "upload"]
|
||||
if: "env.DEPLOY_TOKEN != ''"
|
||||
allow-upstream-skip: true
|
||||
concurrency-key: deploy_lock
|
||||
|
||||
字段映射
|
||||
--------
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 30 30 40
|
||||
|
||||
* - YAML 字段
|
||||
- TaskSpec 字段
|
||||
- 说明
|
||||
* - ``jobs.<id>``
|
||||
- ``name``
|
||||
- job ID 作为任务名
|
||||
* - ``cmd`` / ``run``
|
||||
- ``cmd``
|
||||
- ``cmd`` 为列表形式,``run`` 为 shell 字符串
|
||||
* - ``needs``
|
||||
- ``depends_on``
|
||||
- 依赖列表(矩阵任务自动展开)
|
||||
* - ``if``
|
||||
- ``conditions``
|
||||
- ``success()`` / ``always()`` / ``env.VAR`` / ``env.VAR == 'x'``
|
||||
* - ``strategy.matrix``
|
||||
- 矩阵扇出
|
||||
- 笛卡尔积展开为多个任务
|
||||
* - ``${{ matrix.key }}``
|
||||
- 占位符
|
||||
- 在 cmd/run/cwd/env 中替换
|
||||
* - ``timeout``
|
||||
- ``timeout``
|
||||
- 超时秒数
|
||||
* - ``retry``
|
||||
- ``retry``
|
||||
- ``{max_attempts, delay, backoff, jitter}``
|
||||
* - ``cwd``
|
||||
- ``cwd``
|
||||
- 工作目录
|
||||
* - ``env``
|
||||
- ``env``
|
||||
- 环境变量
|
||||
* - ``verbose``
|
||||
- ``verbose``
|
||||
- 详细输出
|
||||
* - ``continue-on-error``
|
||||
- ``continue_on_error``
|
||||
- 失败不中止整图
|
||||
* - ``skip-if-missing``
|
||||
- ``skip_if_missing``
|
||||
- 命令不存在时跳过
|
||||
* - ``allow-upstream-skip``
|
||||
- ``allow_upstream_skip``
|
||||
- 上游跳过时仍执行
|
||||
* - ``priority``
|
||||
- ``priority``
|
||||
- 同层优先级
|
||||
* - ``concurrency-key``
|
||||
- ``concurrency_key``
|
||||
- 并发限制键
|
||||
* - ``tags``
|
||||
- ``tags``
|
||||
- 自由标签
|
||||
* - ``runs-on``
|
||||
- ``tags``(追加)
|
||||
- 运行环境标签
|
||||
|
||||
CLI 配置段(``cli:``)
|
||||
----------------------
|
||||
|
||||
工具 YAML 还可定义 ``cli:`` 段,声明命令行参数 schema,由 ``pf`` 自动解析:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
cli:
|
||||
description: "FileDate - 文件日期处理工具"
|
||||
usage: "pf filedate <command> [files...]"
|
||||
subcommands:
|
||||
add:
|
||||
help: "添加日期前缀"
|
||||
positional:
|
||||
- name: FILES
|
||||
nargs: "+"
|
||||
type: path
|
||||
help: "文件路径"
|
||||
options:
|
||||
- name: CLEAR
|
||||
flag: "--clear"
|
||||
action: store_true
|
||||
help: "清除已有日期前缀"
|
||||
|
||||
支持的 ``type``:``str`` / ``int`` / ``float`` / ``path``。
|
||||
|
||||
完整 API 说明详见 :doc:`/api`。
|
||||
@@ -0,0 +1,56 @@
|
||||
PyFlowX 文档
|
||||
============
|
||||
|
||||
PyFlowX 是一个轻量、类型安全的 DAG 任务调度器:**参数名就是依赖声明**。
|
||||
无需装饰器、无需样板包装器,写一个普通函数,框架按参数名自动注入上游结果。
|
||||
|
||||
特性
|
||||
----
|
||||
|
||||
- **零样板** —— 参数名即依赖,框架自动注入上游结果
|
||||
- **四种执行策略** —— sequential(串行)、thread(线程池)、async(事件循环)、dependency(依赖驱动,最大化并行)
|
||||
- **类型安全** —— ``TaskSpec[T]`` 把返回类型一路传到 ``RunReport``
|
||||
- **DAG 校验** —— 构建时即时校验重名、缺失依赖、环
|
||||
- **自动分层** —— Kahn 算法分组,同层任务可并行
|
||||
- **重试与超时** —— 每个任务独立配置 ``RetryPolicy`` 与 ``timeout``
|
||||
- **并发限制** —— ``concurrency_key`` + ``concurrency_limits`` 按组限流
|
||||
- **断点续跑** —— ``MemoryBackend`` / ``JSONBackend``,成功结果可缓存复用
|
||||
- **命令任务** —— ``cmd`` 参数直接执行外部命令
|
||||
- **条件执行** —— ``conditions`` 按平台、环境变量等条件跳过任务
|
||||
- **YAML 任务编排** —— GitHub Actions 风格声明式任务图
|
||||
- **pf 统一 CLI** —— ``pf <tool> [command]`` 调用所有工具
|
||||
- **最小依赖** —— 仅依赖标准库 + PyYAML
|
||||
|
||||
文档导航
|
||||
--------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: 入门
|
||||
|
||||
installation
|
||||
quickstart
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: 用户指南
|
||||
|
||||
guide/task
|
||||
guide/graph
|
||||
guide/execution
|
||||
guide/yaml
|
||||
guide/cli
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: 参考
|
||||
|
||||
api
|
||||
changelog
|
||||
|
||||
索引
|
||||
----
|
||||
|
||||
* :ref:`genindex`
|
||||
* :ref:`modindex`
|
||||
* :ref:`search`
|
||||
@@ -0,0 +1,51 @@
|
||||
安装
|
||||
====
|
||||
|
||||
PyFlowX 支持 Python 3.8+,仅依赖标准库与 PyYAML(3.8 额外需要 ``graphlib_backport`` 和 ``typing-extensions``)。
|
||||
|
||||
pip 安装
|
||||
--------
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install pyflowx
|
||||
|
||||
uv 安装
|
||||
-------
|
||||
|
||||
推荐使用 `uv <https://docs.astral.sh/uv/>`_:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
uv add pyflowx
|
||||
|
||||
可选依赖
|
||||
--------
|
||||
|
||||
``office`` —— PDF/图片处理(pdftool、screenshot 等工具需要):
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install pyflowx[office]
|
||||
|
||||
``dev`` —— 开发工具链(ruff、pyrefly、pytest、tox 等):
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install pyflowx[dev]
|
||||
|
||||
验证安装
|
||||
--------
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pf --version
|
||||
|
||||
输出示例::
|
||||
|
||||
PyFlowX 0.4.5
|
||||
|
||||
下一步
|
||||
------
|
||||
|
||||
前往 :doc:`quickstart` 开始使用。
|
||||
@@ -0,0 +1,87 @@
|
||||
快速上手
|
||||
========
|
||||
|
||||
核心思想:**参数名即依赖**。写一个普通函数,参数名匹配上游任务名,框架自动注入结果。
|
||||
|
||||
最小示例
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
def extract() -> list[int]:
|
||||
return [1, 2, 3]
|
||||
|
||||
# 参数名 extract 自动匹配上游任务名 → 自动注入
|
||||
def double(extract: list[int]) -> list[int]:
|
||||
return [x * 2 for x in extract]
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("extract", extract),
|
||||
px.TaskSpec("double", double, ("extract",)),
|
||||
])
|
||||
|
||||
report = px.run(graph, strategy="sequential")
|
||||
print(report["double"]) # [2, 4, 6]
|
||||
|
||||
三种任务形态
|
||||
------------
|
||||
|
||||
1. **函数任务**(``fn``):普通 Python 函数,参数名驱动自动注入
|
||||
2. **命令任务**(``cmd``):执行外部命令,支持 ``list[str]`` / ``str``(shell)/ ``Callable``
|
||||
3. **YAML 声明式**:从 YAML 文件加载任务图
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("list", cmd=["ls", "-la"]),
|
||||
px.TaskSpec("greet", fn=lambda: "hello"),
|
||||
])
|
||||
|
||||
执行策略
|
||||
--------
|
||||
|
||||
PyFlowX 提供四种执行策略:
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 20 20 60
|
||||
|
||||
* - 策略
|
||||
- 并发模型
|
||||
- 适用场景
|
||||
* - ``sequential``
|
||||
- 串行
|
||||
- 调试、CPU 密集
|
||||
* - ``thread``
|
||||
- 线程池
|
||||
- I/O 密集同步
|
||||
* - ``async``
|
||||
- 事件循环
|
||||
- I/O 密集异步
|
||||
* - ``dependency``
|
||||
- 依赖驱动
|
||||
- 最大化并行度(默认推荐)
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
report = px.run(graph, strategy="dependency")
|
||||
|
||||
结果访问
|
||||
--------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
report["task_name"] # 任务返回值
|
||||
report.result_of("task_name") # 完整 TaskResult
|
||||
report.success # 整体是否成功
|
||||
report.summary() # 统计字典
|
||||
report.failed_tasks() # 失败任务名列表
|
||||
|
||||
下一步
|
||||
------
|
||||
|
||||
- :doc:`guide/task` —— TaskSpec 详细配置
|
||||
- :doc:`guide/yaml` —— YAML 声明式任务编排
|
||||
- :doc:`guide/cli` —— ``pf`` 统一 CLI 入口
|
||||
+14
-34
@@ -13,7 +13,7 @@ classifiers = [
|
||||
]
|
||||
dependencies = [
|
||||
"graphlib_backport >= 1.0.0; python_version < '3.9'",
|
||||
"typing-extensions>=4.13.2",
|
||||
"typing-extensions>=4.13.2; python_version < '3.13'",
|
||||
]
|
||||
description = "Lightweight, type-safe DAG task scheduler with multi-strategy execution."
|
||||
keywords = ["async", "dag", "scheduler", "task", "workflow"]
|
||||
@@ -21,34 +21,12 @@ license = { text = "MIT" }
|
||||
name = "pyflowx"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.8"
|
||||
version = "0.2.8"
|
||||
version = "0.4.8"
|
||||
|
||||
[project.scripts]
|
||||
autofmt = "pyflowx.cli.autofmt:main"
|
||||
bumpversion = "pyflowx.cli.bumpversion:main"
|
||||
clr = "pyflowx.cli.clearscreen:main"
|
||||
emlman = "pyflowx.cli.emlmanager:main"
|
||||
envdev = "pyflowx.cli.envdev:main"
|
||||
envpy = "pyflowx.cli.envpy:main"
|
||||
envqt = "pyflowx.cli.envqt:main"
|
||||
envrs = "pyflowx.cli.envrs:main"
|
||||
filedate = "pyflowx.cli.filedate:main"
|
||||
filelvl = "pyflowx.cli.filelevel:main"
|
||||
foldback = "pyflowx.cli.folderback:main"
|
||||
foldzip = "pyflowx.cli.folderzip:main"
|
||||
gitt = "pyflowx.cli.gittool:main"
|
||||
lscalc = "pyflowx.cli.lscalc:main"
|
||||
msdown = "pyflowx.cli.llm.msdownload:main"
|
||||
packtool = "pyflowx.cli.packtool:main"
|
||||
pdftool = "pyflowx.cli.pdftool:main"
|
||||
piptool = "pyflowx.cli.piptool:main"
|
||||
pymake = "pyflowx.cli.pymake:main"
|
||||
reseticon = "pyflowx.cli.reseticoncache:main"
|
||||
scrcap = "pyflowx.cli.screenshot:main"
|
||||
sglang = "pyflowx.cli.llm.sglang:main"
|
||||
sshcopy = "pyflowx.cli.sshcopyid:main"
|
||||
taskk = "pyflowx.cli.taskkill:main"
|
||||
wch = "pyflowx.cli.which:main"
|
||||
emlman = "pyflowx.cli.emlmanager:main"
|
||||
pf = "pyflowx.cli.pf:main"
|
||||
pxp = "pyflowx.cli.profiler:main"
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
@@ -66,9 +44,7 @@ dev = [
|
||||
"tox-uv>=1.13.1",
|
||||
"tox>=4.25.0",
|
||||
]
|
||||
llm = [
|
||||
"sglang[all]==0.5.10rc0; python_version >= '3.10' and sys_platform == 'linux'",
|
||||
]
|
||||
docs = ["myst-parser>=3.0", "sphinx-rtd-theme>=2.0", "sphinx>=7.0"]
|
||||
office = [
|
||||
"pillow>=10.4.0",
|
||||
"pymupdf>=1.24.11",
|
||||
@@ -80,6 +56,9 @@ office = [
|
||||
build-backend = "hatchling.build"
|
||||
requires = ["hatchling"]
|
||||
|
||||
[tool.uv]
|
||||
required-version = ">=0.5.0"
|
||||
|
||||
[[tool.uv.index]]
|
||||
default = true
|
||||
url = "https://mirrors.aliyun.com/pypi/simple/"
|
||||
@@ -94,12 +73,12 @@ packages = ["src/pyflowx"]
|
||||
pyflowx = { workspace = true }
|
||||
|
||||
[dependency-groups]
|
||||
dev = ["pyflowx[dev,office,llm]"]
|
||||
dev = ["pyflowx[dev,docs,office]"]
|
||||
|
||||
[tool.coverage.run]
|
||||
branch = true
|
||||
concurrency = ["thread"]
|
||||
omit = ["src/pyflowx/examples/*", "tests/*"]
|
||||
omit = ["src/pyflowx/cli/*", "tests/*"]
|
||||
source = ["pyflowx"]
|
||||
|
||||
[tool.coverage.report]
|
||||
@@ -109,7 +88,7 @@ exclude_lines = [
|
||||
"pragma: no cover",
|
||||
"raise NotImplementedError",
|
||||
]
|
||||
fail_under = 80
|
||||
fail_under = 95
|
||||
show_missing = true
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
@@ -152,7 +131,8 @@ select = [
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"**/tests/**" = ["ARG001", "ARG002"]
|
||||
"**/tests/**" = ["ARG001", "ARG002"]
|
||||
"src/pyflowx/tools.py" = ["PLR0911"]
|
||||
|
||||
[tool.pyrefly]
|
||||
preset = "strict"
|
||||
|
||||
+19
-11
@@ -58,15 +58,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .conditions import (
|
||||
IS_LINUX,
|
||||
IS_MACOS,
|
||||
IS_POSIX,
|
||||
IS_WINDOWS,
|
||||
BuiltinConditions,
|
||||
Condition,
|
||||
Constants,
|
||||
)
|
||||
from .command import run_command
|
||||
from .compose import GraphComposer, compose
|
||||
from .conditions import IS_LINUX, IS_MACOS, IS_POSIX, IS_WINDOWS, BuiltinConditions, Condition, Constants
|
||||
from .context import Context, build_call_args, describe_injection
|
||||
from .errors import (
|
||||
CycleError,
|
||||
@@ -79,7 +73,8 @@ from .errors import (
|
||||
TaskTimeoutError,
|
||||
)
|
||||
from .executors import Strategy, run
|
||||
from .graph import Graph, GraphComposer, GraphDefaults, compose
|
||||
from .graph import Graph, GraphDefaults
|
||||
from .profiling import ProfileReport, TaskProfile
|
||||
from .report import RunReport
|
||||
from .runner import CliExitCode, CliRunner
|
||||
from .storage import JSONBackend, MemoryBackend, StateBackend
|
||||
@@ -92,10 +87,13 @@ from .task import (
|
||||
TaskResult,
|
||||
TaskSpec,
|
||||
TaskStatus,
|
||||
cmd,
|
||||
task,
|
||||
task_template,
|
||||
)
|
||||
from .tools import ToolSpec, list_subcommands, list_tools, run_tool, tool
|
||||
|
||||
__version__ = "0.3.2"
|
||||
__version__ = "0.4.8"
|
||||
|
||||
__all__ = [
|
||||
"IS_LINUX",
|
||||
@@ -118,6 +116,7 @@ __all__ = [
|
||||
"JSONBackend",
|
||||
"MemoryBackend",
|
||||
"MissingDependencyError",
|
||||
"ProfileReport",
|
||||
"PyFlowXError",
|
||||
"RetryPolicy",
|
||||
"RunReport",
|
||||
@@ -128,13 +127,22 @@ __all__ = [
|
||||
"TaskEvent",
|
||||
"TaskFailedError",
|
||||
"TaskHooks",
|
||||
"TaskProfile",
|
||||
"TaskResult",
|
||||
"TaskSpec",
|
||||
"TaskStatus",
|
||||
"TaskTimeoutError",
|
||||
"ToolSpec",
|
||||
"build_call_args",
|
||||
"cmd",
|
||||
"compose",
|
||||
"describe_injection",
|
||||
"list_subcommands",
|
||||
"list_tools",
|
||||
"run",
|
||||
"run_command",
|
||||
"run_tool",
|
||||
"task",
|
||||
"task_template",
|
||||
"tool",
|
||||
]
|
||||
|
||||
@@ -1,263 +0,0 @@
|
||||
"""版本号自动管理工具.
|
||||
|
||||
使用 TaskSpec 模式实现, 支持语义化版本管理和多文件格式的版本号更新.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
BumpVersionType = Literal["patch", "minor", "major"]
|
||||
|
||||
# 针对不同文件类型的版本号匹配模式
|
||||
# pyproject.toml: version = "X.Y.Z" 或 version = 'X.Y.Z'
|
||||
_PYPROJECT_VERSION_PATTERN = re.compile(
|
||||
r'(?:^|\n)\s*version\s*=\s*["\']'
|
||||
r"(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
|
||||
r"(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
|
||||
r"(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?"
|
||||
r'["\']',
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
# __init__.py: __version__ = "X.Y.Z" 或 __version__ = 'X.Y.Z'
|
||||
_INIT_VERSION_PATTERN = re.compile(
|
||||
r'(?:^|\n)\s*__version__\s*=\s*["\']'
|
||||
r"(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
|
||||
r"(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
|
||||
r"(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?"
|
||||
r'["\']',
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def _get_pattern_for_file(file_name: str) -> re.Pattern[str] | None:
|
||||
"""根据文件类型获取对应的正则表达式.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_name : str
|
||||
文件名
|
||||
|
||||
Returns
|
||||
-------
|
||||
re.Pattern[str] | None
|
||||
对应的正则表达式,如果无法确定则返回 None
|
||||
"""
|
||||
if file_name == "pyproject.toml":
|
||||
return _PYPROJECT_VERSION_PATTERN
|
||||
if file_name == "__init__.py":
|
||||
return _INIT_VERSION_PATTERN
|
||||
return None
|
||||
|
||||
|
||||
def _calculate_new_version(major: int, minor: int, patch: int, part: BumpVersionType) -> str:
|
||||
"""计算新版本号.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
major : int
|
||||
当前主版本号
|
||||
minor : int
|
||||
当前次版本号
|
||||
patch : int
|
||||
当前补丁版本号
|
||||
part : BumpVersionType
|
||||
要更新的部分
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
新版本号
|
||||
"""
|
||||
if part == "major":
|
||||
return f"{major + 1}.0.0"
|
||||
if part == "minor":
|
||||
return f"{major}.{minor + 1}.0"
|
||||
return f"{major}.{minor}.{patch + 1}"
|
||||
|
||||
|
||||
def _build_replacement_string(original_match: str, new_version: str, file_name: str) -> str:
|
||||
"""构建替换字符串,保留原始格式.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
original_match : str
|
||||
原始匹配的字符串
|
||||
new_version : str
|
||||
新版本号
|
||||
file_name : str
|
||||
文件名
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
替换字符串
|
||||
"""
|
||||
quote_char = '"' if '"' in original_match else "'"
|
||||
|
||||
if file_name == "pyproject.toml":
|
||||
prefix_match = re.match(r'(\s*version\s*=\s*)["\']', original_match)
|
||||
prefix = prefix_match.group(1) if prefix_match else "version = "
|
||||
return f"{prefix}{quote_char}{new_version}{quote_char}"
|
||||
|
||||
if file_name == "__init__.py":
|
||||
prefix_match = re.match(r'(\s*__version__\s*=\s*)["\']', original_match)
|
||||
prefix = prefix_match.group(1) if prefix_match else "__version__ = "
|
||||
return f"{prefix}{quote_char}{new_version}{quote_char}"
|
||||
|
||||
return new_version
|
||||
|
||||
|
||||
def bump_file_version(file_path: Path, part: BumpVersionType = "patch") -> str | None:
|
||||
"""更新文件中的版本号.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_path : Path
|
||||
要更新的文件路径
|
||||
part : BumpVersionType
|
||||
版本部分: patch, minor, major
|
||||
|
||||
Returns
|
||||
-------
|
||||
str | None
|
||||
更新后的新版本号,如果文件中未找到版本号则返回 None
|
||||
"""
|
||||
try:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
except Exception as e:
|
||||
print(f"读取文件 {file_path} 时出错: {e}")
|
||||
raise
|
||||
|
||||
# 获取文件对应的正则表达式
|
||||
pattern = _get_pattern_for_file(file_path.name)
|
||||
|
||||
# 对于未知文件类型,尝试两种模式
|
||||
if pattern:
|
||||
match = pattern.search(content)
|
||||
else:
|
||||
match = _PYPROJECT_VERSION_PATTERN.search(content) or _INIT_VERSION_PATTERN.search(content)
|
||||
|
||||
if not match:
|
||||
print(f"文件 {file_path} 中未找到版本号模式")
|
||||
return None
|
||||
|
||||
# 提取当前版本号
|
||||
major = int(match.group("major"))
|
||||
minor = int(match.group("minor"))
|
||||
patch = int(match.group("patch"))
|
||||
|
||||
# 计算新版本号
|
||||
new_version = _calculate_new_version(major, minor, patch, part)
|
||||
|
||||
# 构建替换字符串
|
||||
original_match = match.group(0)
|
||||
replacement = _build_replacement_string(original_match, new_version, file_path.name)
|
||||
|
||||
# 更新文件内容
|
||||
content = content.replace(original_match, replacement)
|
||||
|
||||
try:
|
||||
file_path.write_text(content, encoding="utf-8")
|
||||
except Exception as e:
|
||||
print(f"更新文件 {file_path} 版本号时出错: {e}")
|
||||
raise
|
||||
|
||||
return new_version
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""版本号管理工具主函数."""
|
||||
parser = argparse.ArgumentParser(description="BumpVersion - 版本号自动管理工具")
|
||||
parser.add_argument(
|
||||
"part",
|
||||
type=str,
|
||||
nargs="?",
|
||||
default="patch",
|
||||
choices=get_args(BumpVersionType),
|
||||
help=f"版本部分: {get_args(BumpVersionType)}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-tag",
|
||||
action="store_true",
|
||||
help="提交后不创建 git tag",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
part = args.part
|
||||
|
||||
# 搜索文件,排除常见的虚拟环境和缓存目录
|
||||
ignore_dirs = {".venv", "venv", ".git", "__pycache__", ".tox", "node_modules", "build", "dist", ".eggs"}
|
||||
all_files = set()
|
||||
|
||||
for pattern in ["__init__.py", "pyproject.toml"]:
|
||||
for file in Path.cwd().rglob(pattern):
|
||||
# 检查路径中是否包含需要忽略的目录
|
||||
if not any(ignore_dir in file.parts for ignore_dir in ignore_dirs):
|
||||
all_files.add(file)
|
||||
|
||||
if not all_files:
|
||||
print("未找到包含版本号的文件")
|
||||
return
|
||||
|
||||
print(f"找到 {len(all_files)} 个文件需要更新版本号")
|
||||
for file in sorted(all_files):
|
||||
print(f" - {file.relative_to(Path.cwd())}")
|
||||
|
||||
# 更新所有文件的版本号(使用顺序执行避免竞争条件)
|
||||
# 使用相对于 cwd 的路径作为任务名,确保唯一性
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec(
|
||||
f"bump_{file.relative_to(Path.cwd())}".replace("\\", "_").replace("/", "_").replace(".", "_"),
|
||||
fn=bump_file_version,
|
||||
args=(file, part),
|
||||
)
|
||||
for file in all_files
|
||||
])
|
||||
report = px.run(graph, strategy="sequential")
|
||||
|
||||
# 收集新版本号(取第一个成功的结果)
|
||||
new_version = None
|
||||
for task_name in report:
|
||||
result = report[task_name]
|
||||
if result is not None:
|
||||
new_version = result
|
||||
break
|
||||
|
||||
if not new_version:
|
||||
print("未能获取新版本号")
|
||||
return
|
||||
|
||||
print(f"版本号已更新为: {new_version}")
|
||||
|
||||
# 提交修改并创建标签
|
||||
tasks = [
|
||||
px.TaskSpec("git_add", cmd=["git", "add", "."]),
|
||||
px.TaskSpec(
|
||||
"git_commit",
|
||||
cmd=["git", "commit", "-m", f"bump version to {new_version}"],
|
||||
depends_on=("git_add",),
|
||||
),
|
||||
]
|
||||
|
||||
if not args.no_tag:
|
||||
tag_name = f"v{new_version}"
|
||||
tasks.append(
|
||||
px.TaskSpec(
|
||||
"git_tag",
|
||||
cmd=["git", "tag", "-a", tag_name, "-m", f"Release {tag_name}"],
|
||||
depends_on=("git_commit",),
|
||||
)
|
||||
)
|
||||
|
||||
graph = px.Graph.from_specs(tasks)
|
||||
px.run(graph, strategy="sequential")
|
||||
|
||||
if not args.no_tag:
|
||||
print(f"已创建标签: v{new_version}")
|
||||
@@ -1,15 +0,0 @@
|
||||
"""清屏工具.
|
||||
|
||||
跨平台清屏工具, 支持终端和控制台清屏.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.tasks.system import clr
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""清屏工具主函数."""
|
||||
graph = px.Graph.from_specs([clr()])
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -240,7 +240,7 @@ def _parse_email_date(date_str: str) -> str:
|
||||
try:
|
||||
dt = parsedate_to_datetime(date_str)
|
||||
return dt.isoformat()
|
||||
except Exception:
|
||||
except (ValueError, TypeError, OverflowError):
|
||||
return date_str
|
||||
|
||||
|
||||
@@ -277,11 +277,11 @@ def _extract_email_body_part(part: Any) -> str:
|
||||
decoded_text = payload.decode(charset, errors="replace")
|
||||
except (UnicodeDecodeError, LookupError) as decode_error:
|
||||
# 如果指定编码失败,尝试常见编码
|
||||
logger.warning(f"字符编码 {charset} 解码失败: {decode_error}")
|
||||
logger.warning("字符编码 %s 解码失败: %s", charset, decode_error)
|
||||
for fallback_charset in ["utf-8", "gbk", "gb2312", "latin-1"]:
|
||||
try:
|
||||
decoded_text = payload.decode(fallback_charset, errors="replace")
|
||||
logger.info(f"成功使用备用编码 {fallback_charset} 解码")
|
||||
logger.info("成功使用备用编码 %s 解码", fallback_charset)
|
||||
break
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
continue
|
||||
@@ -293,15 +293,15 @@ def _extract_email_body_part(part: Any) -> str:
|
||||
# 限制长度并返回
|
||||
result = decoded_text[:MAX_BODY_LENGTH]
|
||||
if len(decoded_text) > MAX_BODY_LENGTH:
|
||||
logger.debug(f"正文内容过长,截取前{MAX_BODY_LENGTH}字符")
|
||||
logger.debug("正文内容过长,截取前%d字符", MAX_BODY_LENGTH)
|
||||
|
||||
return result
|
||||
|
||||
except AttributeError as attr_error:
|
||||
logger.error(f"邮件部分对象属性错误: {attr_error}")
|
||||
logger.error("邮件部分对象属性错误: %s", attr_error)
|
||||
return ""
|
||||
except Exception as unexpected_error:
|
||||
logger.error(f"提取邮件正文时发生未知错误: {unexpected_error}")
|
||||
logger.error("提取邮件正文时发生未知错误: %s", unexpected_error)
|
||||
return ""
|
||||
|
||||
|
||||
@@ -567,13 +567,15 @@ class EmlManagerHandler(BaseHTTPRequestHandler):
|
||||
|
||||
emails = self.db.search_emails(keyword, field, limit, offset)
|
||||
total_count = self.db.get_email_count()
|
||||
self._send_json_response({
|
||||
"emails": emails,
|
||||
"count": len(emails),
|
||||
"total": total_count,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
})
|
||||
self._send_json_response(
|
||||
{
|
||||
"emails": emails,
|
||||
"count": len(emails),
|
||||
"total": total_count,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
}
|
||||
)
|
||||
|
||||
def _api_get_email(self, query_params: dict[str, list[str]]) -> None:
|
||||
"""API: 获取单个邮件详情."""
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
from typing import TypedDict
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
|
||||
class EnvConfig(TypedDict):
|
||||
"""环境配置项."""
|
||||
|
||||
name: str
|
||||
value: str
|
||||
description: str
|
||||
|
||||
|
||||
PIP_INDEX_URL_CONFIG: EnvConfig = {
|
||||
"name": "PIP_INDEX_URL",
|
||||
"value": "https://pypi.tuna.tsinghua.edu.cn/simple",
|
||||
"description": "PIP索引URL",
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
PIP_INDEX_URLS: dict[str, str] = {
|
||||
"tsinghua": "https://pypi.tuna.tsinghua.edu.cn/simple",
|
||||
"aliyun": "https://mirrors.aliyun.com/pypi/simple/",
|
||||
}
|
||||
|
||||
PIP_TRUSTED_HOSTS: dict[str, str] = {
|
||||
"tsinghua": "pypi.tuna.tsinghua.edu.cn",
|
||||
"aliyun": "mirrors.aliyun.com",
|
||||
}
|
||||
|
||||
UV_INDEX_URL: str = "https://mirrors.aliyun.com/pypi/simple/"
|
||||
UV_PYTHON_INSTALL_MIRROR: str = "https://registry.npmmirror.com/-/binary/python-build-standalone"
|
||||
|
||||
CONDA_MIRROR_URLS: dict[str, list[str]] = {
|
||||
"tsinghua": [
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/",
|
||||
],
|
||||
"aliyun": [
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/main/",
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/free/",
|
||||
"https://mirrors.aliyun.com/anaconda/cloud/conda-forge/",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""主函数."""
|
||||
# 使用更安全的分步执行方式,便于调试和捕获错误
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("download", cmd="curl -sSL https://linuxmirrors.cn/main.sh -o /tmp/linuxmirrors.sh", verbose=True),
|
||||
px.TaskSpec("install", cmd="sudo bash /tmp/linuxmirrors.sh", verbose=True, depends_on=("download",)),
|
||||
])
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,122 +0,0 @@
|
||||
"""Python 环境配置工具.
|
||||
|
||||
用于设置 pip 镜像源, 支持清华和阿里云等国内镜像源,
|
||||
同时配置 UV 和 Conda 的镜像源.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
PIP_INDEX_URLS: dict[str, str] = {
|
||||
"tsinghua": "https://pypi.tuna.tsinghua.edu.cn/simple",
|
||||
"aliyun": "https://mirrors.aliyun.com/pypi/simple/",
|
||||
}
|
||||
|
||||
PIP_TRUSTED_HOSTS: dict[str, str] = {
|
||||
"tsinghua": "pypi.tuna.tsinghua.edu.cn",
|
||||
"aliyun": "mirrors.aliyun.com",
|
||||
}
|
||||
|
||||
UV_INDEX_URL: str = "https://mirrors.aliyun.com/pypi/simple/"
|
||||
UV_PYTHON_INSTALL_MIRROR: str = "https://registry.npmmirror.com/-/binary/python-build-standalone"
|
||||
|
||||
CONDA_MIRROR_URLS: dict[str, list[str]] = {
|
||||
"tsinghua": [
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/",
|
||||
],
|
||||
"aliyun": [
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/main/",
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/free/",
|
||||
"https://mirrors.aliyun.com/anaconda/cloud/conda-forge/",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def set_pip_mirror(mirror: str = "tsinghua", token: str | None = None) -> None:
|
||||
"""设置 pip 镜像源.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mirror : str
|
||||
镜像源名称: tsinghua, aliyun
|
||||
token : str | None
|
||||
PyPI token for publishing
|
||||
"""
|
||||
index_url = PIP_INDEX_URLS.get(mirror, PIP_INDEX_URLS["tsinghua"])
|
||||
trusted_host = PIP_TRUSTED_HOSTS.get(mirror, "")
|
||||
|
||||
# 设置环境变量
|
||||
os.environ["PIP_INDEX_URL"] = index_url
|
||||
os.environ["UV_INDEX_URL"] = UV_INDEX_URL
|
||||
os.environ["UV_DEFAULT_INDEX"] = UV_INDEX_URL
|
||||
os.environ["UV_PYTHON_INSTALL_MIRROR"] = UV_PYTHON_INSTALL_MIRROR
|
||||
|
||||
# 写入 pip 配置文件
|
||||
pip_dir = Path.home() / "pip"
|
||||
pip_dir.mkdir(exist_ok=True)
|
||||
pip_conf = pip_dir / ("pip.ini" if Constants.IS_WINDOWS else "pip.conf")
|
||||
pip_conf.write_text(f"[global]\nindex-url = {index_url}\n[install]\ntrusted-host = {trusted_host}\n")
|
||||
|
||||
# 写入 conda 配置文件
|
||||
condarc = Path.home() / ".condarc"
|
||||
conda_urls = CONDA_MIRROR_URLS.get(mirror, CONDA_MIRROR_URLS["tsinghua"])
|
||||
condarc.write_text(
|
||||
"show_channel_urls: true\nchannels:\n" + "\n".join(f" - {url}" for url in conda_urls) + "\n - defaults\n"
|
||||
)
|
||||
|
||||
# 写入 pypirc 配置文件 (如果有 token)
|
||||
if token:
|
||||
pypirc = Path.home() / ".pypirc"
|
||||
pypirc.write_text(
|
||||
f"[pypi]\nrepository: https://upload.pypi.org/legacy/\nusername: __token__\npassword: {token}\n"
|
||||
)
|
||||
|
||||
print(f"已设置 pip 镜像源: {mirror} ({index_url})")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Python 环境配置工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="EnvPy - Python 环境配置工具",
|
||||
usage="envpy <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 设置镜像源命令
|
||||
mirror_parser = subparsers.add_parser("mirror", help="设置 pip 镜像源")
|
||||
mirror_parser.add_argument("name", choices=["tsinghua", "aliyun"], help="镜像源名称")
|
||||
mirror_parser.add_argument("--token", type=str, help="PyPI token for publishing")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "mirror":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("set_pip_mirror", fn=set_pip_mirror, args=(args.name,), kwargs={"token": args.token})
|
||||
])
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,57 +0,0 @@
|
||||
"""PyQt 环境配置工具.
|
||||
|
||||
用于设置 PyQt 相关环境变量, 安装依赖环境.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
QT_LIBS: list[str] = [
|
||||
"build-essential",
|
||||
"libgl1",
|
||||
"libegl1",
|
||||
"libglib2.0-0",
|
||||
"libfontconfig1",
|
||||
"libfreetype6",
|
||||
"libxkbcommon0",
|
||||
"libdbus-1-3",
|
||||
"libxcb-xinerama0",
|
||||
"libxcb-icccm4",
|
||||
"libxcb-image0",
|
||||
"libxcb-keysyms1",
|
||||
"libxcb-randr0",
|
||||
"libxcb-render-util0",
|
||||
"libxcb-shape0",
|
||||
"libxcb-xfixes0",
|
||||
"libxcb-cursor0",
|
||||
]
|
||||
|
||||
CHINESE_FONTS: list[str] = [
|
||||
"fonts-noto-cjk",
|
||||
"fonts-wqy-microhei",
|
||||
"fonts-wqy-zenhei",
|
||||
"fonts-noto-color-emoji",
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""PyQt 环境配置工具主函数."""
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"envqt_install",
|
||||
cmd=["sudo", "apt", "install", "-y", *QT_LIBS],
|
||||
conditions=(lambda _: Constants.IS_LINUX,),
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
"envqt_fonts",
|
||||
cmd=["sudo", "apt", "install", "-y", *CHINESE_FONTS],
|
||||
conditions=(lambda _: Constants.IS_LINUX,),
|
||||
verbose=True,
|
||||
),
|
||||
],
|
||||
)
|
||||
px.run(graph, strategy="thread", verbose=True)
|
||||
@@ -1,150 +0,0 @@
|
||||
"""Rust 环境配置工具.
|
||||
|
||||
配置 Rustup 和 Cargo 的国内镜像源,
|
||||
加速 Rust 工具链和依赖包的下载.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
RUSTUP_MIRRORS: dict[str, dict[str, str]] = {
|
||||
"aliyun": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.aliyun.com/rustup",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.aliyun.com/rustup/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.aliyun.com/crates.io-index/",
|
||||
},
|
||||
"ustc": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.ustc.edu.cn/rust-static",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.ustc.edu.cn/rust-static/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.ustc.edu.cn/crates.io-index/",
|
||||
},
|
||||
"tsinghua": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.tuna.tsinghua.edu.cn/rustup",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.tuna.tsinghua.edu.cn/rustup/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.tuna.tsinghua.edu.cn/crates.io-index/",
|
||||
},
|
||||
}
|
||||
|
||||
UsableRustVersion = Literal["stable", "nightly", "beta"]
|
||||
UsableMirror = Literal["aliyun", "ustc", "tsinghua"]
|
||||
|
||||
DEFAULT_RUST_VERSION: UsableRustVersion = "stable"
|
||||
DEFAULT_MIRROR: UsableMirror = "tsinghua"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def set_rust_mirror(mirror: UsableMirror = DEFAULT_MIRROR) -> None:
|
||||
"""设置 Rust 镜像源.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mirror : str
|
||||
镜像源名称: aliyun, ustc, tsinghua
|
||||
"""
|
||||
mirror_dict = RUSTUP_MIRRORS.get(mirror, RUSTUP_MIRRORS[DEFAULT_MIRROR])
|
||||
server = mirror_dict["RUSTUP_DIST_SERVER"]
|
||||
update_root = mirror_dict["RUSTUP_UPDATE_ROOT"]
|
||||
toml_registry = mirror_dict["TOML_REGISTRY"]
|
||||
|
||||
# 设置环境变量
|
||||
os.environ["RUSTUP_DIST_SERVER"] = server
|
||||
os.environ["RUSTUP_UPDATE_ROOT"] = update_root
|
||||
|
||||
# 写入 cargo 配置
|
||||
cargo_dir = Path.home() / ".cargo"
|
||||
cargo_dir.mkdir(exist_ok=True)
|
||||
cargo_config = cargo_dir / "config.toml"
|
||||
cargo_config.write_text(
|
||||
f"""[source.crates-io]
|
||||
replace-with = '{mirror}'
|
||||
|
||||
[source.{mirror}]
|
||||
registry = "sparse+{toml_registry}"
|
||||
|
||||
[registries.{mirror}]
|
||||
index = "sparse+{toml_registry}"
|
||||
"""
|
||||
)
|
||||
|
||||
print(f"已设置 Rust 镜像源: {mirror}")
|
||||
|
||||
|
||||
def install_rust(version: UsableRustVersion = DEFAULT_RUST_VERSION) -> None:
|
||||
"""安装 Rust 工具链.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
version : str
|
||||
Rust 版本: stable, nightly, beta
|
||||
"""
|
||||
try:
|
||||
subprocess.run(["rustup", "toolchain", "install", version], check=True)
|
||||
print(f"已安装 Rust {version}")
|
||||
except FileNotFoundError:
|
||||
print("未找到 rustup,请先安装 Rust: https://rustup.rs")
|
||||
raise
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Rust 环境配置工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="EnvRs - Rust 环境配置工具",
|
||||
usage="envrs <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 设置镜像源命令
|
||||
mirror_parser = subparsers.add_parser("mirror", help="设置 Rust 镜像源")
|
||||
mirror_parser.add_argument(
|
||||
"name",
|
||||
nargs="?",
|
||||
default=DEFAULT_MIRROR,
|
||||
choices=get_args(UsableMirror),
|
||||
help=f"镜像源名称 ({get_args(UsableMirror)})",
|
||||
)
|
||||
|
||||
# 安装 Rust 命令
|
||||
install_parser = subparsers.add_parser("install", help="安装 Rust 工具链")
|
||||
install_parser.add_argument(
|
||||
"version",
|
||||
nargs="?",
|
||||
default=DEFAULT_RUST_VERSION,
|
||||
choices=get_args(UsableRustVersion),
|
||||
help=f"Rust 版本 ({get_args(UsableRustVersion)})",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "mirror":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("set_rust_mirror", fn=set_rust_mirror, args=(args.name,), verbose=True)
|
||||
])
|
||||
elif args.command == "install":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("install_rust", cmd=["rustup", "toolchain", "install", args.version], verbose=True)
|
||||
])
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread", verbose=True)
|
||||
@@ -1,102 +0,0 @@
|
||||
"""Git 工具模块.
|
||||
|
||||
提供 Git 仓库管理的常用操作封装,
|
||||
支持初始化、提交、清理、推送等功能.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
EXCLUDE_DIRS = [
|
||||
# 编辑器相关目录
|
||||
".vscode",
|
||||
".idea",
|
||||
".editorconfig",
|
||||
".trae",
|
||||
".qoder",
|
||||
# 项目相关目录
|
||||
".venv",
|
||||
".git",
|
||||
".tox",
|
||||
".pytest_cache",
|
||||
"node_modules",
|
||||
".ruff_cache",
|
||||
]
|
||||
EXCLUDE_CMDS = [arg for d in EXCLUDE_DIRS for arg in ["-e", d]]
|
||||
|
||||
|
||||
def init_sub_dirs() -> None:
|
||||
"""初始化子目录的Git仓库."""
|
||||
sub_dirs = [subdir for subdir in Path.cwd().iterdir() if subdir.is_dir()]
|
||||
for subdir in sub_dirs:
|
||||
px.run(
|
||||
px.Graph.from_specs([
|
||||
px.TaskSpec(
|
||||
"init",
|
||||
cmd=["git", "init"],
|
||||
conditions=(lambda _: not_has_git_repo(),),
|
||||
cwd=subdir,
|
||||
),
|
||||
px.TaskSpec("add", cmd=["git", "add", "."], depends_on=("init",)),
|
||||
px.TaskSpec("commit", cmd=["git", "commit", "-m", "init commit"], depends_on=("add",)),
|
||||
]),
|
||||
)
|
||||
|
||||
|
||||
isub: px.TaskSpec = px.TaskSpec("isub", fn=init_sub_dirs)
|
||||
push: px.TaskSpec = px.TaskSpec("push", cmd=["git", "push"])
|
||||
pull: px.TaskSpec = px.TaskSpec("pull", cmd=["git", "pull"])
|
||||
kill_tgit: px.TaskSpec = px.TaskSpec("task_kill", cmd=["taskkill", "/f", "/t", "/im", "tgitcache.exe"])
|
||||
|
||||
|
||||
def not_has_git_repo() -> bool:
|
||||
"""检查当前目录没有Git仓库."""
|
||||
return not Path.cwd().exists() or not (Path.cwd() / ".git").is_dir()
|
||||
|
||||
|
||||
def has_files() -> bool:
|
||||
"""检查当前目录是否有文件."""
|
||||
return bool(list(Path.cwd().glob("*")))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Git工具主函数."""
|
||||
runner = px.CliRunner(
|
||||
strategy="thread",
|
||||
description="Gittool - Git 执行工具.",
|
||||
graphs={
|
||||
# 添加并提交
|
||||
"a": px.Graph.from_specs([
|
||||
px.TaskSpec("add", cmd=["git", "add", "."], conditions=(lambda _: has_files(),)),
|
||||
px.TaskSpec("commit", cmd=["git", "commit", "-m", "chore: update"], depends_on=("add",)),
|
||||
]),
|
||||
# 清理
|
||||
"c": px.Graph.from_specs([
|
||||
px.TaskSpec("clean", cmd=["git", "clean", "-xfd", *EXCLUDE_CMDS]),
|
||||
px.TaskSpec("status", cmd=["git", "status", "--porcelain"], depends_on=("clean",)),
|
||||
]),
|
||||
# 初始化、添加并提交
|
||||
"i": px.Graph.from_specs([
|
||||
px.TaskSpec("init", cmd=["git", "init"], conditions=(lambda _: not_has_git_repo(),)),
|
||||
px.TaskSpec("add", cmd=["git", "add", "."], depends_on=("init",), conditions=(lambda _: has_files(),)),
|
||||
px.TaskSpec(
|
||||
"commit",
|
||||
cmd=["git", "commit", "-m", "init commit"],
|
||||
depends_on=("add",),
|
||||
conditions=(lambda _: has_files(),),
|
||||
),
|
||||
]),
|
||||
# 初始化子目录
|
||||
"isub": px.Graph.from_specs([isub]),
|
||||
# 推送
|
||||
"p": px.Graph.from_specs([push]),
|
||||
# 拉取
|
||||
"pl": px.Graph.from_specs([pull]),
|
||||
# 重启TGit缓存
|
||||
"r": px.Graph.from_specs([kill_tgit]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
@@ -1,41 +0,0 @@
|
||||
"""Download from ModelScopeHub."""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
DownloadType = Literal["model", "dataset", "space"]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Download a model from ModelScopeHub.")
|
||||
parser.add_argument("name", help="Target name.")
|
||||
parser.add_argument("--type", "-t", nargs="?", default="model", choices=get_args(DownloadType), help="Target type.")
|
||||
parser.add_argument("--dir", default=None, help="Download directory.")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.name:
|
||||
parser.error("name is required")
|
||||
|
||||
download_dir: Path = Path(args.dir) if args.dir else Path.home() / ".models" / args.name.split("/")[-1]
|
||||
download_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec(
|
||||
name="download",
|
||||
cmd=[
|
||||
"uvx",
|
||||
"modelscope",
|
||||
"download",
|
||||
f"--{args.type}",
|
||||
args.name,
|
||||
"--local_dir",
|
||||
str(download_dir),
|
||||
],
|
||||
verbose=True,
|
||||
),
|
||||
])
|
||||
|
||||
px.run(graph, strategy="thread", verbose=True)
|
||||
@@ -1,63 +0,0 @@
|
||||
"""使用 SGLang 运行本地模型."""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import BuiltinConditions, Constants
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="启动 SGLang 服务")
|
||||
parser.add_argument("--model", default="~/.models/Qwen2.5-Coder-32B-Instruct-AWQ", help="模型路径")
|
||||
parser.add_argument("--port", type=int, default=8000, help="服务端口")
|
||||
parser.add_argument("--ctx-len", type=int, default=28672, help="最大上下文长度")
|
||||
parser.add_argument("--mem", type=float, default=0.75, help="显存占比 (0-1)")
|
||||
parser.add_argument("--host", default="0.0.0.0", help="主机地址")
|
||||
parser.add_argument("--log-level", default="info", help="日志级别")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.model:
|
||||
parser.error("model is required")
|
||||
|
||||
model_dir = Path(args.model).expanduser()
|
||||
if not model_dir.exists():
|
||||
parser.error(f"Model directory {model_dir} does not exist.")
|
||||
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec(
|
||||
name="download",
|
||||
cmd=[
|
||||
"uv",
|
||||
"install",
|
||||
"sglang[all]",
|
||||
],
|
||||
conditions=(BuiltinConditions.NOT(BuiltinConditions.HAS_INSTALLED("sglang")),),
|
||||
verbose=True,
|
||||
),
|
||||
px.TaskSpec(
|
||||
name="run",
|
||||
cmd=[
|
||||
"python" if Constants.IS_WINDOWS else "python3",
|
||||
"-m",
|
||||
"sglang.launch_server",
|
||||
"--model-path",
|
||||
str(model_dir),
|
||||
"--host",
|
||||
str(args.host),
|
||||
"--port",
|
||||
"8000",
|
||||
"--mem-fraction-static",
|
||||
str(args.mem),
|
||||
"--context-length",
|
||||
"32768",
|
||||
"--tool-call-parser",
|
||||
"qwen",
|
||||
"--log-level",
|
||||
str(args.log_level),
|
||||
],
|
||||
verbose=True,
|
||||
),
|
||||
])
|
||||
|
||||
px.run(graph, strategy="sequential", verbose=True)
|
||||
@@ -1,523 +0,0 @@
|
||||
"""PDF 工具模块.
|
||||
|
||||
提供 PDF 文件操作的常用功能封装,
|
||||
支持合并、拆分、压缩、加密、水印、OCR等功能.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
try:
|
||||
import fitz # PyMuPDF
|
||||
|
||||
HAS_PYMUPDF = True
|
||||
except ImportError:
|
||||
HAS_PYMUPDF = False
|
||||
|
||||
try:
|
||||
import pypdf
|
||||
|
||||
HAS_PYPDF = True
|
||||
except ImportError:
|
||||
HAS_PYPDF = False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
PDF_SUFFIX = ".pdf"
|
||||
DEFAULT_QUALITY = 75
|
||||
DEFAULT_PASSWORD = ""
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def pdf_merge(input_paths: list[Path], output_path: Path) -> None:
|
||||
"""合并多个 PDF 文件."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
writer = pypdf.PdfWriter()
|
||||
for input_path in input_paths:
|
||||
if input_path.exists():
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"合并完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_split(input_path: Path, output_dir: Path) -> None:
|
||||
"""拆分 PDF 文件为单页."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for i, page in enumerate(reader.pages):
|
||||
writer = pypdf.PdfWriter()
|
||||
writer.add_page(page)
|
||||
output_file = output_dir / f"{input_path.stem}_page_{i + 1}.pdf"
|
||||
with open(output_file, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"拆分完成: {output_dir}")
|
||||
|
||||
|
||||
def pdf_compress(input_path: Path, output_path: Path) -> None:
|
||||
"""压缩 PDF 文件."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path), garbage=4, deflate=True, clean=True)
|
||||
doc.close()
|
||||
|
||||
original_size = input_path.stat().st_size
|
||||
new_size = output_path.stat().st_size
|
||||
ratio = (1 - new_size / original_size) * 100
|
||||
print(f"压缩完成: {output_path} (缩小 {ratio:.1f}%)")
|
||||
|
||||
|
||||
def pdf_encrypt(input_path: Path, output_path: Path, password: str) -> None:
|
||||
"""加密 PDF 文件."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
writer = pypdf.PdfWriter()
|
||||
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
writer.encrypt(user_password=password, owner_password=password, use_128bit=True)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"加密完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_decrypt(input_path: Path, output_path: Path, password: str) -> None:
|
||||
"""解密 PDF 文件."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
if reader.is_encrypted:
|
||||
reader.decrypt(password)
|
||||
|
||||
writer = pypdf.PdfWriter()
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"解密完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_extract_text(input_path: Path, output_path: Path) -> None:
|
||||
"""提取 PDF 文本."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
text = ""
|
||||
for page in doc:
|
||||
text += str(page.get_text()) + "\n\n"
|
||||
doc.close()
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(text, encoding="utf-8")
|
||||
print(f"文本提取完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_extract_images(input_path: Path, output_dir: Path) -> None:
|
||||
"""提取 PDF 图片."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
image_count = 0
|
||||
# pyrefly: ignore [bad-argument-type]
|
||||
for page_num, page in enumerate(doc):
|
||||
images = page.get_images(full=True)
|
||||
for img_idx, img in enumerate(images):
|
||||
xref = img[0]
|
||||
base_image = doc.extract_image(xref)
|
||||
image_data = base_image["image"]
|
||||
image_ext = base_image["ext"]
|
||||
image_path = output_dir / f"page_{page_num + 1}_img_{img_idx + 1}.{image_ext}"
|
||||
image_path.write_bytes(image_data)
|
||||
image_count += 1
|
||||
|
||||
doc.close()
|
||||
print(f"图片提取完成: {output_dir} (共 {image_count} 张)")
|
||||
|
||||
|
||||
def pdf_add_watermark(input_path: Path, output_path: Path, text: str = "CONFIDENTIAL") -> None:
|
||||
"""添加 PDF 水印."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
for page in doc:
|
||||
rect = page.rect
|
||||
text_width = fitz.get_text_length(text, fontsize=48)
|
||||
x = (rect.width - text_width) / 2
|
||||
y = rect.height / 2
|
||||
page.insert_text((x, y), text, fontsize=48, rotate=45, color=(0, 0, 0))
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"水印添加完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_rotate(input_path: Path, output_path: Path, rotation: int = 90) -> None:
|
||||
"""旋转 PDF 页面."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
for page in doc:
|
||||
page.set_rotation(rotation)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"旋转完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_crop(input_path: Path, output_path: Path, margins: tuple[int, int, int, int]) -> None:
|
||||
"""裁剪 PDF 页面."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
left, top, right, bottom = margins
|
||||
|
||||
for page in doc:
|
||||
rect = page.rect
|
||||
new_rect = fitz.Rect(
|
||||
rect.x0 + left,
|
||||
rect.y0 + top,
|
||||
rect.x1 - right,
|
||||
rect.y1 - bottom,
|
||||
)
|
||||
page.set_cropbox(new_rect)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"裁剪完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_info(input_path: Path) -> None:
|
||||
"""显示 PDF 信息."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
print(f"文件: {input_path}")
|
||||
print(f"页数: {doc.page_count}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"标题: {doc.metadata.get('title', 'N/A')}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"作者: {doc.metadata.get('author', 'N/A')}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"创建日期: {doc.metadata.get('creationDate', 'N/A')}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"修改日期: {doc.metadata.get('modDate', 'N/A')}")
|
||||
print(f"文件大小: {input_path.stat().st_size / 1024:.1f} KB")
|
||||
doc.close()
|
||||
|
||||
|
||||
def pdf_ocr(input_path: Path, output_path: Path, lang: str = "chi_sim+eng") -> None:
|
||||
"""PDF OCR 识别."""
|
||||
try:
|
||||
import pytesseract
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
print("未安装 OCR 相关库,请安装: pip install pytesseract pillow")
|
||||
return
|
||||
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
new_doc = fitz.open()
|
||||
|
||||
for page in doc:
|
||||
pix = page.get_pixmap()
|
||||
img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
|
||||
ocr_text = pytesseract.image_to_string(img, lang=lang)
|
||||
|
||||
new_page = new_doc.new_page(width=page.rect.width, height=page.rect.height)
|
||||
new_page.insert_image(new_page.rect, pixmap=pix)
|
||||
text_rect = fitz.Rect(0, 0, page.rect.width, page.rect.height)
|
||||
# pyrefly: ignore [bad-argument-type]
|
||||
new_page.insert_textbox(text_rect, ocr_text)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
new_doc.save(str(output_path))
|
||||
new_doc.close()
|
||||
doc.close()
|
||||
print(f"OCR 识别完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_reorder(input_path: Path, output_path: Path, order: list[int]) -> None:
|
||||
"""重排 PDF 页面顺序."""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库,请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
writer = pypdf.PdfWriter()
|
||||
|
||||
for page_num in order:
|
||||
if 0 <= page_num < len(reader.pages):
|
||||
writer.add_page(reader.pages[page_num])
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"重排完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_to_images(input_path: Path, output_dir: Path, dpi: int = 300) -> None:
|
||||
"""PDF 转图片."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# pyrefly: ignore [bad-argument-type]
|
||||
for page_num, page in enumerate(doc):
|
||||
pix = page.get_pixmap(dpi=dpi)
|
||||
image_path = output_dir / f"{input_path.stem}_page_{page_num + 1}.png"
|
||||
pix.save(str(image_path))
|
||||
|
||||
doc.close()
|
||||
print(f"转换完成: {output_dir}")
|
||||
|
||||
|
||||
def pdf_repair(input_path: Path, output_path: Path) -> None:
|
||||
"""修复 PDF 文件."""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库,请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path), garbage=4, deflate=True, clean=True)
|
||||
doc.close()
|
||||
print(f"修复完成: {output_path}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None: # noqa: PLR0912
|
||||
"""PDF 工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="PDFTool - PDF 文件工具集",
|
||||
usage="pdftool <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 合并 PDF 命令
|
||||
merge_parser = subparsers.add_parser("m", help="合并 PDF 文件")
|
||||
merge_parser.add_argument("inputs", nargs="+", help="输入 PDF 文件路径")
|
||||
merge_parser.add_argument("--output", type=str, default="merged.pdf", help="输出文件路径")
|
||||
|
||||
# 拆分 PDF 命令
|
||||
split_parser = subparsers.add_parser("s", help="拆分 PDF 文件为单页")
|
||||
split_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
split_parser.add_argument("--output-dir", type=str, default="split", help="输出目录")
|
||||
|
||||
# 压缩 PDF 命令
|
||||
compress_parser = subparsers.add_parser("c", help="压缩 PDF 文件")
|
||||
compress_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
compress_parser.add_argument("--output", type=str, default="compressed.pdf", help="输出文件路径")
|
||||
|
||||
# 加密 PDF 命令
|
||||
encrypt_parser = subparsers.add_parser("e", help="加密 PDF 文件")
|
||||
encrypt_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
encrypt_parser.add_argument("--output", type=str, default="encrypted.pdf", help="输出文件路径")
|
||||
encrypt_parser.add_argument("--password", type=str, required=True, help="密码")
|
||||
|
||||
# 解密 PDF 命令
|
||||
decrypt_parser = subparsers.add_parser("d", help="解密 PDF 文件")
|
||||
decrypt_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
decrypt_parser.add_argument("--output", type=str, default="decrypted.pdf", help="输出文件路径")
|
||||
decrypt_parser.add_argument("--password", type=str, required=True, help="密码")
|
||||
|
||||
# 提取文本命令
|
||||
extract_text_parser = subparsers.add_parser("xt", help="提取 PDF 文本")
|
||||
extract_text_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
extract_text_parser.add_argument("--output", type=str, default="output.txt", help="输出文件路径")
|
||||
|
||||
# 提取图片命令
|
||||
extract_images_parser = subparsers.add_parser("xi", help="提取 PDF 图片")
|
||||
extract_images_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
extract_images_parser.add_argument("--output-dir", type=str, default="images", help="输出目录")
|
||||
|
||||
# 添加水印命令
|
||||
watermark_parser = subparsers.add_parser("w", help="添加 PDF 水印")
|
||||
watermark_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
watermark_parser.add_argument("--output", type=str, default="watermarked.pdf", help="输出文件路径")
|
||||
watermark_parser.add_argument("--text", type=str, default="CONFIDENTIAL", help="水印文本")
|
||||
|
||||
# 旋转 PDF 命令
|
||||
rotate_parser = subparsers.add_parser("r", help="旋转 PDF 页面")
|
||||
rotate_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
rotate_parser.add_argument("--output", type=str, default="rotated.pdf", help="输出文件路径")
|
||||
rotate_parser.add_argument("--rotation", type=int, default=90, help="旋转角度 (90, 180, 270)")
|
||||
|
||||
# 裁剪 PDF 命令
|
||||
crop_parser = subparsers.add_parser("crop", help="裁剪 PDF 页面")
|
||||
crop_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
crop_parser.add_argument("--output", type=str, default="cropped.pdf", help="输出文件路径")
|
||||
crop_parser.add_argument("--left", type=int, default=10, help="左边裁剪")
|
||||
crop_parser.add_argument("--top", type=int, default=10, help="顶部裁剪")
|
||||
crop_parser.add_argument("--right", type=int, default=10, help="右边裁剪")
|
||||
crop_parser.add_argument("--bottom", type=int, default=10, help="底部裁剪")
|
||||
|
||||
# 显示信息命令
|
||||
info_parser = subparsers.add_parser("i", help="显示 PDF 信息")
|
||||
info_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
|
||||
# OCR 识别命令
|
||||
ocr_parser = subparsers.add_parser("ocr", help="PDF OCR 识别")
|
||||
ocr_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
ocr_parser.add_argument("--output", type=str, default="ocr.pdf", help="输出文件路径")
|
||||
ocr_parser.add_argument("--lang", type=str, default="chi_sim+eng", help="OCR 语言")
|
||||
|
||||
# 转换图片命令
|
||||
to_images_parser = subparsers.add_parser("img", help="PDF 转图片")
|
||||
to_images_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
to_images_parser.add_argument("--output-dir", type=str, default="images", help="输出目录")
|
||||
to_images_parser.add_argument("--dpi", type=int, default=300, help="图片 DPI")
|
||||
|
||||
# 修复 PDF 命令
|
||||
repair_parser = subparsers.add_parser("repair", help="修复 PDF 文件")
|
||||
repair_parser.add_argument("input", help="输入 PDF 文件路径")
|
||||
repair_parser.add_argument("--output", type=str, default="repaired.pdf", help="输出文件路径")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "m":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("pdf_merge", fn=pdf_merge, args=([Path(p) for p in args.inputs], Path(args.output)))
|
||||
])
|
||||
elif args.command == "s":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("pdf_split", fn=pdf_split, args=(Path(args.input), Path(args.output_dir)))
|
||||
])
|
||||
elif args.command == "c":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("pdf_compress", fn=pdf_compress, args=(Path(args.input), Path(args.output)))
|
||||
])
|
||||
elif args.command == "e":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("pdf_encrypt", fn=pdf_encrypt, args=(Path(args.input), Path(args.output), args.password))
|
||||
])
|
||||
elif args.command == "d":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("pdf_decrypt", fn=pdf_decrypt, args=(Path(args.input), Path(args.output), args.password))
|
||||
])
|
||||
elif args.command == "xt":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("pdf_extract_text", fn=pdf_extract_text, args=(Path(args.input), Path(args.output)))
|
||||
])
|
||||
elif args.command == "xi":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("pdf_extract_images", fn=pdf_extract_images, args=(Path(args.input), Path(args.output_dir)))
|
||||
])
|
||||
elif args.command == "w":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec(
|
||||
"pdf_watermark",
|
||||
fn=pdf_add_watermark,
|
||||
args=(Path(args.input), Path(args.output)),
|
||||
kwargs={"text": args.text},
|
||||
)
|
||||
])
|
||||
elif args.command == "r":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec(
|
||||
"pdf_rotate",
|
||||
fn=pdf_rotate,
|
||||
args=(Path(args.input), Path(args.output)),
|
||||
kwargs={"rotation": args.rotation},
|
||||
)
|
||||
])
|
||||
elif args.command == "crop":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec(
|
||||
"pdf_crop",
|
||||
fn=pdf_crop,
|
||||
args=(Path(args.input), Path(args.output)),
|
||||
kwargs={"margins": (args.left, args.top, args.right, args.bottom)},
|
||||
)
|
||||
])
|
||||
elif args.command == "i":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("pdf_info", fn=pdf_info, args=(Path(args.input),))])
|
||||
elif args.command == "ocr":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("pdf_ocr", fn=pdf_ocr, args=(Path(args.input), Path(args.output)), kwargs={"lang": args.lang})
|
||||
])
|
||||
elif args.command == "img":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec(
|
||||
"pdf_to_images",
|
||||
fn=pdf_to_images,
|
||||
args=(Path(args.input), Path(args.output_dir)),
|
||||
kwargs={"dpi": args.dpi},
|
||||
)
|
||||
])
|
||||
elif args.command == "repair":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("pdf_repair", fn=pdf_repair, args=(Path(args.input), Path(args.output)))
|
||||
])
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,212 @@
|
||||
"""PyFlowX 统一 CLI 入口.
|
||||
|
||||
通过 ``pf <tool> [command] [options]`` 调用所有工具,
|
||||
工具定义在 ``pyflowx.ops`` 子包中, 每个模块用 ``@px.tool`` 装饰器注册.
|
||||
|
||||
用法
|
||||
----
|
||||
pf # 列出所有可用工具
|
||||
pf filedate # 查看 filedate 工具帮助
|
||||
pf filedate add a.txt # 调用 filedate 的 add 子命令
|
||||
pf pymake b # 调用 pymake 的 b 别名
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import importlib
|
||||
import sys
|
||||
from typing import Sequence
|
||||
|
||||
from pyflowx.tools import _TOOL_REGISTRY, run_tool
|
||||
|
||||
|
||||
class PfApp:
|
||||
"""pf 统一入口应用.
|
||||
|
||||
路由 ``pf <tool> [command]`` 到 ``@px.tool`` 注册的工具或传统 Python 工具.
|
||||
"""
|
||||
|
||||
# 工具名到 ops 模块名的映射 (支持短别名)
|
||||
_TOOL_ALIASES: dict[str, str] = {
|
||||
"autofmt": "autofmt",
|
||||
"af": "autofmt",
|
||||
"bump": "bumpversion",
|
||||
"bumpversion": "bumpversion",
|
||||
"bv": "bumpversion",
|
||||
"clr": "clr",
|
||||
"clearscreen": "clr",
|
||||
"dockercmd": "dockercmd",
|
||||
"docker": "dockercmd",
|
||||
"envdev": "envdev",
|
||||
"env": "envdev",
|
||||
"filedate": "filedate",
|
||||
"fd": "filedate",
|
||||
"filelevel": "filelevel",
|
||||
"fl": "filelevel",
|
||||
"folderback": "folderback",
|
||||
"foldback": "folderback",
|
||||
"fb": "folderback",
|
||||
"folderzip": "folderzip",
|
||||
"foldzip": "folderzip",
|
||||
"fz": "folderzip",
|
||||
"git": "gittool",
|
||||
"gitt": "gittool",
|
||||
"gittool": "gittool",
|
||||
"gt": "gittool",
|
||||
"ls": "lscalc",
|
||||
"lscalc": "lscalc",
|
||||
"msdown": "msdownload",
|
||||
"msdownload": "msdownload",
|
||||
"msd": "msdownload",
|
||||
"pack": "packtool",
|
||||
"packtool": "packtool",
|
||||
"pk": "packtool",
|
||||
"pdf": "pdftool",
|
||||
"pdftool": "pdftool",
|
||||
"pt": "pdftool",
|
||||
"pip": "piptool",
|
||||
"pymake": "pymake",
|
||||
"piptool": "piptool",
|
||||
"pp": "piptool",
|
||||
"reseticon": "reseticoncache",
|
||||
"reseticoncache": "reseticoncache",
|
||||
"ric": "reseticoncache",
|
||||
"screenshot": "screenshot",
|
||||
"scrcap": "screenshot",
|
||||
"ss": "screenshot",
|
||||
"sglang": "sglang",
|
||||
"sg": "sglang",
|
||||
"ssh": "sshcopyid",
|
||||
"sshcopy": "sshcopyid",
|
||||
"sshcopyid": "sshcopyid",
|
||||
"sc": "sshcopyid",
|
||||
"taskk": "taskkill",
|
||||
"taskkill": "taskkill",
|
||||
"tk": "taskkill",
|
||||
"wch": "which",
|
||||
"which": "which",
|
||||
}
|
||||
|
||||
# 传统工具: 有自己的 main() 函数 (无法 @px.tool 化的复杂逻辑)
|
||||
_LEGACY_TOOLS: dict[str, str] = {
|
||||
"emlman": "pyflowx.cli.emlmanager:main",
|
||||
"profiler": "pyflowx.cli.profiler:main",
|
||||
"pxp": "pyflowx.cli.profiler:main",
|
||||
}
|
||||
|
||||
def __init__(self, argv: Sequence[str] | None = None) -> None:
|
||||
self._argv = list(argv) if argv is not None else sys.argv[1:]
|
||||
|
||||
def run(self) -> int:
|
||||
"""主入口, 返回退出码."""
|
||||
if not self._argv:
|
||||
self._list_tools()
|
||||
return 0
|
||||
|
||||
tool_name = self._argv[0]
|
||||
rest_argv = self._argv[1:]
|
||||
|
||||
resolved = self._resolve_tool(tool_name)
|
||||
if resolved is None:
|
||||
print(f"错误: 未知工具 '{tool_name}'", file=sys.stderr)
|
||||
print("运行 'pf' 查看可用工具列表", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
tool_type, target = resolved
|
||||
if tool_type == "legacy":
|
||||
return self._run_legacy(target, rest_argv)
|
||||
return self._run_tool(target, rest_argv)
|
||||
|
||||
def _list_tools(self) -> None:
|
||||
"""列出所有可用工具."""
|
||||
print("PyFlowX 工具列表:")
|
||||
print()
|
||||
print("@px.tool 工具:")
|
||||
yaml_tools = sorted(set(self._TOOL_ALIASES.values()))
|
||||
for tool in yaml_tools:
|
||||
description = self._tool_description(tool)
|
||||
if description:
|
||||
print(f" pf {tool:<15} - {description}")
|
||||
else:
|
||||
print(f" pf {tool:<15}")
|
||||
print()
|
||||
print("传统工具:")
|
||||
for tool in sorted(self._LEGACY_TOOLS):
|
||||
print(f" pf {tool:<15}")
|
||||
print()
|
||||
print("示例:")
|
||||
print(" pf filedate add a.txt")
|
||||
print(" pf pymake b")
|
||||
|
||||
def _tool_description(self, tool_name: str) -> str:
|
||||
"""获取工具描述 (从 _TOOL_REGISTRY 中已注册 ToolSpec 的 description/help)."""
|
||||
# 触发 @px.tool 注册
|
||||
with contextlib.suppress(ImportError):
|
||||
importlib.import_module(f"pyflowx.ops.{tool_name}")
|
||||
|
||||
if tool_name not in _TOOL_REGISTRY:
|
||||
return ""
|
||||
|
||||
subs = _TOOL_REGISTRY[tool_name]
|
||||
# 优先取 description, 否则取任一 subcommand 的 help
|
||||
for spec in subs.values():
|
||||
if spec.description:
|
||||
return spec.description
|
||||
# 取第一个非 hidden subcommand 的 help
|
||||
for spec in subs.values():
|
||||
if not spec.hidden and spec.help:
|
||||
return spec.help
|
||||
return ""
|
||||
|
||||
def _resolve_tool(self, name: str) -> tuple[str, str] | None:
|
||||
"""解析工具名, 返回 (类型, 目标).
|
||||
|
||||
类型: "tool" 或 "legacy"
|
||||
目标: ops 模块名或 legacy 模块路径
|
||||
"""
|
||||
if name in self._TOOL_ALIASES:
|
||||
return ("tool", self._TOOL_ALIASES[name])
|
||||
if name in self._LEGACY_TOOLS:
|
||||
return ("legacy", self._LEGACY_TOOLS[name])
|
||||
return None
|
||||
|
||||
def _run_legacy(self, module_path: str, argv: list[str]) -> int:
|
||||
"""运行传统工具的 main() 函数."""
|
||||
module_name, func_name = module_path.split(":", 1)
|
||||
module = importlib.import_module(module_name)
|
||||
func = getattr(module, func_name)
|
||||
|
||||
original_argv = sys.argv
|
||||
sys.argv = [f"pf {module_name.split('.')[-1]}", *argv]
|
||||
try:
|
||||
func()
|
||||
return 0
|
||||
except SystemExit as e:
|
||||
return int(e.code) if e.code is not None else 0
|
||||
finally:
|
||||
sys.argv = original_argv
|
||||
|
||||
def _run_tool(self, target: str, argv: list[str]) -> int:
|
||||
"""运行 @px.tool 工具.
|
||||
|
||||
导入 ``pyflowx.ops.<target>`` 触发 ``@px.tool`` 注册, 然后调用 ``run_tool``.
|
||||
"""
|
||||
with contextlib.suppress(ImportError):
|
||||
importlib.import_module(f"pyflowx.ops.{target}")
|
||||
|
||||
if target not in _TOOL_REGISTRY:
|
||||
print(f"错误: 未找到工具 '{target}'", file=sys.stderr)
|
||||
print("运行 'pf' 查看可用工具列表", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
return run_tool(target, argv)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""pf 统一入口主函数."""
|
||||
sys.exit(PfApp().run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,195 +0,0 @@
|
||||
"""pip 包管理工具模块.
|
||||
|
||||
提供 pip 包管理操作的封装,
|
||||
支持安装、卸载、下载等功能.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import fnmatch
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
PACKAGE_DIR = "packages"
|
||||
REQUIREMENTS_FILE = "requirements.txt"
|
||||
|
||||
# 受保护的包名集合
|
||||
_PROTECTED_PACKAGES: frozenset[str] = frozenset({
|
||||
"pyflowx",
|
||||
"bitool",
|
||||
})
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _get_installed_packages() -> list[str]:
|
||||
"""获取当前环境中所有已安装的包名."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pip", "list", "--format=freeze"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
packages: list[str] = []
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
if line and "==" in line:
|
||||
pkg_name = line.split("==")[0].strip()
|
||||
packages.append(pkg_name)
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
return []
|
||||
return packages
|
||||
|
||||
|
||||
def _expand_wildcard_packages(pattern: str) -> list[str]:
|
||||
"""展开通配符模式为实际的包名列表."""
|
||||
if not any(char in pattern for char in ["*", "?", "[", "]"]):
|
||||
return [pattern]
|
||||
|
||||
installed_packages = _get_installed_packages()
|
||||
matched = [pkg for pkg in installed_packages if fnmatch.fnmatchcase(pkg.lower(), pattern.lower())]
|
||||
return matched
|
||||
|
||||
|
||||
def _filter_protected_packages(packages: list[str]) -> list[str]:
|
||||
"""过滤掉受保护的包名."""
|
||||
safe = [p for p in packages if p.lower() not in {p.lower() for p in _PROTECTED_PACKAGES}]
|
||||
filtered = [p for p in packages if p.lower() in {p.lower() for p in _PROTECTED_PACKAGES}]
|
||||
if filtered:
|
||||
print(f"跳过受保护的包: {', '.join(filtered)}")
|
||||
return safe
|
||||
|
||||
|
||||
def pip_uninstall(pkg_names: list[str]) -> None:
|
||||
"""卸载包."""
|
||||
packages_to_uninstall: list[str] = []
|
||||
for pattern in pkg_names:
|
||||
packages_to_uninstall.extend(_expand_wildcard_packages(pattern))
|
||||
|
||||
packages_to_uninstall = _filter_protected_packages(packages_to_uninstall)
|
||||
|
||||
if not packages_to_uninstall:
|
||||
return
|
||||
|
||||
subprocess.run(["pip", "uninstall", "-y", *packages_to_uninstall], check=True)
|
||||
|
||||
|
||||
def pip_reinstall(pkg_names: list[str], offline: bool = False) -> None:
|
||||
"""重新安装包."""
|
||||
safe_pkgs = _filter_protected_packages(pkg_names)
|
||||
if not safe_pkgs:
|
||||
print("所有指定的包均为受保护包, 跳过重装")
|
||||
return
|
||||
|
||||
subprocess.run(["pip", "uninstall", "-y", *safe_pkgs], check=True)
|
||||
|
||||
options = ["--no-index", "--find-links", "."] if offline else []
|
||||
subprocess.run(["pip", "install", *options, *safe_pkgs], check=True)
|
||||
|
||||
|
||||
def pip_download(pkg_names: list[str], offline: bool = False) -> None:
|
||||
"""下载包."""
|
||||
options = ["--no-index", "--find-links", "."] if offline else []
|
||||
subprocess.run(
|
||||
["pip", "download", *pkg_names, *options, "-d", PACKAGE_DIR],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def pip_freeze() -> None:
|
||||
"""冻结依赖."""
|
||||
result = subprocess.run(
|
||||
["pip", "freeze", "--exclude-editable"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
Path(REQUIREMENTS_FILE).write_text(result.stdout)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""pip 工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="PipTool - pip 包管理工具",
|
||||
usage="piptool <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 安装命令
|
||||
install_parser = subparsers.add_parser("i", help="安装包")
|
||||
install_parser.add_argument("packages", nargs="+", help="要安装的包名")
|
||||
|
||||
# 卸载命令
|
||||
uninstall_parser = subparsers.add_parser("u", help="卸载包")
|
||||
uninstall_parser.add_argument("packages", nargs="+", help="要卸载的包名 (支持通配符)")
|
||||
|
||||
# 重装命令
|
||||
reinstall_parser = subparsers.add_parser("r", help="重新安装包")
|
||||
reinstall_parser.add_argument("packages", nargs="+", help="要重装的包名")
|
||||
reinstall_parser.add_argument("--offline", action="store_true", help="使用离线模式")
|
||||
|
||||
# 下载命令
|
||||
download_parser = subparsers.add_parser("d", help="下载包")
|
||||
download_parser.add_argument("packages", nargs="+", help="要下载的包名")
|
||||
download_parser.add_argument("--offline", action="store_true", help="使用离线模式")
|
||||
|
||||
# 升级 pip 命令
|
||||
subparsers.add_parser("up", help="升级 pip")
|
||||
|
||||
# 冻结依赖命令
|
||||
subparsers.add_parser("f", help="冻结依赖到 requirements.txt")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "i":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("pip_install", cmd=["pip", "install", *args.packages], verbose=True)])
|
||||
elif args.command == "u":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("pip_uninstall", fn=pip_uninstall, args=(args.packages,), verbose=True)
|
||||
])
|
||||
elif args.command == "r":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec(
|
||||
"pip_reinstall",
|
||||
fn=pip_reinstall,
|
||||
args=(args.packages,),
|
||||
kwargs={"offline": args.offline},
|
||||
verbose=True,
|
||||
)
|
||||
])
|
||||
elif args.command == "d":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec(
|
||||
"pip_download",
|
||||
fn=pip_download,
|
||||
args=(args.packages,),
|
||||
kwargs={"offline": args.offline},
|
||||
verbose=True,
|
||||
)
|
||||
])
|
||||
elif args.command == "up":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("pip_upgrade", cmd=["python", "-m", "pip", "install", "--upgrade", "pip"], verbose=True)
|
||||
])
|
||||
elif args.command == "f":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("pip_freeze", fn=pip_freeze, verbose=True)])
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,272 @@
|
||||
"""pxp —— PyFlowX 性能分析器.
|
||||
|
||||
分析包含 ``px`` 调用的 Python 脚本,生成工作流执行性能剖面报告。
|
||||
|
||||
工作原理
|
||||
--------
|
||||
1. 注入 hook:monkey-patch ``pyflowx.run`` / ``pyflowx.executors.run`` /
|
||||
``pyflowx.runner.run``,捕获最后一次执行的 ``Graph`` 与 ``RunReport``。
|
||||
2. 执行目标脚本:用 ``runpy.run_path`` 以 ``__main__`` 身份执行,
|
||||
捕获 ``SystemExit``(脚本可能调 ``sys.exit``)。
|
||||
3. 生成报告:从捕获的 report + graph 构建 :class:`ProfileReport`,
|
||||
默认输出 HTML 并自动打开浏览器。
|
||||
|
||||
使用方式
|
||||
--------
|
||||
# 分析 pymake.py,生成 HTML 报告并打开浏览器
|
||||
pxp pymake.py
|
||||
|
||||
# 传递参数给被分析脚本(用 -- 分隔)
|
||||
pxp pymake.py -- t
|
||||
|
||||
# 指定输出文件
|
||||
pxp pymake.py -o report.html
|
||||
|
||||
# 不打开浏览器
|
||||
pxp pymake.py --no-browser
|
||||
|
||||
# 输出纯文本报告
|
||||
pxp pymake.py -E text
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["main"]
|
||||
|
||||
import argparse
|
||||
import runpy
|
||||
import sys
|
||||
import webbrowser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .. import executors as _executors
|
||||
from .. import runner as _runner
|
||||
from ..profiling import ProfileReport
|
||||
from ..report import RunReport
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
"""构建参数解析器。"""
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="pxp",
|
||||
description="PyFlowX 性能分析器:分析包含 px 调用的脚本,生成性能剖面报告。",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=(
|
||||
"示例:\n"
|
||||
" pxp pymake.py # 分析并打开 HTML 报告\n"
|
||||
" pxp pymake.py -- t # 传递参数 t 给脚本\n"
|
||||
" pxp pymake.py -E text # 输出纯文本报告\n"
|
||||
" pxp pymake.py -o out.html # 指定输出文件\n"
|
||||
),
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"--export",
|
||||
"-E",
|
||||
choices=["html", "text"],
|
||||
default="html",
|
||||
help="导出格式(默认: html)",
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"--no-browser",
|
||||
action="store_true",
|
||||
help="不自动打开浏览器(仅 HTML 格式有效)",
|
||||
)
|
||||
_ = parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
help="输出文件路径(默认: <script>_profile.html)",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def _capture_px_run() -> dict[str, Any]:
|
||||
"""注入 hook 捕获 px.run() 调用。
|
||||
|
||||
返回一个字典,``run()`` 执行后填充 ``graph`` 与 ``report``。
|
||||
同时返回还原函数用于 finally 块。
|
||||
|
||||
Note
|
||||
-----
|
||||
需同时 patch 三处引用:
|
||||
* ``pyflowx.executors.run`` —— 实际实现
|
||||
* ``pyflowx.runner.run`` —— ``CliRunner`` 直接 import 的引用
|
||||
* ``pyflowx.run`` —— 顶层包导出的引用(用户脚本常用 ``px.run()``)
|
||||
|
||||
另外 patch ``RunReport.__init__`` 以捕获 ``run()`` 内部创建的 report 实例。
|
||||
这对于 ``run()`` 抛出 ``TaskFailedError`` 的场景至关重要:此时 ``run()``
|
||||
不会正常返回 report,但 report 对象已在内部创建并填充了已执行任务的结果。
|
||||
通过 ``capture_enabled`` 标志确保只在 ``patched_run`` 调用期间捕获。
|
||||
"""
|
||||
captured: dict[str, Any] = {}
|
||||
original_exec_run = _executors.run
|
||||
original_runner_run = _runner.run
|
||||
# 惰性获取顶层 pyflowx.run 引用(避免循环导入)
|
||||
import pyflowx as px_mod
|
||||
|
||||
original_px_run = px_mod.run
|
||||
original_report_init = RunReport.__init__
|
||||
capture_enabled = [False]
|
||||
|
||||
def patched_report_init(self: RunReport, *args: Any, **kwargs: Any) -> None:
|
||||
original_report_init(self, *args, **kwargs)
|
||||
if capture_enabled[0]:
|
||||
captured["report"] = self
|
||||
|
||||
RunReport.__init__ = patched_report_init # type: ignore[assignment]
|
||||
|
||||
def patched_run(graph: Any, *args: Any, **kwargs: Any) -> RunReport:
|
||||
captured["graph"] = graph
|
||||
capture_enabled[0] = True
|
||||
try:
|
||||
report = original_exec_run(graph, *args, **kwargs)
|
||||
# 正常返回时确保 captured["report"] 是返回的 report
|
||||
captured["report"] = report
|
||||
return report
|
||||
finally:
|
||||
capture_enabled[0] = False
|
||||
|
||||
# patch 所有引用 run 的入口
|
||||
_executors.run = patched_run # type: ignore[assignment]
|
||||
_runner.run = patched_run # type: ignore[assignment]
|
||||
px_mod.run = patched_run # type: ignore[assignment]
|
||||
|
||||
def _restore() -> None:
|
||||
_executors.run = original_exec_run # type: ignore[assignment]
|
||||
_runner.run = original_runner_run # type: ignore[assignment]
|
||||
px_mod.run = original_px_run # type: ignore[assignment]
|
||||
RunReport.__init__ = original_report_init # type: ignore[assignment]
|
||||
|
||||
captured["_restore"] = _restore
|
||||
return captured
|
||||
|
||||
|
||||
def _run_target_script(script: Path, script_args: list[str]) -> dict[str, Any]:
|
||||
"""执行目标脚本。
|
||||
|
||||
将脚本所在目录加入 ``sys.path``,设置 ``sys.argv``,然后用
|
||||
``runpy.run_path`` 以 ``__main__`` 身份执行。捕获 ``SystemExit``。
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[str, Any]
|
||||
脚本模块的全局变量字典(含 ``main`` 等定义)。
|
||||
"""
|
||||
sys.argv = [str(script), *script_args]
|
||||
script_dir = str(script.parent.resolve())
|
||||
if script_dir not in sys.path:
|
||||
sys.path.insert(0, script_dir)
|
||||
return runpy.run_path(str(script), run_name="__main__")
|
||||
|
||||
|
||||
def _try_call_main(module_globals: dict[str, Any]) -> None:
|
||||
"""若模块定义了 ``main`` 可调用对象,调用它。
|
||||
|
||||
用于脚本无 ``if __name__ == "__main__"`` 块的场景(如通过 entry points
|
||||
注册的 CLI 工具脚本)。``main`` 通常调用 ``CliRunner.run_cli()``,
|
||||
后者读取 ``sys.argv[1:]`` 执行对应命令。
|
||||
"""
|
||||
main_fn = module_globals.get("main")
|
||||
if callable(main_fn):
|
||||
main_fn()
|
||||
|
||||
|
||||
def _output_report(
|
||||
profile: ProfileReport,
|
||||
export: str,
|
||||
output: str | None,
|
||||
script_stem: str,
|
||||
no_browser: bool,
|
||||
) -> None:
|
||||
"""输出性能报告。"""
|
||||
if export == "text":
|
||||
print(profile.describe())
|
||||
return
|
||||
|
||||
# HTML 格式
|
||||
html = profile.to_html()
|
||||
if output:
|
||||
out_path = Path(output)
|
||||
else:
|
||||
out_path = Path.cwd() / f"{script_stem}_profile.html"
|
||||
out_path.write_text(html, encoding="utf-8")
|
||||
print(f"HTML 报告已生成: {out_path}")
|
||||
|
||||
if not no_browser:
|
||||
try:
|
||||
webbrowser.open(f"file://{out_path.resolve()}")
|
||||
except Exception as e:
|
||||
print(f"警告:无法打开浏览器: {e}", file=sys.stderr)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""pxp CLI 入口。"""
|
||||
parser = _build_parser()
|
||||
pxp_args, remaining = parser.parse_known_args()
|
||||
|
||||
if not remaining:
|
||||
parser.print_help()
|
||||
sys.exit(2)
|
||||
|
||||
script_str = remaining[0]
|
||||
script_args = remaining[1:]
|
||||
script_path = Path(script_str).resolve()
|
||||
|
||||
if not script_path.is_file():
|
||||
print(f"错误:脚本不存在: {script_path}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# 注入 hook
|
||||
captured = _capture_px_run()
|
||||
|
||||
# 执行目标脚本
|
||||
print(f"正在分析: {script_path}")
|
||||
if script_args:
|
||||
print(f"脚本参数: {script_args}")
|
||||
print("-" * 60)
|
||||
|
||||
module_globals: dict[str, Any] = {}
|
||||
try:
|
||||
module_globals = _run_target_script(script_path, script_args)
|
||||
except SystemExit:
|
||||
# 脚本调用了 sys.exit,正常情况
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"警告:脚本执行抛出异常: {e}", file=sys.stderr)
|
||||
|
||||
# 若脚本执行未捕获到 run(),尝试调用模块的 main() 函数
|
||||
# (适用于无 ``if __name__ == "__main__"`` 块的 CLI 脚本)
|
||||
if captured.get("report") is None and module_globals:
|
||||
try:
|
||||
_try_call_main(module_globals)
|
||||
except SystemExit:
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"警告:调用 main() 抛出异常: {e}", file=sys.stderr)
|
||||
|
||||
# 还原 hook
|
||||
restore = captured.pop("_restore", None)
|
||||
if restore is not None:
|
||||
restore()
|
||||
|
||||
# 检查是否捕获到 run() 调用
|
||||
report = captured.get("report")
|
||||
graph = captured.get("graph")
|
||||
if report is None or graph is None:
|
||||
print("错误:未捕获到 px.run() 调用,无法生成性能报告", file=sys.stderr)
|
||||
print("请确保脚本通过 px.run() 或 CliRunner 执行任务流图。", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# 生成报告
|
||||
profile = ProfileReport.from_report(report, graph)
|
||||
_output_report(
|
||||
profile,
|
||||
export=pxp_args.export,
|
||||
output=pxp_args.output,
|
||||
script_stem=script_path.stem,
|
||||
no_browser=pxp_args.no_browser,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,125 +0,0 @@
|
||||
"""Python 构建工具模块.
|
||||
|
||||
完全替代传统的 Makefile,
|
||||
提供更好的跨平台兼容性和 Python 生态集成.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
def maturin_build_cmd() -> list[str]:
|
||||
"""获取 maturin 构建命令(根据平台自动添加参数).
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[str]
|
||||
完整的 maturin 构建命令列表.
|
||||
"""
|
||||
command = ["maturin", "build", "-r"].copy()
|
||||
if Constants.IS_WINDOWS:
|
||||
command.extend(["--target", "x86_64-win7-windows-msvc", "-Zbuild-std", "-i", "python3.8"])
|
||||
return command
|
||||
|
||||
|
||||
uv_build: px.TaskSpec = px.TaskSpec("uv_build", cmd=["uv", "build"])
|
||||
maturin_build: px.TaskSpec = px.TaskSpec("maturin_build", cmd=maturin_build_cmd())
|
||||
uv_sync: px.TaskSpec = px.TaskSpec("uv_sync", cmd=["uv", "sync"])
|
||||
git_clean: px.TaskSpec = px.TaskSpec("git_clean", cmd=["gitt", "c"])
|
||||
test: px.TaskSpec = px.TaskSpec(
|
||||
"test", cmd=["pytest", "-m", "not slow", "-n", "8", "--dist", "loadfile", "--color=yes", "--durations=10"]
|
||||
)
|
||||
test_fast: px.TaskSpec = px.TaskSpec(
|
||||
"test_fast", cmd=["pytest", "-m", "not slow", "--dist", "loadfile", "--color=yes", "--durations=10"]
|
||||
)
|
||||
test_coverage: px.TaskSpec = px.TaskSpec(
|
||||
"test_coverage",
|
||||
cmd=["pytest", "--cov", "-n", "8", "--dist", "loadfile", "--tb=short", "-v", "--color=yes", "--durations=10"],
|
||||
)
|
||||
ruff_lint: px.TaskSpec = px.TaskSpec("lint", cmd=["ruff", "check", "--fix", "--unsafe-fixes"])
|
||||
typecheck: px.TaskSpec = px.TaskSpec("pyrefly_check", cmd=["pyrefly", "check", "."])
|
||||
git_add_all: px.TaskSpec = px.TaskSpec("git_add_all", cmd=["git", "add", "-A"])
|
||||
bump: px.TaskSpec = px.TaskSpec("bumpversion", cmd=["bumpversion"])
|
||||
doc: px.TaskSpec = px.TaskSpec("doc", cmd=["sphinx-build", "-b", "html", "docs", "docs/_build"])
|
||||
git_push: px.TaskSpec = px.TaskSpec("git_push", cmd=["git", "push"])
|
||||
git_push_tags: px.TaskSpec = px.TaskSpec("git_push_tags", cmd=["git", "push", "--tags"])
|
||||
hatch_publish: px.TaskSpec = px.TaskSpec("publish_python", cmd=["hatch", "publish"])
|
||||
twine_publish: px.TaskSpec = px.TaskSpec("twine_publish", cmd=["twine", "upload", "--disable-progress-bar"])
|
||||
tox: px.TaskSpec = px.TaskSpec("tox", cmd=["tox", "-p", "auto"])
|
||||
|
||||
|
||||
def main():
|
||||
"""pymake 构建工具.
|
||||
|
||||
🔨 构建命令:
|
||||
pymake b - 构建 Python 主包 (uv build)
|
||||
pymake bc - 构建 Rust 核心模块 (maturin build)
|
||||
pymake ba - 构建所有包 (先 Python 后 Rust)
|
||||
|
||||
📦 安装命令 (开发模式):
|
||||
pymake sync - 安装依赖包 (uv sync)
|
||||
|
||||
🧹 清理命令:
|
||||
pymake c - 清理所有构建产物 (gitt c)
|
||||
|
||||
🛠️ 开发工具:
|
||||
pymake t - 运行测试 (pytest)
|
||||
pymake tc - 运行测试并生成覆盖率报告
|
||||
pymake tf - 运行快速测试 (pytest -m not slow)
|
||||
pymake lint - 代码格式化与检查 (ruff)
|
||||
pymake type - 类型检查 (mypy, ty)
|
||||
pymake doc - 构建文档 (sphinx)
|
||||
|
||||
🔬 多版本测试:
|
||||
pymake tox - 多版本 Python 测试 (tox -p auto)
|
||||
|
||||
📦 发布命令:
|
||||
pymake pb - 发布到 PyPI (twine + hatch)
|
||||
|
||||
� 版本管理:
|
||||
pymake bump - 自动升级版本号并提交修改 (清理 + 检查 + 格式化 + git add + bumpversion)
|
||||
|
||||
�💡 常用工作流:
|
||||
1. 日常开发: pymake lint && pymake t
|
||||
2. 构建发布包: pymake ba
|
||||
3. 多版本兼容性测试: pymake tox
|
||||
4. 发布到 PyPI: pymake pb
|
||||
|
||||
📝 示例:
|
||||
pymake ba # 构建所有包
|
||||
pymake sync # 安装依赖
|
||||
pymake t # 运行测试
|
||||
pymake tox # 多版本兼容性测试
|
||||
pymake lint # 格式化代码
|
||||
pymake type # 类型检查
|
||||
"""
|
||||
runner = px.CliRunner(
|
||||
strategy="sequential",
|
||||
description="PyMake - Python 构建工具",
|
||||
graphs={
|
||||
# 构建命令
|
||||
"b": px.Graph.from_specs([uv_build]),
|
||||
"bc": px.Graph.from_specs([maturin_build]),
|
||||
"ba": px.Graph.from_specs(["b", "bc"]),
|
||||
# 安装命令
|
||||
"sync": px.Graph.from_specs([uv_sync]),
|
||||
# 清理命令
|
||||
"c": px.Graph.from_specs([git_clean]),
|
||||
# 开发工具
|
||||
"bump": px.Graph.from_specs(["c", "tc", git_add_all, bump]),
|
||||
"bumpmi": px.Graph.from_specs([px.TaskSpec("bumpversion_minor", cmd=["bumpversion", "minor"])]),
|
||||
"cov": px.Graph.from_specs([git_clean, test_coverage]),
|
||||
"doc": px.Graph.from_specs([doc]),
|
||||
"lint": px.Graph.from_specs([ruff_lint]),
|
||||
"pb": px.Graph.from_specs([twine_publish, hatch_publish]),
|
||||
"t": px.Graph.from_specs([test]),
|
||||
"tf": px.Graph.from_specs([test_fast]),
|
||||
"tc": px.Graph.from_specs([typecheck, "lint"]),
|
||||
"tox": px.Graph.from_specs([tox]),
|
||||
# 发布命令
|
||||
"p": px.Graph.from_specs([git_clean, git_push, git_push_tags]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
@@ -1,10 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.tasks.system import reset_icon_cache
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""重启图标缓存工具主函数."""
|
||||
graph = px.Graph.from_specs(reset_icon_cache())
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,40 +0,0 @@
|
||||
"""进程终止工具.
|
||||
|
||||
跨平台进程终止工具, 支持按名称终止进程.
|
||||
用法: taskkill proc_name [proc_name ...]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""进程终止工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="TaskKill - 进程终止工具",
|
||||
usage="taskkill <process_name> [process_name ...]",
|
||||
)
|
||||
parser.add_argument(
|
||||
"process_names",
|
||||
type=str,
|
||||
nargs="+",
|
||||
help="进程名称 (如: chrome.exe python node)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if Constants.IS_WINDOWS:
|
||||
cmd = ["taskkill", "/f", "/im"]
|
||||
else:
|
||||
cmd = ["pkill", "-f"]
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(f"kill_{proc_name}", cmd=[*cmd, f"{proc_name}*"], verbose=True)
|
||||
for proc_name in args.process_names
|
||||
]
|
||||
)
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,21 +0,0 @@
|
||||
"""命令查找工具.
|
||||
|
||||
跨平台查找可执行命令路径, 类似 Unix 的 which 命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.tasks.system import which
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""命令查找工具主函数."""
|
||||
parser = argparse.ArgumentParser(description="Which - 命令查找工具")
|
||||
parser.add_argument("commands", nargs="+", help="要查找的命令名称, 如: python ls ps gcc...")
|
||||
args = parser.parse_args()
|
||||
|
||||
graph = px.Graph.from_specs([which(cmd) for cmd in args.commands])
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,100 @@
|
||||
"""命令执行器:把 :class:`~pyflowx.task.TaskSpec` 的 ``cmd`` 字段(list /
|
||||
shell 字符串 / 可调用对象)转换为统一执行入口。
|
||||
|
||||
历史背景:原 ``task.py`` 的模块文档声明其为"纯数据结构",但 ``_run_command``
|
||||
属于命令执行逻辑,违反单一职责。此处将其抽离,``TaskSpec`` 仅持有配置,
|
||||
执行逻辑集中于本模块,便于独立测试与维护。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from typing import Any, List, Union, cast
|
||||
|
||||
from .task import TaskSpec
|
||||
|
||||
__all__ = ["run_command"]
|
||||
|
||||
|
||||
def run_command(spec: TaskSpec[Any]) -> Any: # noqa: PLR0912
|
||||
"""执行 ``spec.cmd`` 指定的命令(list / shell 字符串 / 可调用对象)。
|
||||
|
||||
与原 ``TaskSpec._run_command`` 行为一致:
|
||||
|
||||
- 可调用对象:直接调用,异常包装为 :class:`RuntimeError`。
|
||||
- list / str:通过 :func:`subprocess.run` 执行,非零返回码抛
|
||||
:class:`RuntimeError`(``verbose=False`` 时附 stderr)。
|
||||
- ``verbose=True`` 时打印执行信息与返回码到 stdout。
|
||||
- ``cwd`` / ``env`` 通过 subprocess 参数隔离(进程级状态仅在 fn 任务路径
|
||||
使用,cmd 路径不依赖 ``os.chdir`` / ``os.environ``)。
|
||||
"""
|
||||
cmd = spec.cmd
|
||||
verbose = spec.verbose
|
||||
cwd = spec.cwd
|
||||
timeout = spec.timeout
|
||||
env_override = spec.env
|
||||
|
||||
# 可调用对象:直接调用,返回其结果。
|
||||
if callable(cmd) and not isinstance(cmd, (list, str)):
|
||||
name = getattr(cmd, "__name__", "callable")
|
||||
if verbose:
|
||||
print(f"[verbose] 执行可调用命令: {name}", flush=True)
|
||||
if cwd is not None:
|
||||
print(f"[verbose] 工作目录: {cwd}", flush=True)
|
||||
try:
|
||||
return cmd()
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"可调用命令执行异常: {name}: {e}") from e
|
||||
|
||||
is_list = isinstance(cmd, list)
|
||||
if is_list:
|
||||
cmd_str = " ".join(arg for arg in cmd) # type: ignore[union-attr]
|
||||
verb = "执行命令"
|
||||
label = "命令"
|
||||
else:
|
||||
cmd_str = cast(str, cmd)
|
||||
verb = "执行 Shell"
|
||||
label = "Shell 命令"
|
||||
|
||||
if verbose:
|
||||
print(f"[verbose] {verb}: {cmd_str}", flush=True)
|
||||
if cwd is not None:
|
||||
print(f"[verbose] 工作目录: {cwd}", flush=True)
|
||||
|
||||
# 合并环境变量
|
||||
run_env: dict[str, str] | None = None
|
||||
if env_override:
|
||||
run_env = dict(os.environ)
|
||||
run_env.update(env_override)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cast(Union[str, List[str]], cmd),
|
||||
shell=not is_list,
|
||||
cwd=cwd,
|
||||
env=run_env,
|
||||
timeout=timeout,
|
||||
capture_output=not verbose,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError(f"{label}未找到: {cmd_str}") from None
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError(f"{label}执行超时: {cmd_str} ({timeout}s)") from None
|
||||
except OSError as e:
|
||||
raise RuntimeError(f"{label}执行异常: {cmd_str}: {e}") from e
|
||||
|
||||
if verbose:
|
||||
print(f"[verbose] 返回码: {result.returncode}", flush=True)
|
||||
|
||||
if result.returncode == 0:
|
||||
if not verbose and result.stdout:
|
||||
print(result.stdout, end="", flush=True)
|
||||
return None
|
||||
|
||||
err_msg = f"{label}执行失败: `{cmd_str}`, 返回码: {result.returncode}"
|
||||
if not verbose and result.stderr.strip():
|
||||
err_msg += f"\n{result.stderr.strip()}"
|
||||
raise RuntimeError(err_msg)
|
||||
@@ -0,0 +1,115 @@
|
||||
"""图组合:将带字符串引用的多个图展开为纯 :class:`~pyflowx.graph.Graph`。
|
||||
|
||||
历史背景:原 ``graph.py`` 同时承载 DAG 构建/校验/分层与多图组合逻辑,
|
||||
职责过载。组合逻辑(:class:`GraphComposer` / :func:`compose`)与单图 DAG
|
||||
模型正交,此处抽离为独立模块,便于按需导入与独立演进。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from typing import Any
|
||||
|
||||
from .graph import Graph
|
||||
from .task import TaskSpec
|
||||
|
||||
__all__ = ["GraphComposer", "compose"]
|
||||
|
||||
|
||||
class GraphComposer:
|
||||
"""将带字符串引用的图展开为纯 :class:`TaskSpec` 图。
|
||||
|
||||
引用格式:
|
||||
* ``"command_name"`` —— 引用整个命令图。
|
||||
* ``"command_name.task_name"`` —— 引用特定任务。
|
||||
|
||||
引用按顺序展开,后续引用的任务依赖前面引用的最后一个任务;
|
||||
原始 ``TaskSpec`` 之间也按出现顺序串行依赖。
|
||||
"""
|
||||
|
||||
def __init__(self, graphs: dict[str, Graph]) -> None:
|
||||
self.graphs = graphs
|
||||
|
||||
def resolve_all(self) -> dict[str, Graph]:
|
||||
"""解析所有图的字符串引用,返回展开后的新图映射。"""
|
||||
resolved: dict[str, Graph] = {}
|
||||
for cmd_name, graph in self.graphs.items():
|
||||
resolved[cmd_name] = self.expand_refs(graph, cmd_name)
|
||||
return resolved
|
||||
|
||||
def expand_refs(self, graph: Graph, current_cmd: str) -> Graph:
|
||||
"""展开图中的字符串引用。若无 ``_pending_refs``,原样返回。"""
|
||||
pending_refs = graph._pending_refs
|
||||
if not pending_refs:
|
||||
return graph
|
||||
|
||||
all_specs: list[TaskSpec[Any]] = []
|
||||
previous_ref_last_task: str | None = None
|
||||
|
||||
for ref in pending_refs:
|
||||
expanded_specs = self.parse_ref(ref, current_cmd)
|
||||
if previous_ref_last_task and expanded_specs:
|
||||
for i, task in enumerate(expanded_specs):
|
||||
if i == 0 or not task.depends_on:
|
||||
expanded_specs[i] = replace(task, depends_on=tuple({*task.depends_on, previous_ref_last_task}))
|
||||
if expanded_specs:
|
||||
previous_ref_last_task = expanded_specs[-1].name
|
||||
all_specs.extend(expanded_specs)
|
||||
|
||||
original_specs = list(graph.all_specs().values())
|
||||
if original_specs:
|
||||
if previous_ref_last_task:
|
||||
first = original_specs[0]
|
||||
all_specs.append(replace(first, depends_on=tuple({*first.depends_on, previous_ref_last_task})))
|
||||
else:
|
||||
all_specs.append(original_specs[0])
|
||||
for i in range(1, len(original_specs)):
|
||||
current_task = original_specs[i]
|
||||
previous_task_name = original_specs[i - 1].name
|
||||
all_specs.append(
|
||||
replace(current_task, depends_on=tuple({*current_task.depends_on, previous_task_name}))
|
||||
)
|
||||
|
||||
return Graph.from_specs(all_specs, defaults=graph.defaults)
|
||||
|
||||
def parse_ref(self, ref: str, current_cmd: str) -> list[TaskSpec[Any]]:
|
||||
"""解析单个字符串引用,返回对应的 TaskSpec 列表。"""
|
||||
if ref == current_cmd:
|
||||
raise ValueError(f"循环引用: 命令 '{current_cmd}' 引用了自己")
|
||||
|
||||
if "." in ref:
|
||||
cmd_name, task_name = ref.split(".", 1)
|
||||
if cmd_name not in self.graphs:
|
||||
raise ValueError(f"引用的命令 '{cmd_name}' 不存在")
|
||||
ref_graph = self.graphs[cmd_name]
|
||||
if task_name not in ref_graph.all_specs():
|
||||
raise ValueError(f"任务 '{task_name}' 不存在于命令 '{cmd_name}' 中")
|
||||
return [ref_graph.all_specs()[task_name]]
|
||||
else:
|
||||
cmd_name = ref
|
||||
if cmd_name not in self.graphs:
|
||||
raise ValueError(f"引用的命令 '{cmd_name}' 不存在")
|
||||
ref_graph = self.graphs[cmd_name]
|
||||
ref_graph = self.expand_refs(ref_graph, cmd_name)
|
||||
return list(ref_graph.all_specs().values())
|
||||
|
||||
|
||||
def compose(
|
||||
graphs: dict[str, Graph],
|
||||
) -> dict[str, Graph]:
|
||||
"""编程式解析多图的字符串引用,返回展开后的新图映射。
|
||||
|
||||
与 :class:`GraphComposer` 等价,但作为独立函数暴露,供不使用
|
||||
:class:`~pyflowx.runner.CliRunner` 的编程式用户调用。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> graphs = {
|
||||
... "build": px.Graph.from_specs([px.TaskSpec("b", cmd=["echo", "b"])]),
|
||||
... "all": px.Graph.from_specs(["build", px.TaskSpec("t", cmd=["echo", "t"])]),
|
||||
... }
|
||||
>>> resolved = px.compose(graphs)
|
||||
>>> "b" in resolved["all"].all_specs()
|
||||
True
|
||||
"""
|
||||
return GraphComposer(graphs).resolve_all()
|
||||
@@ -11,6 +11,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -20,6 +21,8 @@ from typing import Any, Callable
|
||||
|
||||
from .task import Condition, Context
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = ["BuiltinConditions", "Condition", "Constants"]
|
||||
|
||||
|
||||
@@ -42,6 +45,11 @@ def _static(predicate: Callable[[], bool], name: str) -> Condition:
|
||||
return _cond
|
||||
|
||||
|
||||
def _cond_name(cond: Condition) -> str:
|
||||
"""获取条件的可读名称。"""
|
||||
return getattr(cond, "__name__", repr(cond))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 模块级静态条件常量
|
||||
# ---------------------------------------------------------------------- #
|
||||
@@ -61,6 +69,26 @@ class BuiltinConditions:
|
||||
# ------------------------------------------------------------------ #
|
||||
# 静态条件
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
def IS_WINDOWS() -> Condition:
|
||||
"""检查是否为 Windows 平台."""
|
||||
return IS_WINDOWS
|
||||
|
||||
@staticmethod
|
||||
def IS_LINUX() -> Condition:
|
||||
"""检查是否为 Linux 平台."""
|
||||
return IS_LINUX
|
||||
|
||||
@staticmethod
|
||||
def IS_MACOS() -> Condition:
|
||||
"""检查是否为 macOS 平台."""
|
||||
return IS_MACOS
|
||||
|
||||
@staticmethod
|
||||
def IS_POSIX() -> Condition:
|
||||
"""检查是否为 POSIX 平台."""
|
||||
return IS_POSIX
|
||||
|
||||
@staticmethod
|
||||
def PYTHON_VERSION(major: int, minor: int | None = None) -> Condition:
|
||||
"""检查 Python 版本是否匹配."""
|
||||
@@ -118,6 +146,21 @@ class BuiltinConditions:
|
||||
f"ENV_VAR_EQUALS({var_name!r},{value!r})",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def FILE_CONTENT_EXISTS(path: Path | str, content: str) -> Condition:
|
||||
"""检查文件是否包含指定内容."""
|
||||
|
||||
def _check() -> bool:
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
return False
|
||||
try:
|
||||
return content in p.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return False
|
||||
|
||||
return _static(_check, f"FILE_CONTENT_EXISTS({path!r},{content!r})")
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 上下文条件:基于上游依赖结果
|
||||
# ------------------------------------------------------------------ #
|
||||
@@ -146,7 +189,8 @@ class BuiltinConditions:
|
||||
return False
|
||||
try:
|
||||
return predicate(ctx[dep_name])
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
logger.warning("DEP_MATCHES predicate %r raised: %r", dep_name, exc)
|
||||
return False
|
||||
|
||||
_cond.__name__ = f"DEP_MATCHES({dep_name!r},{getattr(predicate, '__name__', 'pred')})"
|
||||
@@ -182,7 +226,7 @@ class BuiltinConditions:
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return not condition(ctx)
|
||||
|
||||
_cond.__name__ = f"NOT({getattr(condition, '__name__', repr(condition))})"
|
||||
_cond.__name__ = f"NOT({_cond_name(condition)})"
|
||||
return _cond
|
||||
|
||||
@staticmethod
|
||||
@@ -192,8 +236,7 @@ class BuiltinConditions:
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return all(c(ctx) for c in conditions)
|
||||
|
||||
names = [getattr(c, "__name__", repr(c)) for c in conditions]
|
||||
_cond.__name__ = f"AND({', '.join(names)})"
|
||||
_cond.__name__ = f"AND({', '.join(_cond_name(c) for c in conditions)})"
|
||||
return _cond
|
||||
|
||||
@staticmethod
|
||||
@@ -203,6 +246,5 @@ class BuiltinConditions:
|
||||
def _cond(ctx: Context) -> bool:
|
||||
return any(c(ctx) for c in conditions)
|
||||
|
||||
names = [getattr(c, "__name__", repr(c)) for c in conditions]
|
||||
_cond.__name__ = f"OR({', '.join(names)})"
|
||||
_cond.__name__ = f"OR({', '.join(_cond_name(c) for c in conditions)})"
|
||||
return _cond
|
||||
|
||||
+21
-2
@@ -16,6 +16,7 @@ DAG 库中泛滥的样板包装器。
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from functools import lru_cache
|
||||
from typing import Any, Mapping
|
||||
|
||||
from .errors import InjectionError
|
||||
@@ -24,6 +25,24 @@ from .task import Context, TaskSpec
|
||||
__all__ = ["Context", "_is_context_annotation", "build_call_args", "describe_injection"]
|
||||
|
||||
|
||||
@lru_cache(maxsize=1024)
|
||||
def _cached_signature(fn: Any) -> inspect.Signature:
|
||||
"""缓存 ``inspect.signature`` 结果(按 fn 对象键控)。
|
||||
|
||||
``fn`` 对象在 :meth:`TaskSpec.effective_fn` 缓存后稳定,签名重复内省
|
||||
属纯开销。对不可哈希的可调用对象,调用方回退到直接内省。
|
||||
"""
|
||||
return inspect.signature(fn)
|
||||
|
||||
|
||||
def _signature(fn: Any) -> inspect.Signature:
|
||||
"""获取签名,优先走缓存;``fn`` 不可哈希时回退到直接内省。"""
|
||||
try:
|
||||
return _cached_signature(fn)
|
||||
except TypeError:
|
||||
return inspect.signature(fn)
|
||||
|
||||
|
||||
def _is_context_annotation(annotation: Any) -> bool:
|
||||
"""判断参数标注是否为(或指向)``Context``。"""
|
||||
if annotation is Context:
|
||||
@@ -44,7 +63,7 @@ def build_call_args(
|
||||
执行器填入 :attr:`TaskSpec.defaults` 中的默认值)。
|
||||
"""
|
||||
fn = spec.effective_fn
|
||||
sig = inspect.signature(fn)
|
||||
sig = _signature(fn)
|
||||
params = sig.parameters
|
||||
|
||||
var_keyword = next(
|
||||
@@ -115,7 +134,7 @@ def build_call_args(
|
||||
def describe_injection(spec: TaskSpec[Any]) -> str:
|
||||
"""生成任务参数注入方式的人类可读描述。供 ``dry_run`` 使用。"""
|
||||
fn = spec.effective_fn
|
||||
sig = inspect.signature(fn)
|
||||
sig = _signature(fn)
|
||||
positional_params = [
|
||||
p
|
||||
for p, param in sig.parameters.items()
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
"""Example 3: async aggregation with static args and Context injection.
|
||||
|
||||
Shows:
|
||||
* async task functions executed with strategy="async".
|
||||
* static positional args (TaskSpec.args) for parameterised tasks.
|
||||
* Context annotation to receive the full upstream result mapping.
|
||||
* on_event callback for real-time progress.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
|
||||
async def fetch_user(uid: int) -> dict[str, Any]:
|
||||
await asyncio.sleep(0.2)
|
||||
return {"id": uid, "name": f"User{uid}"}
|
||||
|
||||
|
||||
async def fetch_posts(uid: int) -> list[int]:
|
||||
await asyncio.sleep(0.2)
|
||||
return [uid, uid + 1]
|
||||
|
||||
|
||||
# Context annotation → receives the full mapping of upstream results.
|
||||
def aggregate(ctx: px.Context) -> dict[str, Any]:
|
||||
return dict(ctx)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
graph = px.Graph.from_specs([
|
||||
# Static positional args parameterise the same function twice.
|
||||
px.TaskSpec("fetch_user", fetch_user, args=(1,)),
|
||||
px.TaskSpec("fetch_posts", fetch_posts, args=(1,)),
|
||||
px.TaskSpec("aggregate", aggregate, depends_on=("fetch_user", "fetch_posts")),
|
||||
])
|
||||
|
||||
print("=== Dry run ===")
|
||||
_ = px.run(graph, strategy="async", dry_run=True)
|
||||
|
||||
events: list[px.TaskEvent] = []
|
||||
print("\n=== Async execution ===")
|
||||
report = px.run(graph, strategy="async", on_event=events.append)
|
||||
|
||||
for ev in events:
|
||||
print(f" event: {ev.task} -> {ev.status.value}")
|
||||
|
||||
print(f"\naggregate = {report['aggregate']}")
|
||||
print(report.describe())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,77 +0,0 @@
|
||||
"""Example 1: ETL pipeline (sequential strategy).
|
||||
|
||||
Demonstrates the core PyFlowX workflow:
|
||||
* Define tasks as plain functions.
|
||||
* Declare the DAG with a list of TaskSpec.
|
||||
* Parameter names == dependency names → automatic context injection,
|
||||
no wrappers needed (contrast with flowweaver's get_task_result boilerplate).
|
||||
* dry_run to preview, then execute and read typed results from RunReport.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# --- task functions: pure, testable, no framework coupling ------------- #
|
||||
|
||||
|
||||
def extract_customers() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"id": "C001", "name": "Alice"},
|
||||
{"id": "C002", "name": "Bob"},
|
||||
]
|
||||
|
||||
|
||||
def extract_orders() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"id": "O001", "customer_id": "C001", "amount": 150.0},
|
||||
{"id": "O002", "customer_id": "C002", "amount": 200.5},
|
||||
]
|
||||
|
||||
|
||||
# Parameter names match dependency names → automatic injection.
|
||||
def transform(
|
||||
extract_customers: list[dict[str, Any]],
|
||||
extract_orders: list[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
cmap = {c["id"]: c for c in extract_customers}
|
||||
return [{**o, "customer_name": cmap[o["customer_id"]]["name"]} for o in extract_orders if o["customer_id"] in cmap]
|
||||
|
||||
|
||||
def load(transform: list[dict[str, Any]]) -> int:
|
||||
print(f" loaded {len(transform)} records")
|
||||
return len(transform)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("extract_customers", extract_customers, tags=("extract",)),
|
||||
px.TaskSpec("extract_orders", extract_orders, tags=("extract",)),
|
||||
px.TaskSpec(
|
||||
"transform",
|
||||
transform,
|
||||
depends_on=("extract_customers", "extract_orders"),
|
||||
tags=("transform",),
|
||||
),
|
||||
px.TaskSpec(
|
||||
"load", load, depends_on=("transform",), retry=px.RetryPolicy(max_attempts=1, delay=1.0), tags=("load",)
|
||||
),
|
||||
])
|
||||
|
||||
print("=== Execution plan ===")
|
||||
print(graph.describe())
|
||||
|
||||
print("\n=== Dry run (no execution) ===")
|
||||
_ = px.run(graph, strategy="sequential", dry_run=True)
|
||||
|
||||
print("\n=== Sequential execution ===")
|
||||
report = px.run(graph, strategy="sequential")
|
||||
print(report.describe())
|
||||
print(f"\nload result = {report['load']}")
|
||||
print(f"summary = {report.summary()}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,57 +0,0 @@
|
||||
"""Example 2: parallel execution (thread strategy).
|
||||
|
||||
Same DAG run with sequential vs. thread strategy to show layer-internal
|
||||
parallelism. Tasks within a layer run concurrently; layers are barriers.
|
||||
|
||||
Layer 1: [fetch_a, fetch_b] (parallel)
|
||||
Layer 2: [merge] (waits for both)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
|
||||
def fetch_a() -> str:
|
||||
time.sleep(0.5)
|
||||
return "a"
|
||||
|
||||
|
||||
def fetch_b() -> str:
|
||||
time.sleep(0.5)
|
||||
return "b"
|
||||
|
||||
|
||||
def merge(fetch_a: str, fetch_b: str) -> str:
|
||||
return fetch_a + fetch_b
|
||||
|
||||
|
||||
def main() -> None:
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("fetch_a", fetch_a),
|
||||
px.TaskSpec("fetch_b", fetch_b),
|
||||
px.TaskSpec("merge", merge, depends_on=("fetch_a", "fetch_b")),
|
||||
])
|
||||
|
||||
print("=== Mermaid diagram ===")
|
||||
print(graph.to_mermaid("LR"))
|
||||
|
||||
print("\n=== Sequential (expect ~1.0s) ===")
|
||||
start = time.time()
|
||||
report_seq = px.run(graph, strategy="sequential")
|
||||
t_seq = time.time() - start
|
||||
print(f" result={report_seq['merge']} time={t_seq:.2f}s")
|
||||
|
||||
print("\n=== Threaded (expect ~0.5s) ===")
|
||||
start = time.time()
|
||||
report_thr = px.run(graph, strategy="thread", max_workers=2)
|
||||
t_thr = time.time() - start
|
||||
print(f" result={report_thr['merge']} time={t_thr:.2f}s")
|
||||
|
||||
print(f"\nspeedup = {t_seq / t_thr:.2f}x")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+550
-455
File diff suppressed because it is too large
Load Diff
+180
-122
@@ -12,12 +12,18 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = [
|
||||
"Graph",
|
||||
"GraphDefaults",
|
||||
]
|
||||
|
||||
import inspect
|
||||
import sys
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import Any, Callable, Iterable, Mapping, Sequence
|
||||
|
||||
from .errors import CycleError, DuplicateTaskError, MissingDependencyError
|
||||
from .task import RetryPolicy, TaskSpec
|
||||
from .task import Context, RetryPolicy, TaskSpec
|
||||
|
||||
if sys.version_info >= (3, 9): # pragma: no cover
|
||||
import graphlib # pyright: ignore[reportUnreachable]
|
||||
@@ -49,6 +55,83 @@ class GraphDefaults:
|
||||
verbose: bool = False
|
||||
|
||||
|
||||
def _prune_deps(spec: TaskSpec[Any], keep: Callable[[str], bool]) -> TaskSpec[Any]:
|
||||
"""返回新 spec,其 ``depends_on`` / ``soft_depends_on`` 仅保留 ``keep(dep)`` 为真的依赖。"""
|
||||
return replace(
|
||||
spec,
|
||||
depends_on=tuple(d for d in spec.depends_on if keep(d)),
|
||||
soft_depends_on=tuple(d for d in spec.soft_depends_on if keep(d)),
|
||||
)
|
||||
|
||||
|
||||
def _make_namespaced_fn(orig_fn: Any, ns: str, dep_names: set[str]) -> Any:
|
||||
"""包装 fn,使其能接收带 ``ns:`` 前缀的依赖名,调用时映射回原参数名。
|
||||
|
||||
命名空间合并后,依赖名带前缀(如 ``build:extract``),但 Python 参数名
|
||||
不能含 ``:``。wrapper 用 ``**kwargs`` 接收所有依赖,内部把带前缀的依赖名
|
||||
映射回原参数名后调用原 fn。
|
||||
|
||||
无依赖参数时直接返回原 fn。
|
||||
"""
|
||||
if not dep_names or orig_fn is None:
|
||||
return orig_fn
|
||||
try:
|
||||
orig_sig = inspect.signature(orig_fn)
|
||||
except (TypeError, ValueError):
|
||||
return orig_fn
|
||||
|
||||
# 带前缀依赖名 -> 原参数名
|
||||
name_map: dict[str, str] = {f"{ns}:{orig}": orig for orig in dep_names}
|
||||
prefix = f"{ns}:"
|
||||
|
||||
# 检查原 fn 是否有 Context 标注参数
|
||||
context_param_name: str | None = None
|
||||
for p in orig_sig.parameters.values():
|
||||
ann = p.annotation
|
||||
if ann is not Context and not (isinstance(ann, str) and ann.endswith("Context")):
|
||||
continue
|
||||
context_param_name = p.name
|
||||
break
|
||||
|
||||
if context_param_name is not None:
|
||||
|
||||
def wrapper(ctx: Any = None, **kwargs: Any) -> Any:
|
||||
# ctx 是 dep_context,键为带前缀的依赖名;映射回原始键
|
||||
orig_ctx: dict[str, Any] = {}
|
||||
for k, v in (ctx or {}).items():
|
||||
orig_ctx[name_map.get(k, k)] = v
|
||||
# kwargs 中带前缀的依赖也映射回原参数名
|
||||
for k, v in kwargs.items():
|
||||
if k in name_map:
|
||||
orig_ctx[name_map[k]] = v
|
||||
return orig_fn(**{context_param_name: orig_ctx})
|
||||
|
||||
ctx_param = inspect.Parameter("ctx", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=Context)
|
||||
kw_param = inspect.Parameter("kwargs", inspect.Parameter.VAR_KEYWORD)
|
||||
wrapper.__signature__ = inspect.Signature( # type: ignore[attr-defined]
|
||||
parameters=[ctx_param, kw_param],
|
||||
return_annotation=orig_sig.return_annotation,
|
||||
)
|
||||
else:
|
||||
|
||||
def wrapper(**kwargs: Any) -> Any: # type: ignore[no-redef]
|
||||
orig_kwargs: dict[str, Any] = {}
|
||||
for k, v in kwargs.items():
|
||||
if k.startswith(prefix):
|
||||
orig_kwargs[k[len(prefix) :]] = v
|
||||
return orig_fn(**orig_kwargs)
|
||||
|
||||
kw_param = inspect.Parameter("kwargs", inspect.Parameter.VAR_KEYWORD)
|
||||
wrapper.__signature__ = inspect.Signature( # type: ignore[attr-defined]
|
||||
parameters=[kw_param],
|
||||
return_annotation=orig_sig.return_annotation,
|
||||
)
|
||||
|
||||
wrapper.__name__ = f"{ns}_{getattr(orig_fn, '__name__', 'fn')}"
|
||||
wrapper.__doc__ = getattr(orig_fn, "__doc__", None)
|
||||
return wrapper
|
||||
|
||||
|
||||
@dataclass
|
||||
class Graph:
|
||||
"""校验后的有向无环任务图。
|
||||
@@ -64,9 +147,15 @@ class Graph:
|
||||
specs: dict[str, TaskSpec[Any]] = field(default_factory=dict)
|
||||
deps: dict[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
defaults: GraphDefaults = field(default_factory=GraphDefaults)
|
||||
namespace: str | None = None
|
||||
|
||||
# 待解析的字符串引用列表(由 GraphComposer 消费);为空表示无引用。
|
||||
_pending_refs: list[str] = field(default_factory=list)
|
||||
|
||||
# resolved_spec 缓存:避免执行期每个任务多次重复 dataclasses.replace 判断。
|
||||
# 在 specs / defaults 变更时失效。
|
||||
_resolved_cache: dict[str, TaskSpec[Any]] = field(default_factory=dict)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 构建
|
||||
# ------------------------------------------------------------------ #
|
||||
@@ -76,18 +165,43 @@ class Graph:
|
||||
self._validate_references()
|
||||
return self
|
||||
|
||||
def chain(self, *specs: TaskSpec[Any]) -> Graph:
|
||||
"""链式注册任务:每个 spec 自动依赖前一个。
|
||||
|
||||
``chain(a, b, c)`` 等价于 ``b`` 依赖 ``a``,``c`` 依赖 ``b``。
|
||||
若 spec 已带 ``depends_on``,则前驱名追加到现有依赖前。
|
||||
返回 ``self`` 支持链式调用。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> graph = px.Graph().chain(extract, transform, load)
|
||||
"""
|
||||
prev_name: str | None = None
|
||||
for s in specs:
|
||||
current = s
|
||||
if prev_name is not None:
|
||||
# 将前驱追加到 depends_on 最前(保持显式依赖优先)
|
||||
new_deps = (prev_name, *s.depends_on) if prev_name not in s.depends_on else s.depends_on
|
||||
current = replace(s, depends_on=new_deps)
|
||||
self.add(current)
|
||||
prev_name = current.name
|
||||
return self
|
||||
|
||||
def _register(self, spec: TaskSpec[Any]) -> None:
|
||||
if spec.name in self.specs:
|
||||
raise DuplicateTaskError(spec.name)
|
||||
self.specs[spec.name] = spec
|
||||
# 拓扑依赖仅含硬依赖;软依赖仅用于注入,不影响分层。
|
||||
self.deps[spec.name] = spec.depends_on
|
||||
self._resolved_cache.clear()
|
||||
|
||||
@classmethod
|
||||
def from_specs(
|
||||
cls,
|
||||
specs: Iterable[TaskSpec[Any] | str],
|
||||
defaults: GraphDefaults | None = None,
|
||||
*,
|
||||
namespace: str | None = None,
|
||||
) -> Graph:
|
||||
"""从可迭代的 task spec 构建图。
|
||||
|
||||
@@ -100,10 +214,11 @@ class Graph:
|
||||
TaskSpec 对象或字符串引用的列表。
|
||||
defaults:
|
||||
图级默认值。``None`` 使用空 :class:`GraphDefaults`。
|
||||
namespace:
|
||||
可选命名空间,用于 :meth:`add_subgraph` 合并时加前缀。
|
||||
"""
|
||||
graph = cls(defaults=defaults or GraphDefaults())
|
||||
graph = cls(defaults=defaults or GraphDefaults(), namespace=namespace)
|
||||
pending_refs: list[str] = []
|
||||
|
||||
for spec in specs:
|
||||
if isinstance(spec, str):
|
||||
pending_refs.append(spec)
|
||||
@@ -119,6 +234,46 @@ class Graph:
|
||||
graph.validate()
|
||||
return graph
|
||||
|
||||
def add_subgraph(self, sub: Graph, *, namespace: str | None = None) -> Graph:
|
||||
"""将子图合并到当前图,任务名加命名空间前缀避免冲突。
|
||||
|
||||
参数
|
||||
----
|
||||
sub:
|
||||
待合并的子图。
|
||||
namespace:
|
||||
命名空间前缀。``None`` 时使用 ``sub.namespace``,若子图也无命名空间
|
||||
则抛出 ``ValueError``。最终任务名为 ``f"{ns}:{original_name}"``。
|
||||
|
||||
合并后,子图内任务的依赖名也会被加前缀;与子图外部任务的依赖保持原样。
|
||||
|
||||
返回 ``self`` 支持链式调用。
|
||||
"""
|
||||
ns = namespace or sub.namespace
|
||||
if not ns:
|
||||
raise ValueError("add_subgraph 需要 namespace 或子图自带 namespace")
|
||||
|
||||
def _rename(name: str) -> str:
|
||||
# 仅对子图内部任务名加前缀;外部依赖保持原样
|
||||
return f"{ns}:{name}" if name in sub.specs else name
|
||||
|
||||
sub_names = set(sub.specs.keys())
|
||||
for spec in sub.specs.values():
|
||||
# 子图内部依赖名需加前缀,对应的 fn 参数也需包装
|
||||
internal_deps = (set(spec.depends_on) | set(spec.soft_depends_on)) & sub_names
|
||||
new_fn = _make_namespaced_fn(spec.fn, ns, internal_deps) if spec.fn else spec.fn
|
||||
new_spec = replace(
|
||||
spec,
|
||||
name=_rename(spec.name),
|
||||
fn=new_fn,
|
||||
depends_on=tuple(_rename(d) for d in spec.depends_on),
|
||||
soft_depends_on=tuple(_rename(d) for d in spec.soft_depends_on),
|
||||
)
|
||||
self._register(new_spec)
|
||||
self._validate_references()
|
||||
self.validate()
|
||||
return self
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 校验
|
||||
# ------------------------------------------------------------------ #
|
||||
@@ -160,7 +315,12 @@ class Graph:
|
||||
对于 ``retry``/``timeout``/``strategy``/``env``/``cwd`` 等可空
|
||||
字段,若 spec 字段为默认空值且图级默认值非空,则用
|
||||
:func:`dataclasses.replace` 生成带默认值的副本。
|
||||
|
||||
结果按 ``name`` 缓存;specs / defaults 变更时缓存失效。
|
||||
"""
|
||||
cached = self._resolved_cache.get(name)
|
||||
if cached is not None:
|
||||
return cached
|
||||
spec = self.specs[name]
|
||||
d = self.defaults
|
||||
overrides: dict[str, Any] = {}
|
||||
@@ -184,9 +344,9 @@ class Graph:
|
||||
overrides["verbose"] = True
|
||||
if not spec.tags and d.tags:
|
||||
overrides["tags"] = d.tags
|
||||
if not overrides:
|
||||
return spec
|
||||
return replace(spec, **overrides)
|
||||
resolved = spec if not overrides else replace(spec, **overrides)
|
||||
self._resolved_cache[name] = resolved
|
||||
return resolved
|
||||
|
||||
def dependencies(self, name: str) -> tuple[str, ...]:
|
||||
"""``name`` 的直接硬依赖前驱。"""
|
||||
@@ -206,8 +366,11 @@ class Graph:
|
||||
|
||||
同层任务无相互硬依赖,可并发执行。软依赖不参与分层。
|
||||
层按执行顺序返回。图有环时抛出 :class:`CycleError`。
|
||||
|
||||
.. note::
|
||||
本方法假定图已通过 :meth:`validate` 校验(由 :func:`pyflowx.run`
|
||||
在入口统一执行一次)。若直接调用本方法,需自行先校验。
|
||||
"""
|
||||
self.validate()
|
||||
sorter = _TopologicalSorter(self.deps)
|
||||
result: list[list[str]] = []
|
||||
sorter.prepare()
|
||||
@@ -225,16 +388,13 @@ class Graph:
|
||||
def subgraph(self, tags: Iterable[str]) -> Graph:
|
||||
"""返回仅包含匹配任意标签的任务的新图。依赖边被修剪。"""
|
||||
wanted: set[str] = set(tags)
|
||||
kept: list[TaskSpec[Any]] = []
|
||||
for spec in self.specs.values():
|
||||
if wanted & set(spec.tags):
|
||||
pruned_deps = tuple(
|
||||
d for d in spec.depends_on if d in self.specs and (wanted & set(self.specs[d].tags))
|
||||
)
|
||||
pruned_soft = tuple(
|
||||
d for d in spec.soft_depends_on if d in self.specs and (wanted & set(self.specs[d].tags))
|
||||
)
|
||||
kept.append(replace(spec, depends_on=pruned_deps, soft_depends_on=pruned_soft))
|
||||
|
||||
def _dep_kept(dep: str) -> bool:
|
||||
return dep in self.specs and bool(wanted & set(self.specs[dep].tags))
|
||||
|
||||
kept: list[TaskSpec[Any]] = [
|
||||
_prune_deps(spec, _dep_kept) for spec in self.specs.values() if wanted & set(spec.tags)
|
||||
]
|
||||
return Graph.from_specs(kept, defaults=self.defaults)
|
||||
|
||||
def subgraph_by_names(self, names: Iterable[str]) -> Graph:
|
||||
@@ -243,12 +403,9 @@ class Graph:
|
||||
for n in wanted:
|
||||
if n not in self.specs:
|
||||
raise KeyError(f"Unknown task name: {n!r}")
|
||||
kept: list[TaskSpec[Any]] = []
|
||||
for spec in self.specs.values():
|
||||
if spec.name in wanted:
|
||||
pruned_deps = tuple(d for d in spec.depends_on if d in wanted)
|
||||
pruned_soft = tuple(d for d in spec.soft_depends_on if d in wanted)
|
||||
kept.append(replace(spec, depends_on=pruned_deps, soft_depends_on=pruned_soft))
|
||||
kept: list[TaskSpec[Any]] = [
|
||||
_prune_deps(spec, lambda d: d in wanted) for spec in self.specs.values() if spec.name in wanted
|
||||
]
|
||||
return Graph.from_specs(kept, defaults=self.defaults)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
@@ -346,102 +503,3 @@ class Graph:
|
||||
|
||||
def __contains__(self, name: Any) -> bool:
|
||||
return name in self.specs
|
||||
|
||||
|
||||
class GraphComposer:
|
||||
"""将带字符串引用的图展开为纯 :class:`TaskSpec` 图。
|
||||
|
||||
引用格式:
|
||||
* ``"command_name"`` —— 引用整个命令图。
|
||||
* ``"command_name.task_name"`` —— 引用特定任务。
|
||||
|
||||
引用按顺序展开,后续引用的任务依赖前面引用的最后一个任务;
|
||||
原始 ``TaskSpec`` 之间也按出现顺序串行依赖。
|
||||
"""
|
||||
|
||||
def __init__(self, graphs: dict[str, Graph]) -> None:
|
||||
self.graphs = graphs
|
||||
|
||||
def resolve_all(self) -> dict[str, Graph]:
|
||||
"""解析所有图的字符串引用,返回展开后的新图映射。"""
|
||||
resolved: dict[str, Graph] = {}
|
||||
for cmd_name, graph in self.graphs.items():
|
||||
resolved[cmd_name] = self.expand_refs(graph, cmd_name)
|
||||
return resolved
|
||||
|
||||
def expand_refs(self, graph: Graph, current_cmd: str) -> Graph:
|
||||
"""展开图中的字符串引用。若无 ``_pending_refs``,原样返回。"""
|
||||
pending_refs = graph._pending_refs
|
||||
if not pending_refs:
|
||||
return graph
|
||||
|
||||
all_specs: list[TaskSpec[Any]] = []
|
||||
previous_ref_last_task: str | None = None
|
||||
|
||||
for ref in pending_refs:
|
||||
expanded_specs = self.parse_ref(ref, current_cmd)
|
||||
if previous_ref_last_task and expanded_specs:
|
||||
for i, task in enumerate(expanded_specs):
|
||||
if i == 0 or not task.depends_on:
|
||||
expanded_specs[i] = replace(task, depends_on=tuple({*task.depends_on, previous_ref_last_task}))
|
||||
if expanded_specs:
|
||||
previous_ref_last_task = expanded_specs[-1].name
|
||||
all_specs.extend(expanded_specs)
|
||||
|
||||
original_specs = list(graph.all_specs().values())
|
||||
if original_specs:
|
||||
if previous_ref_last_task:
|
||||
first = original_specs[0]
|
||||
all_specs.append(replace(first, depends_on=tuple({*first.depends_on, previous_ref_last_task})))
|
||||
else:
|
||||
all_specs.append(original_specs[0])
|
||||
for i in range(1, len(original_specs)):
|
||||
current_task = original_specs[i]
|
||||
previous_task_name = original_specs[i - 1].name
|
||||
all_specs.append(
|
||||
replace(current_task, depends_on=tuple({*current_task.depends_on, previous_task_name}))
|
||||
)
|
||||
|
||||
return Graph.from_specs(all_specs, defaults=graph.defaults)
|
||||
|
||||
def parse_ref(self, ref: str, current_cmd: str) -> list[TaskSpec[Any]]:
|
||||
"""解析单个字符串引用,返回对应的 TaskSpec 列表。"""
|
||||
if ref == current_cmd:
|
||||
raise ValueError(f"循环引用: 命令 '{current_cmd}' 引用了自己")
|
||||
|
||||
if "." in ref:
|
||||
cmd_name, task_name = ref.split(".", 1)
|
||||
if cmd_name not in self.graphs:
|
||||
raise ValueError(f"引用的命令 '{cmd_name}' 不存在")
|
||||
ref_graph = self.graphs[cmd_name]
|
||||
if task_name not in ref_graph.all_specs():
|
||||
raise ValueError(f"任务 '{task_name}' 不存在于命令 '{cmd_name}' 中")
|
||||
return [ref_graph.all_specs()[task_name]]
|
||||
else:
|
||||
cmd_name = ref
|
||||
if cmd_name not in self.graphs:
|
||||
raise ValueError(f"引用的命令 '{cmd_name}' 不存在")
|
||||
ref_graph = self.graphs[cmd_name]
|
||||
ref_graph = self.expand_refs(ref_graph, cmd_name)
|
||||
return list(ref_graph.all_specs().values())
|
||||
|
||||
|
||||
def compose(
|
||||
graphs: dict[str, Graph],
|
||||
) -> dict[str, Graph]:
|
||||
"""编程式解析多图的字符串引用,返回展开后的新图映射。
|
||||
|
||||
与 :class:`GraphComposer` 等价,但作为独立函数暴露,供不使用
|
||||
:class:`~pyflowx.runner.CliRunner` 的编程式用户调用。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> graphs = {
|
||||
... "build": px.Graph.from_specs([px.TaskSpec("b", cmd=["echo", "b"])]),
|
||||
... "all": px.Graph.from_specs(["build", px.TaskSpec("t", cmd=["echo", "t"])]),
|
||||
... }
|
||||
>>> resolved = px.compose(graphs)
|
||||
>>> "b" in resolved["all"].all_specs()
|
||||
True
|
||||
"""
|
||||
return GraphComposer(graphs).resolve_all()
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""ops 子包 - CLI 工具模块集合.
|
||||
|
||||
每个子模块使用 ``@px.tool`` 装饰器注册 CLI 工具, 由 ``pf`` 入口按需
|
||||
``importlib.import_module`` 触发注册. 子模块按工具名组织 (一文件一工具).
|
||||
|
||||
子模块
|
||||
------
|
||||
- :mod:`autofmt` —— 代码格式化 (ruff)
|
||||
- :mod:`bumpversion` —— 版本号管理
|
||||
- :mod:`clr` —— 清屏
|
||||
- :mod:`dockercmd` —— Docker 镜像登录
|
||||
- :mod:`envdev` —— 开发环境镜像源配置
|
||||
- :mod:`filedate` —— 文件日期修改
|
||||
- :mod:`filelevel` —— 文件等级
|
||||
- :mod:`folderback` —— 文件夹备份
|
||||
- :mod:`folderzip` —— 文件夹压缩
|
||||
- :mod:`gittool` —— Git 工具
|
||||
- :mod:`lscalc` —— LS-DYNA 计算
|
||||
- :mod:`msdownload` —— ModelScope 下载
|
||||
- :mod:`packtool` —— 打包工具
|
||||
- :mod:`pdftool` —— PDF 工具
|
||||
- :mod:`piptool` —— pip 工具
|
||||
- :mod:`pymake` —— Python 项目构建
|
||||
- :mod:`reseticoncache` —— 重置图标缓存
|
||||
- :mod:`screenshot` —— 截图
|
||||
- :mod:`sglang` —— SGLang 服务
|
||||
- :mod:`sshcopyid` —— SSH 公钥拷贝
|
||||
- :mod:`taskkill` —— 进程终止
|
||||
- :mod:`which` —— 命令查找
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__: list[str] = []
|
||||
@@ -1,25 +1,28 @@
|
||||
"""自动格式化工具模块.
|
||||
"""autofmt - 自动格式化工具.
|
||||
|
||||
提供 Python 代码自动格式化的常用功能封装,
|
||||
支持 docstring 自动生成、pyproject.toml 配置同步等功能.
|
||||
提供格式化代码/代码检查/自动添加文档/同步配置 子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
try:
|
||||
import tomllib # noqa: F401
|
||||
|
||||
HAS_TOMLLIB = True
|
||||
except ImportError:
|
||||
HAS_TOMLLIB = False
|
||||
|
||||
__all__ = [
|
||||
"IGNORE_PATTERNS",
|
||||
"add_docstring",
|
||||
"auto_add_docstrings",
|
||||
"fmt",
|
||||
"format_all",
|
||||
"format_with_ruff",
|
||||
"generate_module_docstring",
|
||||
"lint",
|
||||
"lint_with_ruff",
|
||||
"sync_pyproject_config",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
@@ -47,6 +50,32 @@ IGNORE_PATTERNS = [
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool("autofmt", subcommand="fmt", help="格式化代码")
|
||||
def fmt(target: str = ".") -> None:
|
||||
"""格式化代码 (ruff format).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
target : str
|
||||
目标路径 (默认: .)
|
||||
"""
|
||||
format_with_ruff(Path(target), fix=False)
|
||||
|
||||
|
||||
@px.tool("autofmt", subcommand="lint", help="代码检查")
|
||||
def lint(target: str = ".", fix: bool = False) -> None:
|
||||
"""代码检查 (ruff check).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
target : str
|
||||
目标路径 (默认: .)
|
||||
fix : bool
|
||||
自动修复问题
|
||||
"""
|
||||
lint_with_ruff(Path(target), fix=fix)
|
||||
|
||||
|
||||
def format_with_ruff(target: Path, fix: bool = True) -> None:
|
||||
"""使用 ruff 格式化代码.
|
||||
|
||||
@@ -102,12 +131,10 @@ def add_docstring(file_path: Path, docstring: str) -> bool:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
tree = ast.parse(content)
|
||||
|
||||
# 检查是否已有 docstring
|
||||
first_node = tree.body[0] if tree.body else None
|
||||
if first_node and isinstance(first_node, ast.Expr) and isinstance(first_node.value, ast.Constant):
|
||||
return False
|
||||
|
||||
# 添加 docstring
|
||||
lines = content.splitlines()
|
||||
doc_lines = docstring.splitlines()
|
||||
doc_lines.append("")
|
||||
@@ -138,7 +165,6 @@ def generate_module_docstring(file_path: Path) -> str:
|
||||
stem = file_path.stem
|
||||
parent = file_path.parent.name
|
||||
|
||||
# 关键词匹配
|
||||
keywords = {
|
||||
"cli": f"Command-line interface for {parent}",
|
||||
"gui": f"Graphical user interface for {parent}",
|
||||
@@ -155,13 +181,14 @@ def generate_module_docstring(file_path: Path) -> str:
|
||||
return f'"""{stem.replace("_", " ").title()} module."""'
|
||||
|
||||
|
||||
def auto_add_docstrings(root_dir: Path) -> int:
|
||||
@px.tool("autofmt", subcommand="doc", help="自动添加文档字符串")
|
||||
def auto_add_docstrings(root_dir: Path = Path()) -> int:
|
||||
"""自动为所有 Python 文件添加 docstring.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root_dir : Path
|
||||
根目录
|
||||
根目录 (默认: 当前目录)
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -170,7 +197,6 @@ def auto_add_docstrings(root_dir: Path) -> int:
|
||||
"""
|
||||
count = 0
|
||||
for py_file in root_dir.rglob("*.py"):
|
||||
# 跳过忽略的文件
|
||||
if any(pattern in str(py_file) for pattern in IGNORE_PATTERNS):
|
||||
continue
|
||||
|
||||
@@ -182,20 +208,20 @@ def auto_add_docstrings(root_dir: Path) -> int:
|
||||
return count
|
||||
|
||||
|
||||
def sync_pyproject_config(root_dir: Path) -> None:
|
||||
@px.tool("autofmt", subcommand="sync", help="同步 pyproject 配置")
|
||||
def sync_pyproject_config(root_dir: Path = Path()) -> None:
|
||||
"""同步 pyproject.toml 配置到子项目.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root_dir : Path
|
||||
根目录
|
||||
根目录 (默认: 当前目录)
|
||||
"""
|
||||
main_toml = root_dir / "pyproject.toml"
|
||||
if not main_toml.exists():
|
||||
print(f"主项目配置文件不存在: {main_toml}")
|
||||
return
|
||||
|
||||
# 查找所有子项目的 pyproject.toml
|
||||
sub_tomls = [p for p in root_dir.rglob("pyproject.toml") if p != main_toml and ".venv" not in str(p)]
|
||||
|
||||
if not sub_tomls:
|
||||
@@ -204,7 +230,6 @@ def sync_pyproject_config(root_dir: Path) -> None:
|
||||
|
||||
print(f"找到 {len(sub_tomls)} 个子项目配置文件")
|
||||
|
||||
# 对每个子项目调用 ruff format
|
||||
for sub_toml in sub_tomls:
|
||||
subprocess.run(["ruff", "format", str(sub_toml)], check=False)
|
||||
|
||||
@@ -219,64 +244,6 @@ def format_all(root_dir: Path) -> None:
|
||||
root_dir : Path
|
||||
根目录
|
||||
"""
|
||||
# 使用 ruff format
|
||||
subprocess.run(["ruff", "format", str(root_dir)], check=True)
|
||||
|
||||
# 使用 ruff check
|
||||
subprocess.run(["ruff", "check", "--fix", "--unsafe-fixes", str(root_dir)], check=True)
|
||||
|
||||
print(f"格式化完成: {root_dir}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""自动格式化工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="AutoFmt - 自动格式化工具",
|
||||
usage="autofmt <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# ruff format 命令
|
||||
format_parser = subparsers.add_parser("fmt", help="使用 ruff 格式化代码")
|
||||
format_parser.add_argument("--target", type=str, default=".", help="目标路径")
|
||||
|
||||
# ruff check 命令
|
||||
lint_parser = subparsers.add_parser("lint", help="使用 ruff 检查代码")
|
||||
lint_parser.add_argument("--target", type=str, default=".", help="目标路径")
|
||||
lint_parser.add_argument("--fix", action="store_true", help="自动修复")
|
||||
|
||||
# 自动添加 docstring 命令
|
||||
doc_parser = subparsers.add_parser("doc", help="自动添加 docstring")
|
||||
doc_parser.add_argument("--root-dir", type=str, default=".", help="根目录")
|
||||
|
||||
# 同步配置命令
|
||||
sync_parser = subparsers.add_parser("sync", help="同步 pyproject.toml 配置")
|
||||
sync_parser.add_argument("--root-dir", type=str, default=".", help="根目录")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "fmt":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("ruff_format", cmd=["ruff", "format", args.target], verbose=True)])
|
||||
elif args.command == "lint":
|
||||
cmd = ["ruff", "check", args.target]
|
||||
if args.fix:
|
||||
cmd.extend(["--fix", "--unsafe-fixes"])
|
||||
graph = px.Graph.from_specs([px.TaskSpec("ruff_check", cmd=cmd, verbose=True)])
|
||||
elif args.command == "doc":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("auto_docstring", fn=auto_add_docstrings, args=(Path(args.root_dir),), verbose=True)
|
||||
])
|
||||
elif args.command == "sync":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec("sync_config", fn=sync_pyproject_config, args=(Path(args.root_dir),), verbose=True)
|
||||
])
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,232 @@
|
||||
"""版本号管理模块.
|
||||
|
||||
提供单文件版本号更新 (``bump_file_version``) 与项目级批量版本号同步
|
||||
(``bump_project_version``) 能力. ``bump_project_version`` 通过 ``@px.tool``
|
||||
注册为 CLI 工具, 可通过 ``pf bumpversion`` 调用.
|
||||
|
||||
设计要点
|
||||
--------
|
||||
``bump_project_version`` 采用 "先读取基准、再统一写入" 的两阶段策略:
|
||||
先扫描所有 ``__init__.py`` / ``pyproject.toml`` 文件, 读取各自的版本号,
|
||||
取最大值作为基准版本计算新版本号, 然后把新版本号统一写入所有文件,
|
||||
避免文件间版本号不同步导致的跳号问题.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"BumpVersionType",
|
||||
"bump_file_version",
|
||||
"bump_project_version",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
BumpVersionType = Literal["patch", "minor", "major"]
|
||||
|
||||
_PYPROJECT_VERSION_PATTERN = re.compile(
|
||||
r'(?:^|\n)\s*version\s*=\s*["\']'
|
||||
r"(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
|
||||
r"(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
|
||||
r"(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?"
|
||||
r'["\']',
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
_INIT_VERSION_PATTERN = re.compile(
|
||||
r'(?:^|\n)\s*__version__\s*=\s*["\']'
|
||||
r"(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
|
||||
r"(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
|
||||
r"(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?"
|
||||
r'["\']',
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
_IGNORE_DIRS = frozenset({".venv", "venv", ".git", "__pycache__", ".tox", "node_modules", "build", "dist", ".eggs"})
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 私有辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _get_pattern_for_file(file_name: str) -> re.Pattern[str] | None:
|
||||
"""根据文件类型获取对应的正则表达式."""
|
||||
if file_name == "pyproject.toml":
|
||||
return _PYPROJECT_VERSION_PATTERN
|
||||
if file_name == "__init__.py":
|
||||
return _INIT_VERSION_PATTERN
|
||||
return None
|
||||
|
||||
|
||||
def _calculate_new_version(major: int, minor: int, patch: int, part: BumpVersionType) -> str:
|
||||
"""计算新版本号."""
|
||||
if part == "major":
|
||||
return f"{major + 1}.0.0"
|
||||
if part == "minor":
|
||||
return f"{major}.{minor + 1}.0"
|
||||
return f"{major}.{minor}.{patch + 1}"
|
||||
|
||||
|
||||
def _build_replacement_string(original_match: str, new_version: str, file_name: str) -> str:
|
||||
"""构建替换字符串, 保留原始格式."""
|
||||
quote_char = '"' if '"' in original_match else "'"
|
||||
key = "__version__" if file_name == "__init__.py" else "version"
|
||||
prefix_match = re.match(rf"(\s*{key}\s*=\s*)[\"']", original_match)
|
||||
prefix = prefix_match.group(1) if prefix_match else f"{key} = "
|
||||
return f"{prefix}{quote_char}{new_version}{quote_char}"
|
||||
|
||||
|
||||
def _read_version_tuple(file_path: Path) -> tuple[int, int, int] | None:
|
||||
"""从文件中读取版本号, 返回 (major, minor, patch) 元组; 未找到返回 None.
|
||||
|
||||
读取失败时抛出 ``OSError`` / ``UnicodeDecodeError`` 由调用方处理.
|
||||
"""
|
||||
pattern = _get_pattern_for_file(file_path.name)
|
||||
if pattern is None:
|
||||
return None
|
||||
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
match = pattern.search(content)
|
||||
if not match:
|
||||
return None
|
||||
|
||||
return int(match.group("major")), int(match.group("minor")), int(match.group("patch"))
|
||||
|
||||
|
||||
def _write_version_to_file(file_path: Path, new_version: str) -> bool:
|
||||
"""把新版本号写入指定文件; 成功返回 True, 未匹配到版本号返回 False."""
|
||||
pattern = _get_pattern_for_file(file_path.name)
|
||||
if pattern is None: # pragma: no cover - 调用方已保证 pattern 不为 None
|
||||
return False
|
||||
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
match = pattern.search(content)
|
||||
if not match: # pragma: no cover - 调用方已通过 _read_version_tuple 验证
|
||||
return False
|
||||
|
||||
replacement = _build_replacement_string(match.group(0), new_version, file_path.name)
|
||||
content = content.replace(match.group(0), replacement)
|
||||
|
||||
try:
|
||||
file_path.write_text(content, encoding="utf-8")
|
||||
except OSError as e:
|
||||
print(f"更新文件 {file_path} 版本号时出错: {e}")
|
||||
raise
|
||||
|
||||
return True
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 公共函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def bump_file_version(file_path: Path, part: BumpVersionType = "patch") -> str | None:
|
||||
"""更新单个文件中的版本号.
|
||||
|
||||
读取文件当前版本号, 按 ``part`` 指定的部分递增, 写回文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_path : Path
|
||||
要更新的文件路径 (``pyproject.toml`` 或 ``__init__.py``)
|
||||
part : BumpVersionType
|
||||
版本部分: patch, minor, major
|
||||
|
||||
Returns
|
||||
-------
|
||||
str | None
|
||||
更新后的新版本号; 文件中未找到版本号或读取失败时返回 None
|
||||
"""
|
||||
version_tuple = _read_version_tuple(file_path)
|
||||
if version_tuple is None:
|
||||
print(f"文件 {file_path} 中未找到版本号模式")
|
||||
return None
|
||||
|
||||
major, minor, patch = version_tuple
|
||||
new_version = _calculate_new_version(major, minor, patch, part)
|
||||
|
||||
if not _write_version_to_file(file_path, new_version): # pragma: no cover - _read_version_tuple 已验证
|
||||
return None
|
||||
|
||||
return new_version
|
||||
|
||||
|
||||
@px.tool("bumpversion", help="版本号自动管理")
|
||||
def bump_project_version(part: BumpVersionType = "patch", no_tag: bool = False) -> str | None:
|
||||
"""批量同步项目所有版本号文件并提交.
|
||||
|
||||
扫描当前目录下所有 ``__init__.py`` 和 ``pyproject.toml`` 文件
|
||||
(排除虚拟环境和缓存目录), 先读取每个文件的当前版本号取最大值作为基准,
|
||||
计算新版本号后统一写入所有文件, 最后执行 git add (按文件名) + commit + tag.
|
||||
|
||||
采用 "先读取基准、再统一写入" 的两阶段策略, 即使某些文件版本号不同步,
|
||||
也能在一次 bump 后重新对齐, 避免跳号.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
part : BumpVersionType
|
||||
版本部分: patch, minor, major
|
||||
no_tag : bool
|
||||
提交后不创建 git tag
|
||||
|
||||
Returns
|
||||
-------
|
||||
str | None
|
||||
更新后的新版本号; 未找到版本号文件时返回 None
|
||||
"""
|
||||
all_files: set[Path] = set()
|
||||
for pattern in ("__init__.py", "pyproject.toml"):
|
||||
for file in Path.cwd().rglob(pattern):
|
||||
if not any(ignore_dir in file.parts for ignore_dir in _IGNORE_DIRS):
|
||||
all_files.add(file)
|
||||
|
||||
if not all_files:
|
||||
print("未找到包含版本号的文件")
|
||||
return None
|
||||
|
||||
print(f"找到 {len(all_files)} 个文件需要更新版本号")
|
||||
cwd = Path.cwd()
|
||||
for file in sorted(all_files):
|
||||
print(f" - {file.relative_to(cwd)}")
|
||||
|
||||
# 阶段 1: 读取所有文件版本号, 取最大值作为基准
|
||||
versions: list[tuple[int, int, int]] = []
|
||||
for file in sorted(all_files):
|
||||
v = _read_version_tuple(file)
|
||||
if v is not None:
|
||||
versions.append(v)
|
||||
|
||||
if not versions:
|
||||
print("未能从任何文件读取版本号")
|
||||
return None
|
||||
|
||||
major, minor, patch = max(versions)
|
||||
new_version = _calculate_new_version(major, minor, patch, part)
|
||||
print(f"基准版本: {major}.{minor}.{patch} -> 新版本: {new_version}")
|
||||
|
||||
# 阶段 2: 统一写入新版本号到所有文件
|
||||
for file in sorted(all_files):
|
||||
_write_version_to_file(file, new_version)
|
||||
|
||||
# 阶段 3: git add (按文件名) + commit + tag
|
||||
relative_files = [str(file.relative_to(cwd)) for file in sorted(all_files)]
|
||||
subprocess.run(["git", "add", *relative_files], check=True)
|
||||
subprocess.run(["git", "commit", "-m", f"bump version to {new_version}"], check=True)
|
||||
|
||||
if not no_tag:
|
||||
tag_name = f"v{new_version}"
|
||||
subprocess.run(["git", "tag", "-a", tag_name, "-m", f"Release {tag_name}"], check=True)
|
||||
print(f"已创建标签: {tag_name}")
|
||||
|
||||
return new_version
|
||||
@@ -0,0 +1,21 @@
|
||||
"""clr - 清屏工具.
|
||||
|
||||
跨平台清屏: Windows 调用 ``cls``, Linux/macOS 调用 ``clear``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
@px.tool("clr", help="清屏 (跨平台)")
|
||||
def clear_screen_run() -> None:
|
||||
"""清屏 (跨平台).
|
||||
|
||||
Windows 调用 ``cls``, Linux/macOS 调用 ``clear``.
|
||||
"""
|
||||
cmd = ["cls"] if Constants.IS_WINDOWS else ["clear"]
|
||||
subprocess.run(cmd, check=False)
|
||||
@@ -0,0 +1,27 @@
|
||||
"""dockercmd - Docker 镜像登录工具.
|
||||
|
||||
登录腾讯云 Docker 镜像仓库.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import getpass
|
||||
import subprocess
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
_DOCKER_MIRROR_TENCENT: str = "ccr.ccs.tencentyun.com"
|
||||
|
||||
|
||||
@px.tool("dockercmd", subcommand="login", help="登录腾讯云 Docker 镜像仓库")
|
||||
def docker_login_tencent(username: str = "") -> None:
|
||||
"""登录腾讯云 Docker 镜像仓库.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
username : str
|
||||
Docker 用户名 (为空时由 docker 交互式提示输入)
|
||||
"""
|
||||
user = username or getpass.getuser()
|
||||
subprocess.run(["docker", "login", "--username", user, _DOCKER_MIRROR_TENCENT], check=False)
|
||||
print(f"已尝试登录腾讯云镜像仓库 (用户: {user})")
|
||||
@@ -0,0 +1,421 @@
|
||||
"""envdev - 开发环境镜像源配置工具.
|
||||
|
||||
配置 Python / Conda / Rust 镜像源 (Linux 还会安装 Qt 库、中文字体、Docker).
|
||||
所有镜像源参数互不影响, 可单独使用.
|
||||
Linux 专用操作 (系统镜像/Qt/字体/Docker) 在非 Linux 平台上由函数内部跳过.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import getpass
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
__all__ = [
|
||||
"download_rustup_script",
|
||||
"install_linux_docker",
|
||||
"install_linux_fonts",
|
||||
"install_linux_qt_libs",
|
||||
"install_rust_toolchain",
|
||||
"setup_conda_mirror",
|
||||
"setup_linux_system_mirror",
|
||||
"setup_python_mirror",
|
||||
"setup_rust_mirror",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# 配置常量
|
||||
# ============================================================================
|
||||
|
||||
PyMirrorType = Literal["tsinghua", "aliyun", "huaweicloud", "ustc", "zju"]
|
||||
CondaMirrorType = Literal["tsinghua", "ustc", "bsfu", "aliyun"]
|
||||
RustMirrorType = Literal["tsinghua", "ustc", "aliyun"]
|
||||
|
||||
_PIP_INDEX_URLS: dict[str, str] = {
|
||||
"tsinghua": "https://pypi.tuna.tsinghua.edu.cn/simple",
|
||||
"aliyun": "https://mirrors.aliyun.com/pypi/simple/",
|
||||
"huaweicloud": "https://mirrors.huaweicloud.com/repository/pypi/simple/",
|
||||
"ustc": "https://pypi.mirrors.ustc.edu.cn/simple/",
|
||||
"zju": "https://mirrors.zju.edu.cn/pypi/simple/",
|
||||
}
|
||||
|
||||
_PIP_TRUSTED_HOSTS: dict[str, str] = {
|
||||
"tsinghua": "pypi.tuna.tsinghua.edu.cn",
|
||||
"aliyun": "mirrors.aliyun.com",
|
||||
"huaweicloud": "mirrors.huaweicloud.com",
|
||||
"ustc": "pypi.mirrors.ustc.edu.cn",
|
||||
"zju": "mirrors.zju.edu.cn",
|
||||
}
|
||||
|
||||
_UV_PYTHON_INSTALL_MIRROR: str = "https://registry.npmmirror.com/-/binary/python-build-standalone"
|
||||
|
||||
_CONDA_MIRROR_URLS: dict[str, list[str]] = {
|
||||
"tsinghua": [
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/pro/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/bioconda/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/menpo/",
|
||||
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/pytorch/",
|
||||
],
|
||||
"ustc": [
|
||||
"https://mirrors.ustc.edu.cn/anaconda/pkgs/main/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/pkgs/free/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/pkgs/r/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/pkgs/msys2/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/pkgs/pro/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/pkgs/dev/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/cloud/conda-forge/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/cloud/bioconda/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/cloud/menpo/",
|
||||
"https://mirrors.ustc.edu.cn/anaconda/cloud/pytorch/",
|
||||
],
|
||||
"bsfu": [
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/pkgs/main/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/pkgs/free/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/pkgs/r/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/pkgs/msys2/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/pkgs/pro/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/pkgs/dev/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/cloud/conda-forge/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/cloud/bioconda/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/cloud/menpo/",
|
||||
"https://mirrors.bsfu.edu.cn/anaconda/cloud/pytorch/",
|
||||
],
|
||||
"aliyun": [
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/main/",
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/free/",
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/r/",
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/msys2/",
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/pro/",
|
||||
"https://mirrors.aliyun.com/anaconda/pkgs/dev/",
|
||||
"https://mirrors.aliyun.com/anaconda/cloud/conda-forge/",
|
||||
"https://mirrors.aliyun.com/anaconda/cloud/bioconda/",
|
||||
"https://mirrors.aliyun.com/anaconda/cloud/menpo/",
|
||||
"https://mirrors.aliyun.com/anaconda/cloud/pytorch/",
|
||||
],
|
||||
}
|
||||
|
||||
_RUSTUP_MIRRORS: dict[str, dict[str, str]] = {
|
||||
"tsinghua": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.tuna.tsinghua.edu.cn/rustup",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.tuna.tsinghua.edu.cn/rustup/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.tuna.tsinghua.edu.cn/crates.io-index/",
|
||||
},
|
||||
"aliyun": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.aliyun.com/rustup",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.aliyun.com/rustup/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.aliyun.com/crates.io-index/",
|
||||
},
|
||||
"ustc": {
|
||||
"RUSTUP_DIST_SERVER": "https://mirrors.ustc.edu.cn/rust-static",
|
||||
"RUSTUP_UPDATE_ROOT": "https://mirrors.ustc.edu.cn/rust-static/rustup",
|
||||
"TOML_REGISTRY": "https://mirrors.ustc.edu.cn/crates.io-index/",
|
||||
},
|
||||
}
|
||||
|
||||
_RUST_SCCACHE_DIR: Path = Path.home() / ".cargo" / "sccache"
|
||||
_RUST_SCCACHE_CACHE_SIZE: str = "20G"
|
||||
|
||||
_QT_LIBS: list[str] = [
|
||||
"build-essential",
|
||||
"libgl1",
|
||||
"libegl1",
|
||||
"libglib2.0-0",
|
||||
"libfontconfig1",
|
||||
"libfreetype6",
|
||||
"libxkbcommon0",
|
||||
"libdbus-1-3",
|
||||
"libxcb-xinerama0",
|
||||
"libxcb-icccm4",
|
||||
"libxcb-image0",
|
||||
"libxcb-keysyms1",
|
||||
"libxcb-randr0",
|
||||
"libxcb-render-util0",
|
||||
"libxcb-shape0",
|
||||
"libxcb-xfixes0",
|
||||
"libxcb-cursor0",
|
||||
]
|
||||
|
||||
_CHINESE_FONTS: list[str] = [
|
||||
"fonts-noto-cjk",
|
||||
"fonts-wqy-microhei",
|
||||
"fonts-wqy-zenhei",
|
||||
"fonts-noto-color-emoji",
|
||||
]
|
||||
|
||||
_DOWNLOAD_MIRROR_SCRIPT: str = "curl -sSL https://linuxmirrors.cn/main.sh -o /tmp/linuxmirrors.sh"
|
||||
_INSTALL_MIRROR_SCRIPT: str = "sudo bash /tmp/linuxmirrors.sh"
|
||||
|
||||
_RUSTUP_DOWNLOAD_URL_LINUX: str = "https://mirrors.aliyun.com/repo/rust/rustup-init.sh"
|
||||
_RUSTUP_DOWNLOAD_URL_WINDOWS: str = "https://static.rust-lang.org/rustup/dist/x86_64-pc-windows-msvc/rustup-init.exe"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 私有辅助
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _pip_config_path() -> Path:
|
||||
"""返回当前平台的 pip 配置文件路径."""
|
||||
if Constants.IS_LINUX:
|
||||
return Path.home() / ".pip" / "pip.conf"
|
||||
return Path.home() / "pip" / "pip.ini"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 镜像源配置函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool("envdev", subcommand="setup-python", help="配置 Python 镜像源")
|
||||
def setup_python_mirror(mirror: str = "tsinghua") -> None:
|
||||
"""配置 Python 镜像源 (设置环境变量 + 写入 pip 配置文件).
|
||||
|
||||
设置 ``PIP_INDEX_URL`` / ``PIP_TRUSTED_HOSTS`` / ``UV_INDEX_URL`` /
|
||||
``UV_PYTHON_INSTALL_MIRROR`` 等环境变量, 并写入 pip 配置文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mirror : str
|
||||
镜像源名称: tsinghua/aliyun/huaweicloud/ustc/zju (默认: tsinghua)
|
||||
"""
|
||||
if mirror not in _PIP_INDEX_URLS:
|
||||
print(f"未知 Python 镜像源: {mirror}")
|
||||
return
|
||||
|
||||
index_url = _PIP_INDEX_URLS[mirror]
|
||||
trusted_host = _PIP_TRUSTED_HOSTS[mirror]
|
||||
|
||||
os.environ["PIP_INDEX_URL"] = index_url
|
||||
os.environ["PIP_TRUSTED_HOSTS"] = trusted_host
|
||||
os.environ["UV_INDEX_URL"] = index_url
|
||||
os.environ["UV_PYTHON_INSTALL_MIRROR"] = _UV_PYTHON_INSTALL_MIRROR
|
||||
os.environ["UV_HTTP_TIMEOUT"] = "600"
|
||||
os.environ["UV_LINK_MODE"] = "copy"
|
||||
|
||||
config_path = _pip_config_path()
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
content = f"[global]\nindex-url = {index_url}\ntrusted-host = {trusted_host}\n"
|
||||
config_path.write_text(content, encoding="utf-8")
|
||||
print(f"Python 镜像源已配置: {mirror} -> {config_path}")
|
||||
|
||||
|
||||
@px.tool("envdev", subcommand="setup-conda", help="配置 Conda 镜像源")
|
||||
def setup_conda_mirror(mirror: str = "tsinghua") -> None:
|
||||
"""配置 Conda 镜像源 (写入 ~/.condarc).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mirror : str
|
||||
镜像源名称: tsinghua/ustc/bsfu/aliyun (默认: tsinghua)
|
||||
"""
|
||||
if mirror not in _CONDA_MIRROR_URLS:
|
||||
print(f"未知 Conda 镜像源: {mirror}")
|
||||
return
|
||||
|
||||
urls = _CONDA_MIRROR_URLS[mirror]
|
||||
config_path = Path.home() / ".condarc"
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
content = "show_channel_urls: true\nchannels:\n - " + "\n - ".join(urls) + "\n - defaults\n"
|
||||
config_path.write_text(content, encoding="utf-8")
|
||||
print(f"Conda 镜像源已配置: {mirror} -> {config_path}")
|
||||
|
||||
|
||||
@px.tool("envdev", subcommand="setup-rust", help="配置 Rust 镜像源")
|
||||
def setup_rust_mirror(mirror: str = "tsinghua", version: str = "stable") -> None:
|
||||
"""配置 Rust 镜像源 (设置环境变量 + 写入 cargo config + 创建 sccache 目录).
|
||||
|
||||
设置 ``RUSTUP_DIST_SERVER`` / ``RUSTUP_UPDATE_ROOT`` / ``RUST_SCCACHE_DIR``
|
||||
等环境变量, 写入 ``~/.cargo/config.toml``, 并创建 sccache 缓存目录.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mirror : str
|
||||
镜像源名称: tsinghua/ustc/aliyun (默认: tsinghua)
|
||||
version : str
|
||||
Rust 版本 (未使用, 保留以与原 envdev 参数对齐)
|
||||
"""
|
||||
del version # 兼容旧参数, 实际安装由独立 job 处理
|
||||
|
||||
if mirror not in _RUSTUP_MIRRORS:
|
||||
print(f"未知 Rust 镜像源: {mirror}")
|
||||
return
|
||||
|
||||
mirrors = _RUSTUP_MIRRORS[mirror]
|
||||
os.environ["RUSTUP_DIST_SERVER"] = mirrors["RUSTUP_DIST_SERVER"]
|
||||
os.environ["RUSTUP_UPDATE_ROOT"] = mirrors["RUSTUP_UPDATE_ROOT"]
|
||||
os.environ["RUST_SCCACHE_DIR"] = str(_RUST_SCCACHE_DIR)
|
||||
os.environ["RUST_SCCACHE_CACHE_SIZE"] = _RUST_SCCACHE_CACHE_SIZE
|
||||
|
||||
_RUST_SCCACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
config_path = Path.home() / ".cargo" / "config.toml"
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
registry = mirrors["TOML_REGISTRY"]
|
||||
content = (
|
||||
f"\n[source.crates-io]\nreplace-with = '{mirror}'\n\n"
|
||||
f'[source.{mirror}]\nregistry = "sparse+{registry}"\n\n'
|
||||
f'[registries.{mirror}]\nindex = "sparse+{registry}"\n'
|
||||
)
|
||||
config_path.write_text(content, encoding="utf-8")
|
||||
print(f"Rust 镜像源已配置: {mirror} -> {config_path}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Rust 工具链安装
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool("envdev", subcommand="download-rustup", help="下载 Rustup 安装脚本")
|
||||
def download_rustup_script() -> None:
|
||||
"""下载 Rustup 安装脚本 (跨平台, 已安装 rustup 时跳过).
|
||||
|
||||
Linux 下载 ``rustup-init.sh``, Windows 下载 ``rustup-init.exe``.
|
||||
"""
|
||||
if shutil.which("rustup") is not None:
|
||||
print("rustup 已安装, 跳过下载")
|
||||
return
|
||||
|
||||
if Constants.IS_WINDOWS:
|
||||
print("下载 rustup-init.exe...")
|
||||
subprocess.run(
|
||||
[
|
||||
"powershell",
|
||||
"-Command",
|
||||
"Invoke-WebRequest",
|
||||
"-Uri",
|
||||
_RUSTUP_DOWNLOAD_URL_WINDOWS,
|
||||
"-OutFile",
|
||||
"rustup-init.exe",
|
||||
],
|
||||
check=False,
|
||||
)
|
||||
else:
|
||||
print("下载 rustup-init.sh...")
|
||||
subprocess.run(
|
||||
["curl", "-fsSL", _RUSTUP_DOWNLOAD_URL_LINUX, "-o", "rustup-init.sh"],
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
@px.tool(
|
||||
"envdev",
|
||||
subcommand="install-rust",
|
||||
help="安装 Rust 工具链",
|
||||
needs=["setup-rust", "download-rustup"],
|
||||
allow_upstream_skip=True,
|
||||
)
|
||||
def install_rust_toolchain(version: str = "stable") -> None:
|
||||
"""安装 Rust 工具链 (rustup 未安装时跳过).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
version : str
|
||||
Rust 版本: ``stable`` / ``nightly`` / ``beta`` (默认: ``stable``)
|
||||
"""
|
||||
if shutil.which("rustup") is None:
|
||||
print("rustup 未安装, 跳过工具链安装")
|
||||
return
|
||||
|
||||
subprocess.run(["rustup", "toolchain", "install", version], check=False)
|
||||
print(f"Rust 工具链 {version} 安装完成")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Linux 专用函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool("envdev", subcommand="setup-linux-mirror", help="配置 Linux 系统镜像源")
|
||||
def setup_linux_system_mirror() -> None:
|
||||
"""下载并安装 Linux 系统镜像源 (仅 Linux, 已配置国内镜像时跳过).
|
||||
|
||||
检查 ``/etc/apt/sources.list`` 与 ``/etc/apt/sources.list.d/ubuntu.sources``
|
||||
是否已配置国内镜像, 已配置则跳过; 未配置则下载并执行 linuxmirrors 脚本.
|
||||
"""
|
||||
if not Constants.IS_LINUX:
|
||||
print("setup_linux_system_mirror: 仅在 Linux 上执行")
|
||||
return
|
||||
|
||||
apt_files = ["/etc/apt/sources.list", "/etc/apt/sources.list.d/ubuntu.sources"]
|
||||
mirror_keys = list(_PIP_INDEX_URLS.keys())
|
||||
already_configured = False
|
||||
for apt_file in apt_files:
|
||||
try:
|
||||
content = Path(apt_file).read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
if any(mirror in content for mirror in mirror_keys):
|
||||
already_configured = True
|
||||
break
|
||||
|
||||
if already_configured:
|
||||
print("已配置国内镜像源, 跳过系统镜像配置")
|
||||
return
|
||||
|
||||
print("下载 linuxmirrors 脚本...")
|
||||
subprocess.run(_DOWNLOAD_MIRROR_SCRIPT, shell=True, check=False)
|
||||
print("安装 linuxmirrors...")
|
||||
subprocess.run(_INSTALL_MIRROR_SCRIPT, shell=True, check=False)
|
||||
|
||||
|
||||
@px.tool(
|
||||
"envdev",
|
||||
subcommand="install-qt-libs",
|
||||
help="安装 Qt 依赖库",
|
||||
needs=["setup-linux-mirror"],
|
||||
allow_upstream_skip=True,
|
||||
)
|
||||
def install_linux_qt_libs() -> None:
|
||||
"""安装 Qt 依赖库 (仅 Linux)."""
|
||||
if not Constants.IS_LINUX:
|
||||
print("install_linux_qt_libs: 仅在 Linux 上执行")
|
||||
return
|
||||
|
||||
subprocess.run(["sudo", "apt", "install", "-y", *_QT_LIBS], check=False)
|
||||
print("Qt 依赖库安装完成")
|
||||
|
||||
|
||||
@px.tool(
|
||||
"envdev",
|
||||
subcommand="install-fonts",
|
||||
help="安装中文字体",
|
||||
needs=["setup-linux-mirror"],
|
||||
allow_upstream_skip=True,
|
||||
)
|
||||
def install_linux_fonts() -> None:
|
||||
"""安装中文字体 (仅 Linux)."""
|
||||
if not Constants.IS_LINUX:
|
||||
print("install_linux_fonts: 仅在 Linux 上执行")
|
||||
return
|
||||
|
||||
subprocess.run(["sudo", "apt", "install", "-y", *_CHINESE_FONTS], check=False)
|
||||
print("中文字体安装完成")
|
||||
|
||||
|
||||
@px.tool(
|
||||
"envdev",
|
||||
subcommand="install-docker",
|
||||
help="安装 Docker",
|
||||
needs=["setup-linux-mirror"],
|
||||
allow_upstream_skip=True,
|
||||
)
|
||||
def install_linux_docker() -> None:
|
||||
"""安装 Docker (仅 Linux)."""
|
||||
if not Constants.IS_LINUX:
|
||||
print("install_linux_docker: 仅在 Linux 上执行")
|
||||
return
|
||||
|
||||
subprocess.run(["sudo", "apt", "install", "-y", "docker-compose-v2"], check=False)
|
||||
subprocess.run(["sudo", "usermod", "-aG", "docker", getpass.getuser()], check=False)
|
||||
print("Docker 安装完成 (需重新登录以生效 docker 用户组)")
|
||||
@@ -1,18 +1,26 @@
|
||||
"""文件日期处理工具.
|
||||
"""filedate - 文件日期处理工具.
|
||||
|
||||
自动检测文件名的日期前缀,
|
||||
并根据文件的实际创建或修改时间重命名文件.
|
||||
提供添加日期前缀/清除日期前缀 子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"DATE_PATTERN",
|
||||
"SEP",
|
||||
"add_date_prefix",
|
||||
"get_file_timestamp",
|
||||
"process_file_date",
|
||||
"process_files_date",
|
||||
"remove_date_prefix",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
@@ -22,7 +30,7 @@ SEP = "_"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# 公共函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@@ -69,11 +77,34 @@ def process_file_date(filepath: Path, clear: bool = False) -> None:
|
||||
if clear:
|
||||
remove_date_prefix(filepath)
|
||||
else:
|
||||
# 先移除旧日期前缀,再添加新日期前缀
|
||||
new_path = remove_date_prefix(filepath)
|
||||
add_date_prefix(new_path)
|
||||
|
||||
|
||||
@px.tool("filedate", subcommand="add", help="添加日期前缀")
|
||||
def process_files_date_add(files: list[Path]) -> None:
|
||||
"""添加日期前缀.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
files : list[Path]
|
||||
文件路径列表
|
||||
"""
|
||||
process_files_date(files, clear=False)
|
||||
|
||||
|
||||
@px.tool("filedate", subcommand="clear", help="清除日期前缀")
|
||||
def process_files_date_clear(files: list[Path]) -> None:
|
||||
"""清除日期前缀.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
files : list[Path]
|
||||
文件路径列表
|
||||
"""
|
||||
process_files_date(files, clear=True)
|
||||
|
||||
|
||||
def process_files_date(targets: list[Path], clear: bool = False) -> None:
|
||||
"""批量处理文件日期前缀.
|
||||
|
||||
@@ -87,51 +118,3 @@ def process_files_date(targets: list[Path], clear: bool = False) -> None:
|
||||
for target in targets:
|
||||
if target.exists() and not target.name.startswith("."):
|
||||
process_file_date(target, clear)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""文件日期处理工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="FileDate - 文件日期处理工具",
|
||||
usage="filedate <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 添加日期前缀命令
|
||||
add_parser = subparsers.add_parser("add", help="添加日期前缀")
|
||||
add_parser.add_argument("files", nargs="+", help="文件路径")
|
||||
|
||||
# 清除日期前缀命令
|
||||
clear_parser = subparsers.add_parser("clear", help="清除日期前缀")
|
||||
clear_parser.add_argument("files", nargs="+", help="文件路径")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "add":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec(
|
||||
"process_files_date",
|
||||
fn=process_files_date,
|
||||
args=([Path(f) for f in args.files],),
|
||||
kwargs={"clear": False},
|
||||
)
|
||||
])
|
||||
elif args.command == "clear":
|
||||
graph = px.Graph.from_specs([
|
||||
px.TaskSpec(
|
||||
"process_files_date",
|
||||
fn=process_files_date,
|
||||
args=([Path(f) for f in args.files],),
|
||||
kwargs={"clear": True},
|
||||
)
|
||||
])
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,16 +1,22 @@
|
||||
"""文件等级重命名工具.
|
||||
"""filelevel - 文件等级重命名工具.
|
||||
|
||||
根据文件等级配置自动重命名文件,
|
||||
支持多种等级标识和括号格式.
|
||||
提供设置文件等级 子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"BRACKETS",
|
||||
"LEVELS",
|
||||
"process_file_level",
|
||||
"process_files_level",
|
||||
"remove_marks",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
@@ -27,7 +33,7 @@ BRACKETS: tuple[str, str] = (" ([_(【-", " )]_)】")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# 公共函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@@ -69,72 +75,34 @@ def process_file_level(filepath: Path, level: int = 0) -> None:
|
||||
filestem = filepath.stem
|
||||
original_stem = filestem
|
||||
|
||||
# 移除所有等级标记
|
||||
for level_names in LEVELS.values():
|
||||
if level_names:
|
||||
filestem = remove_marks(filestem, level_names.split(","))
|
||||
|
||||
# 移除数字标记
|
||||
for digit in map(str, range(1, 10)):
|
||||
filestem = remove_marks(filestem, [digit])
|
||||
|
||||
# 添加等级标记
|
||||
if level > 0:
|
||||
levelstr = LEVELS.get(str(level), "").split(",")[0]
|
||||
if levelstr:
|
||||
filestem = f"{filestem}({levelstr})"
|
||||
|
||||
# 重命名文件
|
||||
if filestem != original_stem:
|
||||
new_path = filepath.with_name(filestem + filepath.suffix)
|
||||
filepath.rename(new_path)
|
||||
print(f"重命名: {filepath} -> {new_path}")
|
||||
|
||||
|
||||
def process_files_level(targets: list[Path], level: int = 0) -> None:
|
||||
@px.tool("filelevel", subcommand="set", help="设置文件等级")
|
||||
def process_files_level(files: list[Path], level: int = 0) -> None:
|
||||
"""批量处理文件等级标记.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
targets : list[Path]
|
||||
files : list[Path]
|
||||
文件路径列表
|
||||
level : int
|
||||
文件等级 (0-4)
|
||||
"""
|
||||
for target in targets:
|
||||
for target in files:
|
||||
process_file_level(target, level)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""文件等级重命名工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="FileLevel - 文件等级重命名工具",
|
||||
usage="filelevel <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 设置等级命令
|
||||
level_parser = subparsers.add_parser("set", help="设置文件等级")
|
||||
level_parser.add_argument("files", nargs="+", help="文件路径")
|
||||
level_parser.add_argument("--level", type=int, choices=[0, 1, 2, 3, 4], required=True, help="文件等级 (0-4)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "set":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"process_files_level", fn=process_files_level, args=([Path(f) for f in args.files], args.level)
|
||||
)
|
||||
]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -1,7 +1,6 @@
|
||||
"""文件夹备份工具.
|
||||
"""folderback - 文件夹备份工具.
|
||||
|
||||
备份文件和文件夹为 zip 文件,
|
||||
自动删除超过最大数量的旧备份文件.
|
||||
备份当前文件夹到指定目录, 自动清理旧备份.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -12,8 +11,16 @@ from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"backup_folder",
|
||||
"folderback_default",
|
||||
"remove_dump",
|
||||
"zip_target",
|
||||
]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# 公共函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@@ -40,17 +47,18 @@ def zip_target(src: Path, dst: Path, max_zip: int) -> None:
|
||||
print(f"备份完成: {target_path}")
|
||||
|
||||
|
||||
def backup_folder(src: str, dst: str, max_zip: int = 5) -> None:
|
||||
@px.tool("folderback", help="备份文件夹")
|
||||
def backup_folder(src: str = ".", dst: str = "./backup", max_zip: int = 5) -> None:
|
||||
"""备份文件夹.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
src : str
|
||||
源文件夹路径
|
||||
源文件夹路径 (默认: 当前目录)
|
||||
dst : str
|
||||
目标文件夹路径
|
||||
目标文件夹路径 (默认: ./backup)
|
||||
max_zip : int
|
||||
最大备份数量
|
||||
最大备份数量 (默认: 5)
|
||||
"""
|
||||
src_path = Path(src)
|
||||
dst_path = Path(dst)
|
||||
@@ -66,29 +74,6 @@ def backup_folder(src: str, dst: str, max_zip: int = 5) -> None:
|
||||
zip_target(src_path, dst_path, max_zip)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# TaskSpec 定义
|
||||
# ============================================================================
|
||||
|
||||
folderback_default: px.TaskSpec = px.TaskSpec(
|
||||
"folderback_default",
|
||||
fn=lambda: backup_folder(".", "./backup", 5),
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""文件夹备份工具主函数."""
|
||||
runner = px.CliRunner(
|
||||
strategy="thread",
|
||||
description="FolderBack - 文件夹备份工具",
|
||||
graphs={
|
||||
# 备份当前目录到 ./backup
|
||||
"b": px.Graph.from_specs([folderback_default]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
def folderback_default() -> None:
|
||||
"""备份当前目录到 ./backup."""
|
||||
backup_folder(".", "./backup", 5)
|
||||
@@ -1,7 +1,6 @@
|
||||
"""文件夹压缩工具.
|
||||
"""folderzip - 文件夹压缩工具.
|
||||
|
||||
压缩目录下的所有文件/文件夹为 zip 文件,
|
||||
默认压缩当前目录下的所有子文件夹.
|
||||
压缩当前目录下的所有子文件夹为 zip 文件.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,6 +10,15 @@ from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"IGNORE_DIRS",
|
||||
"IGNORE_EXT",
|
||||
"IGNORE_FILES",
|
||||
"archive_folder",
|
||||
"folderzip_default",
|
||||
"zip_folders",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
@@ -22,7 +30,7 @@ IGNORE_EXT: list[str] = [".zip", ".rar", ".7z", ".tar", ".gz"]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# 公共函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@@ -36,13 +44,14 @@ def archive_folder(folder: Path) -> None:
|
||||
print(f"压缩完成: {folder.name}.zip")
|
||||
|
||||
|
||||
@px.tool("folderzip", help="压缩文件夹")
|
||||
def zip_folders(cwd: str = ".") -> None:
|
||||
"""压缩目录下的所有文件夹.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
cwd : str
|
||||
工作目录
|
||||
工作目录 (默认: 当前目录)
|
||||
"""
|
||||
cwd_path = Path(cwd)
|
||||
if not cwd_path.exists():
|
||||
@@ -57,26 +66,6 @@ def zip_folders(cwd: str = ".") -> None:
|
||||
archive_folder(dir_path)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# TaskSpec 定义
|
||||
# ============================================================================
|
||||
|
||||
folderzip_default: px.TaskSpec = px.TaskSpec("folderzip_default", fn=lambda: zip_folders("."))
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""文件夹压缩工具主函数."""
|
||||
runner = px.CliRunner(
|
||||
strategy="thread",
|
||||
description="FolderZip - 文件夹压缩工具",
|
||||
graphs={
|
||||
# 压缩当前目录下的所有文件夹
|
||||
"z": px.Graph.from_specs([folderzip_default]),
|
||||
},
|
||||
)
|
||||
runner.run_cli()
|
||||
def folderzip_default() -> None:
|
||||
"""压缩当前目录下的所有文件夹."""
|
||||
zip_folders(".")
|
||||
@@ -0,0 +1,148 @@
|
||||
"""gittool - Git 执行工具.
|
||||
|
||||
提供添加提交/清理/初始化/推送/拉取 子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
EXCLUDE_DIRS = [
|
||||
# 编辑器相关目录
|
||||
".vscode",
|
||||
".idea",
|
||||
".editorconfig",
|
||||
".trae",
|
||||
".qoder",
|
||||
# 项目相关目录
|
||||
".venv",
|
||||
".git",
|
||||
".ruff_cache",
|
||||
".tox",
|
||||
"node_modules",
|
||||
]
|
||||
|
||||
EXCLUDE_CMDS = [arg for d in EXCLUDE_DIRS for arg in ["-e", d]]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 私有辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def not_has_git_repo() -> bool:
|
||||
"""检查当前目录没有 Git 仓库."""
|
||||
return not Path.cwd().exists() or not (Path.cwd() / ".git").is_dir()
|
||||
|
||||
|
||||
def has_files() -> bool:
|
||||
"""检查当前 Git 仓库是否有未提交的更改."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "status", "--porcelain"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return bool(result.stdout.strip())
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# fn 子命令
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool("gittool", subcommand="a", help="添加并提交")
|
||||
def git_add_commit(message: str = "chore: update") -> None:
|
||||
"""执行 git add + git commit (仅当有未提交更改时).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
message : str
|
||||
提交信息 (默认: ``chore: update``)
|
||||
"""
|
||||
if not has_files():
|
||||
print("没有文件需要提交")
|
||||
return
|
||||
subprocess.run(["git", "add", "."], check=True)
|
||||
subprocess.run(["git", "commit", "-m", message], check=True)
|
||||
|
||||
|
||||
@px.tool("gittool", subcommand="i", help="初始化并提交")
|
||||
def git_init_add_commit(message: str = "init commit") -> None:
|
||||
"""执行 git init (若需) + git add + git commit (若有更改).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
message : str
|
||||
提交信息 (默认: ``init commit``)
|
||||
"""
|
||||
if not_has_git_repo():
|
||||
subprocess.run(["git", "init"], check=True)
|
||||
if has_files():
|
||||
subprocess.run(["git", "add", "."], check=True)
|
||||
subprocess.run(["git", "commit", "-m", message], check=True)
|
||||
else:
|
||||
print("没有文件需要提交")
|
||||
|
||||
|
||||
@px.tool("gittool", subcommand="isub", help="初始化子目录")
|
||||
def init_sub_dirs() -> None:
|
||||
"""初始化子目录的 Git 仓库."""
|
||||
sub_dirs = [subdir for subdir in Path.cwd().iterdir() if subdir.is_dir()]
|
||||
for subdir in sub_dirs:
|
||||
px.run(
|
||||
px.Graph().chain(
|
||||
px.cmd(["git", "init"], conditions=(lambda _: not_has_git_repo(),), cwd=subdir),
|
||||
px.cmd(["git", "add", "."]),
|
||||
px.cmd(["git", "commit", "-m", "init commit"]),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# 兼容别名: 与 dev.py 旧函数名对齐, 便于外部调用
|
||||
a = git_add_commit
|
||||
i = git_init_add_commit
|
||||
isub = init_sub_dirs
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# cmd 子命令
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool(
|
||||
"gittool",
|
||||
subcommand="clean",
|
||||
help="清理未跟踪文件",
|
||||
cmd=["git", "clean", "-xfd", *EXCLUDE_CMDS],
|
||||
hidden=True,
|
||||
)
|
||||
def clean() -> None:
|
||||
"""清理 Git 未跟踪文件 (隐藏命令, 被 c 依赖)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"gittool",
|
||||
subcommand="c",
|
||||
help="清理并查看状态",
|
||||
cmd=["git", "status", "--porcelain"],
|
||||
needs=["clean"],
|
||||
)
|
||||
def c() -> None:
|
||||
"""清理未跟踪文件并查看 Git 状态."""
|
||||
|
||||
|
||||
@px.tool("gittool", subcommand="p", help="推送", cmd=["git", "push"])
|
||||
def p() -> None:
|
||||
"""推送代码到远程仓库."""
|
||||
|
||||
|
||||
@px.tool("gittool", subcommand="pl", help="拉取", cmd=["git", "pull"])
|
||||
def pl() -> None:
|
||||
"""从远程仓库拉取代码."""
|
||||
@@ -1,37 +1,19 @@
|
||||
"""LS-DYNA 计算工具.
|
||||
"""lscalc - LS-DYNA 计算工具.
|
||||
|
||||
用于管理 LS-DYNA 仿真计算任务,
|
||||
支持启动、监控和管理计算进程.
|
||||
运行 LS-DYNA 计算 (单机/MPI) 与进程状态检查.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
LS_DYNA_COMMANDS: dict[str, list[str]] = {
|
||||
"windows": ["ls-dyna_mpp", "i=input.k", "ncpu=4"],
|
||||
"linux": ["ls-dyna_mpp", "i=input.k", "ncpu=8"],
|
||||
"macos": ["ls-dyna_mpp", "i=input.k", "ncpu=4"],
|
||||
}
|
||||
|
||||
DEFAULT_INPUT_FILE: str = "input.k"
|
||||
DEFAULT_NCPU: int = 4
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def get_ls_dyna_command(input_file: str, ncpu: int) -> list[str]:
|
||||
"""获取 LS-DYNA 命令.
|
||||
|
||||
@@ -53,6 +35,7 @@ def get_ls_dyna_command(input_file: str, ncpu: int) -> list[str]:
|
||||
return ["ls-dyna_mpp", f"i={input_file}", f"ncpu={ncpu}"]
|
||||
|
||||
|
||||
@px.tool("lscalc", subcommand="run", help="运行 LS-DYNA 计算")
|
||||
def run_ls_dyna(input_file: str, ncpu: int = DEFAULT_NCPU) -> None:
|
||||
"""运行 LS-DYNA 计算.
|
||||
|
||||
@@ -78,6 +61,7 @@ def run_ls_dyna(input_file: str, ncpu: int = DEFAULT_NCPU) -> None:
|
||||
print(f"LS-DYNA 计算失败: {e}")
|
||||
|
||||
|
||||
@px.tool("lscalc", subcommand="mpi", help="运行 LS-DYNA MPI 计算")
|
||||
def run_ls_dyna_mpi(input_file: str, ncpu: int = DEFAULT_NCPU) -> None:
|
||||
"""运行 LS-DYNA MPI 计算.
|
||||
|
||||
@@ -103,6 +87,7 @@ def run_ls_dyna_mpi(input_file: str, ncpu: int = DEFAULT_NCPU) -> None:
|
||||
print(f"LS-DYNA MPI 计算失败: {e}")
|
||||
|
||||
|
||||
@px.tool("lscalc", subcommand="status", help="检查 LS-DYNA 进程状态")
|
||||
def check_ls_dyna_status() -> None:
|
||||
"""检查 LS-DYNA 进程状态."""
|
||||
try:
|
||||
@@ -127,48 +112,3 @@ def check_ls_dyna_status() -> None:
|
||||
print("没有运行中的 LS-DYNA 进程")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"检查进程状态失败: {e}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""LS-DYNA 计算工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="LSCalc - LS-DYNA 计算工具",
|
||||
usage="lscalc <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 运行计算命令
|
||||
run_parser = subparsers.add_parser("run", help="运行 LS-DYNA 计算")
|
||||
run_parser.add_argument("input_file", help="输入文件路径")
|
||||
run_parser.add_argument("--ncpu", type=int, default=DEFAULT_NCPU, help="CPU 核心数")
|
||||
|
||||
# 运行 MPI 计算命令
|
||||
mpi_parser = subparsers.add_parser("mpi", help="运行 LS-DYNA MPI 计算")
|
||||
mpi_parser.add_argument("input_file", help="输入文件路径")
|
||||
mpi_parser.add_argument("--ncpu", type=int, default=DEFAULT_NCPU, help="CPU 核心数")
|
||||
|
||||
# 检查进程状态命令
|
||||
subparsers.add_parser("status", help="检查 LS-DYNA 进程状态")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "run":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("run_ls_dyna", fn=run_ls_dyna, args=(args.input_file,), kwargs={"ncpu": args.ncpu})]
|
||||
)
|
||||
elif args.command == "mpi":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("run_ls_dyna_mpi", fn=run_ls_dyna_mpi, args=(args.input_file,), kwargs={"ncpu": args.ncpu})]
|
||||
)
|
||||
elif args.command == "status":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("check_ls_dyna_status", fn=check_ls_dyna_status)])
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,41 @@
|
||||
"""msdownload - ModelScope 模型/数据集下载工具.
|
||||
|
||||
从 ModelScope 下载模型、数据集或空间.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = ["msdownload_run"]
|
||||
|
||||
|
||||
@px.tool("msdownload", help="ModelScope 模型/数据集下载工具")
|
||||
def msdownload_run(name: str, target_type: str = "model", download_dir: str | None = None) -> None:
|
||||
"""从 ModelScope 下载模型/数据集/空间.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
目标名称 (如: ``Qwen/Qwen2.5-Coder-32B-Instruct``)
|
||||
target_type : str
|
||||
目标类型: ``model`` / ``dataset`` / ``space`` (默认: ``model``)
|
||||
download_dir : str | None
|
||||
下载目录; 为 None 时默认 ``~/.models/<name 最后一段>``
|
||||
"""
|
||||
if not name:
|
||||
print("msdownload: name 不能为空")
|
||||
return
|
||||
|
||||
if download_dir:
|
||||
out_dir = Path(download_dir)
|
||||
else:
|
||||
out_dir = Path.home() / ".models" / name.rsplit("/", 1)[-1]
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
cmd = ["uvx", "modelscope", "download", f"--{target_type}", name, "--local_dir", str(out_dir)]
|
||||
print(f"下载 {target_type}: {name} -> {out_dir}")
|
||||
subprocess.run(cmd, check=False)
|
||||
@@ -1,19 +1,28 @@
|
||||
"""Python 打包工具模块.
|
||||
"""packtool - Python 打包工具.
|
||||
|
||||
提供 Python 项目打包的常用功能封装,
|
||||
支持源码打包、依赖打包、嵌入式 Python 安装等功能.
|
||||
提供源码打包/依赖打包/wheel 构建/嵌入式 Python 安装/zip 包创建/清理 子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"clean_build_dir",
|
||||
"create_zip_package",
|
||||
"install_embed_python",
|
||||
"pack_dependencies",
|
||||
"pack_source",
|
||||
"pack_wheel",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
@@ -41,23 +50,23 @@ IGNORE_PATTERNS = [
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# 公共函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def pack_source(project_dir: Path, output_dir: Path) -> None:
|
||||
@px.tool("packtool", subcommand="src", help="打包源码")
|
||||
def pack_source(project_dir: Path = Path(), output_dir: Path = Path(".pypack")) -> None:
|
||||
"""打包项目源码.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
project_dir : Path
|
||||
项目目录
|
||||
项目目录 (默认: 当前目录)
|
||||
output_dir : Path
|
||||
输出目录
|
||||
输出目录 (默认: .pypack)
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 检测项目名称
|
||||
pyproject_file = project_dir / "pyproject.toml"
|
||||
project_name = project_dir.name
|
||||
|
||||
@@ -71,11 +80,9 @@ def pack_source(project_dir: Path, output_dir: Path) -> None:
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# 打包源码
|
||||
source_dir = output_dir / "src" / project_name
|
||||
source_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 复制文件
|
||||
src_subdir = project_dir / "src"
|
||||
if src_subdir.exists():
|
||||
shutil.copytree(
|
||||
@@ -102,23 +109,26 @@ def pack_source(project_dir: Path, output_dir: Path) -> None:
|
||||
print(f"源码打包完成: {source_dir}")
|
||||
|
||||
|
||||
def pack_dependencies(lib_dir: Path, dependencies: list[str]) -> None:
|
||||
@px.tool("packtool", subcommand="deps", help="打包依赖")
|
||||
def pack_dependencies(lib_dir: Path = Path("libs"), dependencies: list[str] | None = None) -> None:
|
||||
"""打包项目依赖.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lib_dir : Path
|
||||
依赖库目录
|
||||
dependencies : list[str]
|
||||
依赖库目录 (默认: libs)
|
||||
dependencies : list[str] | None
|
||||
依赖列表
|
||||
"""
|
||||
if dependencies is None:
|
||||
dependencies = []
|
||||
|
||||
lib_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if not dependencies:
|
||||
print("没有依赖需要打包")
|
||||
return
|
||||
|
||||
# 使用 pip 安装依赖到目标目录
|
||||
cmd = [
|
||||
"pip",
|
||||
"install",
|
||||
@@ -133,19 +143,19 @@ def pack_dependencies(lib_dir: Path, dependencies: list[str]) -> None:
|
||||
print(f"依赖打包完成: {lib_dir}")
|
||||
|
||||
|
||||
def pack_wheel(project_dir: Path, output_dir: Path) -> None:
|
||||
@px.tool("packtool", subcommand="wheel", help="构建 wheel")
|
||||
def pack_wheel(project_dir: Path = Path(), output_dir: Path = Path("dist")) -> None:
|
||||
"""打包项目为 wheel 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
project_dir : Path
|
||||
项目目录
|
||||
项目目录 (默认: 当前目录)
|
||||
output_dir : Path
|
||||
输出目录
|
||||
输出目录 (默认: dist)
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 使用 pip wheel 打包
|
||||
cmd = [
|
||||
"pip",
|
||||
"wheel",
|
||||
@@ -159,28 +169,25 @@ def pack_wheel(project_dir: Path, output_dir: Path) -> None:
|
||||
print(f"Wheel 打包完成: {output_dir}")
|
||||
|
||||
|
||||
def install_embed_python(version: str, output_dir: Path) -> None:
|
||||
@px.tool("packtool", subcommand="embed", help="安装嵌入式 Python")
|
||||
def install_embed_python(version: str = "3.10", output_dir: Path = Path("python")) -> None:
|
||||
"""安装嵌入式 Python.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
version : str
|
||||
Python 版本 (如: 3.10, 3.11)
|
||||
Python 版本 (如: 3.10, 3.11), 默认 3.10
|
||||
output_dir : Path
|
||||
输出目录
|
||||
输出目录 (默认: python)
|
||||
"""
|
||||
import platform
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 构建下载 URL
|
||||
arch = platform.machine().lower()
|
||||
if arch in ["x86_64", "amd64"]:
|
||||
arch = "amd64"
|
||||
elif arch in ["arm64", "aarch64"]:
|
||||
arch = "arm64"
|
||||
|
||||
# 解析完整版本号
|
||||
version_map = {
|
||||
"3.8": "3.8.10",
|
||||
"3.9": "3.9.13",
|
||||
@@ -190,36 +197,32 @@ def install_embed_python(version: str, output_dir: Path) -> None:
|
||||
}
|
||||
full_version = version_map.get(version, f"{version}.0")
|
||||
|
||||
# Windows 嵌入式 Python 下载 URL
|
||||
url = f"https://www.python.org/ftp/python/{full_version}/python-{full_version}-embed-{arch}.zip"
|
||||
|
||||
# 下载并解压
|
||||
cache_file = Path(DEFAULT_CACHE_DIR) / f"python-{full_version}-embed-{arch}.zip"
|
||||
cache_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if not cache_file.exists():
|
||||
print(f"正在下载嵌入式 Python {full_version}...")
|
||||
import urllib.request
|
||||
|
||||
urllib.request.urlretrieve(url, cache_file)
|
||||
print(f"下载完成: {cache_file}")
|
||||
|
||||
# 解压
|
||||
with zipfile.ZipFile(cache_file, "r") as zf:
|
||||
zf.extractall(output_dir)
|
||||
|
||||
print(f"嵌入式 Python 安装完成: {output_dir}")
|
||||
|
||||
|
||||
def create_zip_package(source_dir: Path, output_file: Path) -> None:
|
||||
@px.tool("packtool", subcommand="zip", help="创建 zip 包")
|
||||
def create_zip_package(source_dir: Path = Path(), output_file: Path = Path("package.zip")) -> None:
|
||||
"""创建 ZIP 打包文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
source_dir : Path
|
||||
源目录
|
||||
源目录 (默认: 当前目录)
|
||||
output_file : Path
|
||||
输出文件
|
||||
输出文件 (默认: package.zip)
|
||||
"""
|
||||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -232,118 +235,17 @@ def create_zip_package(source_dir: Path, output_file: Path) -> None:
|
||||
print(f"ZIP 打包完成: {output_file}")
|
||||
|
||||
|
||||
def clean_build_dir(build_dir: Path) -> None:
|
||||
@px.tool("packtool", subcommand="clean", help="清理构建目录")
|
||||
def clean_build_dir(build_dir: Path = Path(".pypack")) -> None:
|
||||
"""清理构建目录.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
build_dir : Path
|
||||
构建目录
|
||||
构建目录 (默认: .pypack)
|
||||
"""
|
||||
if build_dir.exists():
|
||||
shutil.rmtree(build_dir)
|
||||
print(f"清理完成: {build_dir}")
|
||||
else:
|
||||
print(f"目录不存在: {build_dir}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Python 打包工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="PackTool - Python 打包工具",
|
||||
usage="packtool <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 源码打包命令
|
||||
src_parser = subparsers.add_parser("src", help="打包项目源码")
|
||||
src_parser.add_argument("--project-dir", type=str, default=".", help="项目目录")
|
||||
src_parser.add_argument("--output-dir", type=str, default=DEFAULT_BUILD_DIR, help="输出目录")
|
||||
|
||||
# 依赖打包命令
|
||||
deps_parser = subparsers.add_parser("deps", help="打包项目依赖")
|
||||
deps_parser.add_argument("--lib-dir", type=str, default=DEFAULT_LIB_DIR, help="依赖库目录")
|
||||
deps_parser.add_argument("dependencies", nargs="*", help="依赖列表")
|
||||
|
||||
# Wheel 打包命令
|
||||
wheel_parser = subparsers.add_parser("wheel", help="打包项目为 wheel 文件")
|
||||
wheel_parser.add_argument("--project-dir", type=str, default=".", help="项目目录")
|
||||
wheel_parser.add_argument("--output-dir", type=str, default=DEFAULT_DIST_DIR, help="输出目录")
|
||||
|
||||
# 嵌入式 Python 安装命令
|
||||
embed_parser = subparsers.add_parser("embed", help="安装嵌入式 Python")
|
||||
embed_parser.add_argument("--version", type=str, default="3.10", help="Python 版本")
|
||||
embed_parser.add_argument("--output-dir", type=str, default="python", help="输出目录")
|
||||
|
||||
# ZIP 打包命令
|
||||
zip_parser = subparsers.add_parser("zip", help="创建 ZIP 打包文件")
|
||||
zip_parser.add_argument("--source-dir", type=str, default=".", help="源目录")
|
||||
zip_parser.add_argument("--output-file", type=str, default="package.zip", help="输出文件")
|
||||
|
||||
# 清理命令
|
||||
subparsers.add_parser("clean", help="清理构建目录")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "src":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pack_source",
|
||||
fn=pack_source,
|
||||
args=(Path(args.project_dir), Path(args.output_dir)),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "deps":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pack_deps",
|
||||
fn=pack_dependencies,
|
||||
args=(Path(args.lib_dir), args.dependencies),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "wheel":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"pack_wheel",
|
||||
fn=pack_wheel,
|
||||
args=(Path(args.project_dir), Path(args.output_dir)),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "embed":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"install_embed",
|
||||
fn=install_embed_python,
|
||||
args=(args.version, Path(args.output_dir)),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "zip":
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"create_zip",
|
||||
fn=create_zip_package,
|
||||
args=(Path(args.source_dir), Path(args.output_file)),
|
||||
)
|
||||
]
|
||||
)
|
||||
elif args.command == "clean":
|
||||
graph = px.Graph.from_specs([px.TaskSpec("clean_build", fn=clean_build_dir, args=(Path(DEFAULT_BUILD_DIR),))])
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,511 @@
|
||||
"""pdftool - PDF 文件工具集.
|
||||
|
||||
提供 PDF 合并/拆分/压缩/加密/解密/提取文本/提取图片/水印/旋转/裁剪/
|
||||
信息/OCR/转图片/修复 等子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"pdf_add_watermark",
|
||||
"pdf_compress",
|
||||
"pdf_crop",
|
||||
"pdf_decrypt",
|
||||
"pdf_encrypt",
|
||||
"pdf_extract_images",
|
||||
"pdf_extract_text",
|
||||
"pdf_info",
|
||||
"pdf_merge",
|
||||
"pdf_ocr",
|
||||
"pdf_reorder",
|
||||
"pdf_repair",
|
||||
"pdf_rotate",
|
||||
"pdf_split",
|
||||
"pdf_to_images",
|
||||
]
|
||||
|
||||
try:
|
||||
import fitz # PyMuPDF
|
||||
|
||||
HAS_PYMUPDF = True
|
||||
except ImportError:
|
||||
HAS_PYMUPDF = False
|
||||
|
||||
try:
|
||||
import pypdf
|
||||
|
||||
HAS_PYPDF = True
|
||||
except ImportError:
|
||||
HAS_PYPDF = False
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="m", help="合并 PDF")
|
||||
def pdf_merge(input_paths: list[Path], output_path: Path = Path("merged.pdf")) -> None:
|
||||
"""合并多个 PDF 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_paths : list[Path]
|
||||
输入 PDF 文件列表
|
||||
output_path : Path
|
||||
输出文件 (默认: merged.pdf)
|
||||
"""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库, 请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
writer = pypdf.PdfWriter()
|
||||
for input_path in input_paths:
|
||||
if input_path.exists():
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"合并完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="s", help="拆分 PDF")
|
||||
def pdf_split(input_path: Path, output_dir: Path = Path("split")) -> None:
|
||||
"""拆分 PDF 文件为单页.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_dir : Path
|
||||
输出目录 (默认: split)
|
||||
"""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库, 请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for i, page in enumerate(reader.pages):
|
||||
writer = pypdf.PdfWriter()
|
||||
writer.add_page(page)
|
||||
output_file = output_dir / f"{input_path.stem}_page_{i + 1}.pdf"
|
||||
with open(output_file, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"拆分完成: {output_dir}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="c", help="压缩 PDF")
|
||||
def pdf_compress(input_path: Path, output_path: Path = Path("compressed.pdf")) -> None:
|
||||
"""压缩 PDF 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: compressed.pdf)
|
||||
"""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path), garbage=4, deflate=True, clean=True)
|
||||
doc.close()
|
||||
|
||||
original_size = input_path.stat().st_size
|
||||
new_size = output_path.stat().st_size
|
||||
ratio = (1 - new_size / original_size) * 100
|
||||
print(f"压缩完成: {output_path} (缩小 {ratio:.1f}%)")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="e", help="加密 PDF")
|
||||
def pdf_encrypt(input_path: Path, output_path: Path = Path("encrypted.pdf"), password: str = "") -> None:
|
||||
"""加密 PDF 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: encrypted.pdf)
|
||||
password : str
|
||||
密码 (必填)
|
||||
"""
|
||||
if not password:
|
||||
print("错误: --password 为必填参数")
|
||||
return
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库, 请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
writer = pypdf.PdfWriter()
|
||||
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
writer.encrypt(user_password=password, owner_password=password, use_128bit=True)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"加密完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="d", help="解密 PDF")
|
||||
def pdf_decrypt(input_path: Path, output_path: Path = Path("decrypted.pdf"), password: str = "") -> None:
|
||||
"""解密 PDF 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: decrypted.pdf)
|
||||
password : str
|
||||
密码 (必填)
|
||||
"""
|
||||
if not password:
|
||||
print("错误: --password 为必填参数")
|
||||
return
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库, 请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
if reader.is_encrypted:
|
||||
reader.decrypt(password)
|
||||
|
||||
writer = pypdf.PdfWriter()
|
||||
for page in reader.pages:
|
||||
writer.add_page(page)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"解密完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="xt", help="提取文本")
|
||||
def pdf_extract_text(input_path: Path, output_path: Path = Path("output.txt")) -> None:
|
||||
"""提取 PDF 文本.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: output.txt)
|
||||
"""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
text = ""
|
||||
for page in doc:
|
||||
text += str(page.get_text()) + "\n\n"
|
||||
doc.close()
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(text, encoding="utf-8")
|
||||
print(f"文本提取完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="xi", help="提取图片")
|
||||
def pdf_extract_images(input_path: Path, output_dir: Path = Path("images")) -> None:
|
||||
"""提取 PDF 图片.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_dir : Path
|
||||
输出目录 (默认: images)
|
||||
"""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
image_count = 0
|
||||
# pyrefly: ignore [bad-argument-type]
|
||||
for page_num, page in enumerate(doc):
|
||||
images = page.get_images(full=True)
|
||||
for img_idx, img in enumerate(images):
|
||||
xref = img[0]
|
||||
base_image = doc.extract_image(xref)
|
||||
image_data = base_image["image"]
|
||||
image_ext = base_image["ext"]
|
||||
image_path = output_dir / f"page_{page_num + 1}_img_{img_idx + 1}.{image_ext}"
|
||||
image_path.write_bytes(image_data)
|
||||
image_count += 1
|
||||
|
||||
doc.close()
|
||||
print(f"图片提取完成: {output_dir} (共 {image_count} 张)")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="w", help="添加水印")
|
||||
def pdf_add_watermark(
|
||||
input_path: Path, output_path: Path = Path("watermarked.pdf"), text: str = "CONFIDENTIAL"
|
||||
) -> None:
|
||||
"""添加 PDF 水印.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: watermarked.pdf)
|
||||
text : str
|
||||
水印文字 (默认: CONFIDENTIAL)
|
||||
"""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
for page in doc:
|
||||
rect = page.rect
|
||||
text_width = fitz.get_text_length(text, fontsize=48)
|
||||
x = (rect.width - text_width) / 2
|
||||
y = rect.height / 2
|
||||
page.insert_text((x, y), text, fontsize=48, rotate=45, color=(0, 0, 0))
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"水印添加完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="r", help="旋转 PDF")
|
||||
def pdf_rotate(input_path: Path, output_path: Path = Path("rotated.pdf"), rotation: int = 90) -> None:
|
||||
"""旋转 PDF 页面.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: rotated.pdf)
|
||||
rotation : int
|
||||
旋转角度 (默认: 90)
|
||||
"""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
for page in doc:
|
||||
page.set_rotation(rotation)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"旋转完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="crop", help="裁剪 PDF")
|
||||
def pdf_crop(
|
||||
input_path: Path,
|
||||
output_path: Path = Path("cropped.pdf"),
|
||||
margins: tuple[int, int, int, int] = (10, 10, 10, 10),
|
||||
) -> None:
|
||||
"""裁剪 PDF 页面.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: cropped.pdf)
|
||||
margins : tuple[int, int, int, int]
|
||||
边距 (左, 上, 右, 下), 默认 (10, 10, 10, 10)
|
||||
"""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
left, top, right, bottom = margins
|
||||
|
||||
for page in doc:
|
||||
rect = page.rect
|
||||
new_rect = fitz.Rect(
|
||||
rect.x0 + left,
|
||||
rect.y0 + top,
|
||||
rect.x1 - right,
|
||||
rect.y1 - bottom,
|
||||
)
|
||||
page.set_cropbox(new_rect)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path))
|
||||
doc.close()
|
||||
print(f"裁剪完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="i", help="查看 PDF 信息")
|
||||
def pdf_info(input_path: Path) -> None:
|
||||
"""显示 PDF 信息.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
"""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
print(f"文件: {input_path}")
|
||||
print(f"页数: {doc.page_count}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"标题: {doc.metadata.get('title', 'N/A')}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"作者: {doc.metadata.get('author', 'N/A')}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"创建日期: {doc.metadata.get('creationDate', 'N/A')}")
|
||||
# pyrefly: ignore [missing-attribute]
|
||||
print(f"修改日期: {doc.metadata.get('modDate', 'N/A')}")
|
||||
print(f"文件大小: {input_path.stat().st_size / 1024:.1f} KB")
|
||||
doc.close()
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="ocr", help="PDF OCR 识别")
|
||||
def pdf_ocr(input_path: Path, output_path: Path = Path("ocr.pdf"), lang: str = "chi_sim+eng") -> None:
|
||||
"""PDF OCR 识别.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: ocr.pdf)
|
||||
lang : str
|
||||
识别语言 (默认: chi_sim+eng)
|
||||
"""
|
||||
try:
|
||||
import pytesseract
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
print("未安装 OCR 相关库, 请安装: pip install pytesseract pillow")
|
||||
return
|
||||
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
new_doc = fitz.open()
|
||||
|
||||
for page in doc:
|
||||
pix = page.get_pixmap()
|
||||
img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
|
||||
ocr_text = pytesseract.image_to_string(img, lang=lang)
|
||||
|
||||
new_page = new_doc.new_page(width=page.rect.width, height=page.rect.height)
|
||||
new_page.insert_image(new_page.rect, pixmap=pix)
|
||||
text_rect = fitz.Rect(0, 0, page.rect.width, page.rect.height)
|
||||
# pyrefly: ignore [bad-argument-type]
|
||||
new_page.insert_textbox(text_rect, ocr_text, fontname="china-ss", fontsize=11)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
new_doc.save(str(output_path))
|
||||
new_doc.close()
|
||||
doc.close()
|
||||
print(f"OCR 识别完成: {output_path}")
|
||||
|
||||
|
||||
def pdf_reorder(input_path: Path, output_path: Path, order: list[int]) -> None:
|
||||
"""重排 PDF 页面顺序.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件
|
||||
order : list[int]
|
||||
页面顺序列表 (0-based)
|
||||
"""
|
||||
if not HAS_PYPDF:
|
||||
print("未安装 pypdf 库, 请安装: pip install pypdf")
|
||||
return
|
||||
|
||||
reader = pypdf.PdfReader(str(input_path))
|
||||
writer = pypdf.PdfWriter()
|
||||
|
||||
for page_num in order:
|
||||
if 0 <= page_num < len(reader.pages):
|
||||
writer.add_page(reader.pages[page_num])
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
print(f"重排完成: {output_path}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="img", help="PDF 转图片")
|
||||
def pdf_to_images(input_path: Path, output_dir: Path = Path("images"), dpi: int = 300) -> None:
|
||||
"""PDF 转图片.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_dir : Path
|
||||
输出目录 (默认: images)
|
||||
dpi : int
|
||||
DPI (默认: 300)
|
||||
"""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# pyrefly: ignore [bad-argument-type]
|
||||
for page_num, page in enumerate(doc):
|
||||
pix = page.get_pixmap(dpi=dpi)
|
||||
image_path = output_dir / f"{input_path.stem}_page_{page_num + 1}.png"
|
||||
pix.save(str(image_path))
|
||||
|
||||
doc.close()
|
||||
print(f"转换完成: {output_dir}")
|
||||
|
||||
|
||||
@px.tool("pdftool", subcommand="repair", help="修复 PDF")
|
||||
def pdf_repair(input_path: Path, output_path: Path = Path("repaired.pdf")) -> None:
|
||||
"""修复 PDF 文件.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_path : Path
|
||||
输入 PDF 文件
|
||||
output_path : Path
|
||||
输出文件 (默认: repaired.pdf)
|
||||
"""
|
||||
if not HAS_PYMUPDF:
|
||||
print("未安装 PyMuPDF 库, 请安装: pip install PyMuPDF")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(input_path))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc.save(str(output_path), garbage=4, deflate=True, clean=True)
|
||||
doc.close()
|
||||
print(f"修复完成: {output_path}")
|
||||
@@ -0,0 +1,177 @@
|
||||
"""piptool - pip 包管理工具.
|
||||
|
||||
提供安装/卸载/重装/下载/升级 pip/冻结依赖 子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
__all__ = [
|
||||
"pip_download",
|
||||
"pip_freeze",
|
||||
"pip_install",
|
||||
"pip_reinstall",
|
||||
"pip_uninstall",
|
||||
"pip_upgrade",
|
||||
]
|
||||
|
||||
# ============================================================================
|
||||
# 配置
|
||||
# ============================================================================
|
||||
|
||||
PACKAGE_DIR = "packages"
|
||||
REQUIREMENTS_FILE = "requirements.txt"
|
||||
|
||||
_PROTECTED_PACKAGES: frozenset[str] = frozenset(
|
||||
{
|
||||
"pyflowx",
|
||||
"bitool",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 私有辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _get_installed_packages() -> list[str]:
|
||||
"""获取当前环境中所有已安装的包名."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pip", "list", "--format=freeze"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
packages: list[str] = []
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
if line and "==" in line:
|
||||
pkg_name = line.split("==")[0].strip()
|
||||
packages.append(pkg_name)
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
return []
|
||||
return packages
|
||||
|
||||
|
||||
def _expand_wildcard_packages(pattern: str) -> list[str]:
|
||||
"""展开通配符模式为实际的包名列表."""
|
||||
if not any(char in pattern for char in ["*", "?", "[", "]"]):
|
||||
return [pattern]
|
||||
|
||||
installed_packages = _get_installed_packages()
|
||||
matched = [pkg for pkg in installed_packages if fnmatch.fnmatchcase(pkg.lower(), pattern.lower())]
|
||||
return matched
|
||||
|
||||
|
||||
def _filter_protected_packages(packages: list[str]) -> list[str]:
|
||||
"""过滤掉受保护的包名."""
|
||||
safe = [p for p in packages if p.lower() not in {p.lower() for p in _PROTECTED_PACKAGES}]
|
||||
filtered = [p for p in packages if p.lower() in {p.lower() for p in _PROTECTED_PACKAGES}]
|
||||
if filtered:
|
||||
print(f"跳过受保护的包: {', '.join(filtered)}")
|
||||
return safe
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 公共函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool("piptool", subcommand="i", help="安装包")
|
||||
def pip_install(packages: list[str]) -> None:
|
||||
"""安装包.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
packages : list[str]
|
||||
包名列表
|
||||
"""
|
||||
subprocess.run(["pip", "install", *packages], check=True)
|
||||
print(f"安装完成: {', '.join(packages)}")
|
||||
|
||||
|
||||
@px.tool("piptool", subcommand="u", help="卸载包")
|
||||
def pip_uninstall(packages: list[str]) -> None:
|
||||
"""卸载包.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
packages : list[str]
|
||||
包名列表
|
||||
"""
|
||||
packages_to_uninstall: list[str] = []
|
||||
for pattern in packages:
|
||||
packages_to_uninstall.extend(_expand_wildcard_packages(pattern))
|
||||
|
||||
packages_to_uninstall = _filter_protected_packages(packages_to_uninstall)
|
||||
|
||||
if not packages_to_uninstall:
|
||||
return
|
||||
|
||||
subprocess.run(["pip", "uninstall", "-y", *packages_to_uninstall], check=True)
|
||||
|
||||
|
||||
@px.tool("piptool", subcommand="r", help="重装包")
|
||||
def pip_reinstall(packages: list[str], offline: bool = False) -> None:
|
||||
"""重新安装包.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
packages : list[str]
|
||||
包名列表
|
||||
offline : bool
|
||||
离线模式
|
||||
"""
|
||||
safe_ps = _filter_protected_packages(packages)
|
||||
if not safe_ps:
|
||||
print("所有指定的包均为受保护包, 跳过重装")
|
||||
return
|
||||
|
||||
subprocess.run(["pip", "uninstall", "-y", *safe_ps], check=True)
|
||||
|
||||
options = ["--no-index", "--find-links", "."] if offline else []
|
||||
subprocess.run(["pip", "install", *options, *safe_ps], check=True)
|
||||
|
||||
|
||||
@px.tool("piptool", subcommand="d", help="下载包")
|
||||
def pip_download(packages: list[str], offline: bool = False) -> None:
|
||||
"""下载包.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
packages : list[str]
|
||||
包名列表
|
||||
offline : bool
|
||||
离线模式
|
||||
"""
|
||||
options = ["--no-index", "--find-links", "."] if offline else []
|
||||
subprocess.run(
|
||||
["pip", "download", *packages, *options, "-d", PACKAGE_DIR],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
@px.tool("piptool", subcommand="up", help="升级 pip")
|
||||
def pip_upgrade() -> None:
|
||||
"""升级 pip 到最新版本."""
|
||||
subprocess.run(["python", "-m", "pip", "install", "--upgrade", "pip"], check=True)
|
||||
print("pip 升级完成")
|
||||
|
||||
|
||||
@px.tool("piptool", subcommand="f", help="导出依赖")
|
||||
def pip_freeze() -> None:
|
||||
"""冻结依赖到 requirements.txt."""
|
||||
result = subprocess.run(
|
||||
["pip", "freeze", "--exclude-editable"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
Path(REQUIREMENTS_FILE).write_text(result.stdout)
|
||||
print(f"依赖已导出到 {REQUIREMENTS_FILE}")
|
||||
@@ -0,0 +1,231 @@
|
||||
"""pymake - 项目构建工具.
|
||||
|
||||
提供构建/测试/清理/文档/格式化/发布等子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = [
|
||||
"b",
|
||||
"ba",
|
||||
"bc",
|
||||
"bump",
|
||||
"bumpmi",
|
||||
"bumpversion",
|
||||
"c",
|
||||
"cov",
|
||||
"doc",
|
||||
"git_add_all",
|
||||
"git_push",
|
||||
"git_push_tags",
|
||||
"lint",
|
||||
"p",
|
||||
"pb",
|
||||
"publish_python",
|
||||
"pyrefly_check",
|
||||
"sync",
|
||||
"t",
|
||||
"tc",
|
||||
"test_coverage",
|
||||
"tf",
|
||||
"tox",
|
||||
"twine_publish",
|
||||
]
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 单任务别名 (cmd 任务)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="b", help="构建 Python 主包 (uv build)", cmd=["uv", "build"])
|
||||
def b(cwd: Path = Path()) -> None:
|
||||
"""构建 Python 主包."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="bc", help="构建 Rust 核心模块 (maturin build)", cmd=["maturin", "build", "-r"])
|
||||
def bc(cwd: Path = Path()) -> None:
|
||||
"""构建 Rust 核心模块."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="sync", help="同步依赖 (uv sync)", cmd=["uv", "sync"])
|
||||
def sync(cwd: Path = Path()) -> None:
|
||||
"""同步依赖."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="c", help="清理构建产物 (调用 gitt c)", cmd=["pf", "gitt", "c"])
|
||||
def c(cwd: Path = Path()) -> None:
|
||||
"""清理构建产物."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="t",
|
||||
help="运行测试",
|
||||
cmd=["pytest", "-m", "not slow", "-n", "8", "--dist", "loadfile", "--color=yes", "--durations=10"],
|
||||
)
|
||||
def t(cwd: Path = Path()) -> None:
|
||||
"""运行测试 (含并行)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="tf",
|
||||
help="快速测试 (无 slow)",
|
||||
cmd=["pytest", "-m", "not slow", "--dist", "loadfile", "--color=yes", "--durations=10"],
|
||||
)
|
||||
def tf(cwd: Path = Path()) -> None:
|
||||
"""快速测试 (无 slow, 无并行)."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="bumpmi", help="升级次版本号 (bumpversion minor)", cmd=["pf", "bumpversion", "minor"])
|
||||
def bumpmi(cwd: Path = Path()) -> None:
|
||||
"""升级次版本号."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="doc",
|
||||
help="构建 Sphinx 文档",
|
||||
cmd=["sphinx-build", "-b", "html", "docs", "docs/_build"],
|
||||
)
|
||||
def doc(cwd: Path = Path()) -> None:
|
||||
"""构建 Sphinx 文档."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="lint", help="代码格式化与检查 (ruff)", cmd=["ruff", "check", "--fix", "--unsafe-fixes"])
|
||||
def lint(cwd: Path = Path()) -> None:
|
||||
"""代码格式化与检查."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="tox", help="多版本测试 (tox)", cmd=["tox", "-p", "auto"])
|
||||
def tox(cwd: Path = Path()) -> None:
|
||||
"""多版本测试."""
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 内部 job (hidden, 不暴露为 subcommand)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="bumpversion",
|
||||
help="升级版本号 (patch)",
|
||||
cmd=["pf", "bumpversion", "patch"],
|
||||
needs=["git_add_all"],
|
||||
hidden=True,
|
||||
)
|
||||
def bumpversion(cwd: Path = Path()) -> None:
|
||||
"""升级版本号 (patch, 内部 job)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="test_coverage",
|
||||
help="测试并生成覆盖率",
|
||||
cmd=["pytest", "--cov", "-n", "8", "--dist", "loadfile", "--tb=short", "-v", "--color=yes", "--durations=10"],
|
||||
needs=["c"],
|
||||
hidden=True,
|
||||
)
|
||||
def test_coverage(cwd: Path = Path()) -> None:
|
||||
"""测试并生成覆盖率 (内部 job)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="pyrefly_check",
|
||||
help="pyrefly 类型检查",
|
||||
cmd=["pyrefly", "check", "."],
|
||||
hidden=True,
|
||||
)
|
||||
def pyrefly_check(cwd: Path = Path()) -> None:
|
||||
"""pyrefly 类型检查 (内部 job)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="git_add_all",
|
||||
help="git add -A",
|
||||
cmd=["git", "add", "-A"],
|
||||
needs=["tc"],
|
||||
hidden=True,
|
||||
)
|
||||
def git_add_all(cwd: Path = Path()) -> None:
|
||||
"""git add -A (内部 job)."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="git_push", help="git push", cmd=["git", "push"], hidden=True)
|
||||
def git_push(cwd: Path = Path()) -> None:
|
||||
"""git push (内部 job)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="git_push_tags",
|
||||
help="git push --tags",
|
||||
cmd=["git", "push", "--tags"],
|
||||
hidden=True,
|
||||
)
|
||||
def git_push_tags(cwd: Path = Path()) -> None:
|
||||
"""git push --tags (内部 job)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="twine_publish",
|
||||
help="twine upload",
|
||||
cmd=["twine", "upload", "--disable-progress-bar"],
|
||||
hidden=True,
|
||||
)
|
||||
def twine_publish(cwd: Path = Path()) -> None:
|
||||
"""twine upload (内部 job)."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="publish_python", help="hatch publish", cmd=["hatch", "publish"], hidden=True)
|
||||
def publish_python(cwd: Path = Path()) -> None:
|
||||
"""hatch publish (内部 job)."""
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 聚合 job (有 needs 无 cmd)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="ba", help="构建所有包 (Python + Rust)", needs=["b", "bc"], strategy="thread")
|
||||
def ba(cwd: Path = Path()) -> None:
|
||||
"""构建所有包 (Python + Rust)."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="bump", help="升级版本号 (清理 + 检查 + add + bumpversion)", needs=["bumpversion"])
|
||||
def bump(cwd: Path = Path()) -> None:
|
||||
"""升级版本号 (聚合)."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="cov", help="测试并生成覆盖率", needs=["test_coverage"])
|
||||
def cov(cwd: Path = Path()) -> None:
|
||||
"""测试并生成覆盖率 (聚合)."""
|
||||
|
||||
|
||||
@px.tool(
|
||||
"pymake",
|
||||
subcommand="tc",
|
||||
help="类型检查 (pyrefly + ruff)",
|
||||
needs=["c", "pyrefly_check", "lint"],
|
||||
strategy="thread",
|
||||
)
|
||||
def tc(cwd: Path = Path()) -> None:
|
||||
"""类型检查 (聚合)."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="p", help="推送代码 (清理 + push + push tags)", needs=["c", "git_push", "git_push_tags"], strategy="thread")
|
||||
def p(cwd: Path = Path()) -> None:
|
||||
"""推送代码 (聚合)."""
|
||||
|
||||
|
||||
@px.tool("pymake", subcommand="pb", help="发布到 PyPI (twine + hatch)", needs=["twine_publish", "publish_python"], strategy="thread")
|
||||
def pb(cwd: Path = Path()) -> None:
|
||||
"""发布到 PyPI (聚合)."""
|
||||
@@ -0,0 +1,52 @@
|
||||
"""reseticoncache - 重置 Windows 图标缓存工具.
|
||||
|
||||
执行流程: 杀掉 explorer → 删除 IconCache.db → 删除 iconcache* → 重启 explorer.
|
||||
仅在 Windows 上执行, 非 Windows 平台打印提示并跳过.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
@px.tool("reseticoncache", help="重置 Windows 图标缓存")
|
||||
def reset_icon_cache_run() -> None:
|
||||
"""重置 Windows 图标缓存.
|
||||
|
||||
执行流程: 杀掉 explorer → 删除 IconCache.db → 删除 iconcache* → 重启 explorer.
|
||||
仅在 Windows 上执行, 非 Windows 平台打印提示并跳过.
|
||||
"""
|
||||
if not Constants.IS_WINDOWS:
|
||||
print("reset_icon_cache: 仅在 Windows 上支持")
|
||||
return
|
||||
|
||||
local_app_data = os.environ.get("LOCALAPPDATA", "")
|
||||
if not local_app_data:
|
||||
print("reset_icon_cache: LOCALAPPDATA 环境变量未设置")
|
||||
return
|
||||
|
||||
icon_cache_db = Path(local_app_data) / "IconCache.db"
|
||||
explorer_cache_dir = Path(local_app_data) / "Microsoft" / "Windows" / "Explorer"
|
||||
|
||||
print("正在终止 explorer 进程...")
|
||||
subprocess.run(["taskkill", "/f", "/im", "explorer.exe"], check=False)
|
||||
|
||||
if icon_cache_db.exists():
|
||||
print(f"删除图标缓存: {icon_cache_db}")
|
||||
subprocess.run(["cmd", "/c", "del", "/a", "/q", str(icon_cache_db)], check=False)
|
||||
|
||||
if explorer_cache_dir.exists():
|
||||
print(f"清理 Explorer 缓存: {explorer_cache_dir}")
|
||||
subprocess.run(
|
||||
["cmd", "/c", "del", "/a", "/q", str(explorer_cache_dir / "iconcache*")],
|
||||
check=False,
|
||||
)
|
||||
|
||||
print("重启 explorer...")
|
||||
subprocess.run(["cmd", "/c", "start", "explorer.exe"], check=False)
|
||||
print("图标缓存已重置")
|
||||
@@ -1,11 +1,10 @@
|
||||
"""截图工具.
|
||||
"""screenshot - 截图工具.
|
||||
|
||||
跨平台截图工具, 支持全屏截图和区域截图.
|
||||
跨平台截图: Windows 用 PowerShell, macOS 用 screencapture, Linux 用 gnome-screenshot/scrot.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -13,10 +12,6 @@ from pathlib import Path
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def get_screenshot_path(filename: str | None = None) -> Path:
|
||||
"""获取截图保存路径.
|
||||
@@ -40,6 +35,7 @@ def get_screenshot_path(filename: str | None = None) -> Path:
|
||||
return screenshots_dir / filename
|
||||
|
||||
|
||||
@px.tool("screenshot", subcommand="full", help="全屏截图")
|
||||
def take_screenshot_full(filename: str | None = None) -> None:
|
||||
"""全屏截图.
|
||||
|
||||
@@ -51,7 +47,6 @@ def take_screenshot_full(filename: str | None = None) -> None:
|
||||
output_path = get_screenshot_path(filename)
|
||||
|
||||
if Constants.IS_WINDOWS:
|
||||
# Windows: 使用 PowerShell 截图
|
||||
ps_script = f"""
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
@@ -66,10 +61,8 @@ $bitmap.Dispose()
|
||||
"""
|
||||
subprocess.run(["powershell", "-Command", ps_script], check=True)
|
||||
elif Constants.IS_MACOS:
|
||||
# macOS: 使用 screencapture
|
||||
subprocess.run(["screencapture", "-x", str(output_path)], check=True)
|
||||
else:
|
||||
# Linux: 使用 gnome-screenshot 或 scrot
|
||||
try:
|
||||
subprocess.run(["gnome-screenshot", "-f", str(output_path)], check=True)
|
||||
except FileNotFoundError:
|
||||
@@ -78,6 +71,7 @@ $bitmap.Dispose()
|
||||
print(f"截图已保存: {output_path}")
|
||||
|
||||
|
||||
@px.tool("screenshot", subcommand="area", help="区域截图")
|
||||
def take_screenshot_area(filename: str | None = None) -> None:
|
||||
"""区域截图.
|
||||
|
||||
@@ -89,7 +83,6 @@ def take_screenshot_area(filename: str | None = None) -> None:
|
||||
output_path = get_screenshot_path(filename)
|
||||
|
||||
if Constants.IS_WINDOWS:
|
||||
# Windows: 使用 PowerShell 截图 (需要用户选择区域)
|
||||
ps_script = f"""
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
@@ -113,51 +106,11 @@ $bitmap.Dispose()
|
||||
"""
|
||||
subprocess.run(["powershell", "-Command", ps_script], check=True)
|
||||
elif Constants.IS_MACOS:
|
||||
# macOS: 使用 screencapture 交互模式
|
||||
subprocess.run(["screencapture", "-i", str(output_path)], check=True)
|
||||
else:
|
||||
# Linux: 使用 gnome-screenshot 交互模式
|
||||
try:
|
||||
subprocess.run(["gnome-screenshot", "-a", "-f", str(output_path)], check=True)
|
||||
except FileNotFoundError:
|
||||
subprocess.run(["scrot", "-s", str(output_path)], check=True)
|
||||
|
||||
print(f"截图已保存: {output_path}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""截图工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Screenshot - 截图工具",
|
||||
usage="screenshot <command> [options]",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
|
||||
# 全屏截图命令
|
||||
full_parser = subparsers.add_parser("full", help="全屏截图")
|
||||
full_parser.add_argument("--filename", type=str, help="文件名")
|
||||
|
||||
# 区域截图命令
|
||||
area_parser = subparsers.add_parser("area", help="区域截图")
|
||||
area_parser.add_argument("--filename", type=str, help="文件名")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "full":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("screenshot_full", fn=take_screenshot_full, kwargs={"filename": args.filename})]
|
||||
)
|
||||
elif args.command == "area":
|
||||
graph = px.Graph.from_specs(
|
||||
[px.TaskSpec("screenshot_area", fn=take_screenshot_area, kwargs={"filename": args.filename})]
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,87 @@
|
||||
"""sglang - SGLang 本地模型服务工具.
|
||||
|
||||
提供安装与启动 SGLang 服务的子命令.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
__all__ = [
|
||||
"install_sglang",
|
||||
"run_sglang",
|
||||
]
|
||||
|
||||
|
||||
@px.tool("sglang", subcommand="install", help="安装 sglang")
|
||||
def install_sglang() -> None:
|
||||
"""安装 sglang (若未安装).
|
||||
|
||||
通过 ``shutil.which`` 检测 sglang 是否已安装, 未安装时执行 ``uv install sglang[all]``.
|
||||
"""
|
||||
if shutil.which("sglang") is not None:
|
||||
print("sglang 已安装, 跳过安装步骤")
|
||||
return
|
||||
|
||||
print("正在安装 sglang[all]...")
|
||||
subprocess.run(["uv", "install", "sglang[all]"], check=False)
|
||||
|
||||
|
||||
@px.tool("sglang", subcommand="run", help="启动 SGLang 服务", needs=["install"])
|
||||
def run_sglang(
|
||||
model: str = "~/.models/Qwen2.5-Coder-32B-Instruct-AWQ",
|
||||
port: int = 8000,
|
||||
ctx_len: int = 32768,
|
||||
mem_fraction: float = 0.75,
|
||||
host: str = "0.0.0.0",
|
||||
log_level: str = "info",
|
||||
) -> None:
|
||||
"""启动 SGLang 本地模型服务.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
model : str
|
||||
模型路径 (默认: ``~/.models/Qwen2.5-Coder-32B-Instruct-AWQ``)
|
||||
port : int
|
||||
服务端口 (默认: 8000)
|
||||
ctx_len : int
|
||||
最大上下文长度 (默认: 32768)
|
||||
mem_fraction : float
|
||||
显存占比 0-1 (默认: 0.75)
|
||||
host : str
|
||||
主机地址 (默认: 0.0.0.0)
|
||||
log_level : str
|
||||
日志级别 (默认: info)
|
||||
"""
|
||||
model_dir = Path(model).expanduser()
|
||||
if not model_dir.exists():
|
||||
print(f"模型目录不存在: {model_dir}")
|
||||
return
|
||||
|
||||
python_bin = "python" if Constants.IS_WINDOWS else "python3"
|
||||
cmd = [
|
||||
python_bin,
|
||||
"-m",
|
||||
"sglang.launch_server",
|
||||
"--model-path",
|
||||
str(model_dir),
|
||||
"--host",
|
||||
host,
|
||||
"--port",
|
||||
str(port),
|
||||
"--mem-fraction-static",
|
||||
str(mem_fraction),
|
||||
"--context-length",
|
||||
str(ctx_len),
|
||||
"--tool-call-parser",
|
||||
"qwen",
|
||||
"--log-level",
|
||||
log_level,
|
||||
]
|
||||
print(f"启动 SGLang: {model_dir} (port={port}, ctx={ctx_len}, mem={mem_fraction})")
|
||||
subprocess.run(cmd, check=False)
|
||||
@@ -1,23 +1,20 @@
|
||||
"""SSH 密钥部署工具.
|
||||
"""sshcopyid - SSH 密钥部署工具.
|
||||
|
||||
类似 ssh-copy-id, 自动将 SSH 公钥部署到远程服务器,
|
||||
支持密码认证和密钥认证两种方式.
|
||||
将本地 SSH 公钥部署到远程服务器.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pyflowx as px
|
||||
|
||||
# ============================================================================
|
||||
# 辅助函数
|
||||
# ============================================================================
|
||||
__all__ = ["ssh_copy_id"]
|
||||
|
||||
|
||||
@px.tool("sshcopyid", help="部署 SSH 公钥到远程服务器")
|
||||
def ssh_copy_id(
|
||||
hostname: str,
|
||||
username: str,
|
||||
@@ -43,7 +40,6 @@ def ssh_copy_id(
|
||||
timeout : int
|
||||
SSH 操作超时秒数, 默认 30
|
||||
"""
|
||||
# 读取公钥
|
||||
pub_key_path = Path(keypath).expanduser()
|
||||
if not pub_key_path.exists():
|
||||
print(f"公钥文件不存在: {pub_key_path}")
|
||||
@@ -51,12 +47,10 @@ def ssh_copy_id(
|
||||
|
||||
pub_key = pub_key_path.read_text().strip()
|
||||
|
||||
# 构建部署脚本
|
||||
script = f"""mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||
cd ~/.ssh && touch authorized_keys && chmod 600 authorized_keys
|
||||
grep -qF '{pub_key.split()[1]}' authorized_keys 2>/dev/null || echo '{pub_key}' >> authorized_keys"""
|
||||
|
||||
# 使用 sshpass 执行
|
||||
try:
|
||||
subprocess.run(
|
||||
[
|
||||
@@ -80,7 +74,7 @@ grep -qF '{pub_key.split()[1]}' authorized_keys 2>/dev/null || echo '{pub_key}'
|
||||
)
|
||||
print(f"SSH 密钥已部署到 {username}@{hostname}:{port}")
|
||||
except FileNotFoundError:
|
||||
print(f"未找到 sshpass 工具,请手动执行: ssh-copy-id -p {port} {username}@{hostname}")
|
||||
print(f"未找到 sshpass 工具, 请手动执行: ssh-copy-id -p {port} {username}@{hostname}")
|
||||
sys.exit(1)
|
||||
except subprocess.TimeoutExpired:
|
||||
print("SSH 连接超时")
|
||||
@@ -88,35 +82,3 @@ grep -qF '{pub_key.split()[1]}' authorized_keys 2>/dev/null || echo '{pub_key}'
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"SSH 执行失败: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CLI Runner
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""SSH 密钥部署工具主函数."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="SSHCopyID - SSH 密钥部署工具",
|
||||
usage="sshcopyid <hostname> <username> <password> [--port PORT] [--keypath KEYPATH]",
|
||||
)
|
||||
parser.add_argument("hostname", type=str, help="远程服务器主机名或 IP 地址")
|
||||
parser.add_argument("username", type=str, help="远程服务器用户名")
|
||||
parser.add_argument("password", type=str, help="远程服务器密码")
|
||||
parser.add_argument("--port", type=int, default=22, help="SSH 端口 (默认: 22)")
|
||||
parser.add_argument("--keypath", type=str, default="~/.ssh/id_rsa.pub", help="公钥文件路径")
|
||||
parser.add_argument("--timeout", type=int, default=30, help="SSH 操作超时秒数 (默认: 30)")
|
||||
args = parser.parse_args()
|
||||
|
||||
graph = px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec(
|
||||
"ssh_deploy",
|
||||
fn=ssh_copy_id,
|
||||
args=(args.hostname, args.username, args.password),
|
||||
kwargs={"port": args.port, "keypath": args.keypath, "timeout": args.timeout},
|
||||
)
|
||||
]
|
||||
)
|
||||
px.run(graph, strategy="thread")
|
||||
@@ -0,0 +1,33 @@
|
||||
"""taskkill - 进程终止工具.
|
||||
|
||||
跨平台按名称终止进程: Windows 用 ``taskkill``, Linux/macOS 用 ``pkill``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
@px.tool("taskkill", help="按名称终止进程 (跨平台)")
|
||||
def taskkill_run(process_names: list[str]) -> None:
|
||||
"""按名称终止进程 (跨平台).
|
||||
|
||||
Windows 使用 ``taskkill /f /im <name>*``,
|
||||
Linux/macOS 使用 ``pkill -f <name>*``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
process_names : list[str]
|
||||
进程名称列表 (如: ``["chrome.exe", "python"]``)
|
||||
"""
|
||||
if Constants.IS_WINDOWS:
|
||||
cmd_prefix: list[str] = ["taskkill", "/f", "/im"]
|
||||
else:
|
||||
cmd_prefix = ["pkill", "-f"]
|
||||
|
||||
for name in process_names:
|
||||
print(f"终止进程: {name}")
|
||||
subprocess.run([*cmd_prefix, f"{name}*"], check=False)
|
||||
@@ -0,0 +1,35 @@
|
||||
"""which - 命令查找工具.
|
||||
|
||||
跨平台查找可执行命令路径: Windows 用 ``where``, Linux/macOS 用 ``which``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.conditions import Constants
|
||||
|
||||
|
||||
@px.tool("which", help="查找可执行命令路径 (跨平台)")
|
||||
def which_run(commands: list[str]) -> None:
|
||||
"""查找可执行命令路径 (跨平台).
|
||||
|
||||
Windows 使用 ``where``, Linux/macOS 使用 ``which``.
|
||||
对每个命令打印 ``<cmd> -> <path>`` 或 ``<cmd> -> 未找到``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
commands : list[str]
|
||||
要查找的命令名称列表
|
||||
"""
|
||||
which_cmd = "where" if Constants.IS_WINDOWS else "which"
|
||||
|
||||
for cmd in commands:
|
||||
result = subprocess.run([which_cmd, cmd], capture_output=True, text=True, check=False)
|
||||
if result.returncode == 0:
|
||||
# Windows 的 where 可能返回多行, 取第一个
|
||||
path = result.stdout.strip().split("\n")[0].strip()
|
||||
print(f"{cmd} -> {path}")
|
||||
else:
|
||||
print(f"{cmd} -> 未找到")
|
||||
@@ -0,0 +1,705 @@
|
||||
"""工作流执行性能评估。
|
||||
|
||||
基于 :class:`~pyflowx.report.RunReport` 中已有的 ``started_at`` /
|
||||
``finished_at`` 时间戳进行离线分析,**零运行时开销**——不修改执行流程,
|
||||
不注册回调,不引入额外计时器。
|
||||
|
||||
核心指标
|
||||
--------
|
||||
* **任务级**:每个任务的 wall-clock 耗时、状态、重试次数、等待时间
|
||||
(从最早依赖完成到本任务开始)。
|
||||
* **图级**:总耗时(wall-clock)、关键路径耗时(理论最短耗时)、
|
||||
并行度效率(关键路径耗时 / 总耗时)。
|
||||
* **关键路径**:从源点到汇点的最长依赖路径,识别真正的串行瓶颈。
|
||||
* **并行度**:基于时间线重叠计算瞬时并行度,给出平均并行度与峰值并行度。
|
||||
* **瓶颈识别**:按耗时排序的 Top-N 任务。
|
||||
|
||||
设计原则
|
||||
--------
|
||||
* 数据来源于 ``RunReport`` + ``Graph``,无副作用。
|
||||
* 计算复杂度 O(V+E):拓扑排序 + 单次松弛,适合大规模图。
|
||||
* 所有时间戳用 ``datetime``,与 :class:`TaskResult` 保持一致。
|
||||
|
||||
快速上手
|
||||
--------
|
||||
import pyflowx as px
|
||||
|
||||
report = px.run(graph)
|
||||
profile = px.ProfileReport.from_report(report, graph)
|
||||
print(profile.describe())
|
||||
bottlenecks = profile.top_bottlenecks(3)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = [
|
||||
"ProfileReport",
|
||||
"TaskProfile",
|
||||
]
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from .graph import Graph
|
||||
from .report import RunReport
|
||||
from .task import TaskResult, TaskStatus
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TaskProfile:
|
||||
"""单个任务的性能剖面。
|
||||
|
||||
属性
|
||||
----
|
||||
name:
|
||||
任务名。
|
||||
status:
|
||||
终态(SUCCESS/FAILED/SKIPPED)。
|
||||
duration:
|
||||
wall-clock 执行耗时(秒)。SKIPPED 任务为 0.0。
|
||||
attempts:
|
||||
尝试次数(含首次)。
|
||||
wait_time:
|
||||
从最早硬依赖完成到本任务开始的等待时间(秒)。
|
||||
无硬依赖或 SKIPPED 时为 0.0。
|
||||
is_on_critical_path:
|
||||
是否位于关键路径上。
|
||||
deps:
|
||||
硬依赖任务名列表。
|
||||
"""
|
||||
|
||||
name: str
|
||||
status: TaskStatus
|
||||
duration: float
|
||||
attempts: int
|
||||
wait_time: float
|
||||
is_on_critical_path: bool
|
||||
deps: tuple[str, ...]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""转为 JSON 友好的字典。"""
|
||||
return {
|
||||
"name": self.name,
|
||||
"status": self.status.value,
|
||||
"duration_seconds": round(self.duration, 6),
|
||||
"attempts": self.attempts,
|
||||
"wait_time_seconds": round(self.wait_time, 6),
|
||||
"is_on_critical_path": self.is_on_critical_path,
|
||||
"deps": list(self.deps),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProfileReport:
|
||||
"""工作流执行的性能剖面报告。
|
||||
|
||||
通过 :meth:`from_report` 从 :class:`RunReport` + :class:`Graph` 构建。
|
||||
所有字段在构造时一次性计算完毕,后续访问为 O(1)。
|
||||
"""
|
||||
|
||||
tasks: tuple[TaskProfile, ...]
|
||||
"""所有任务的性能剖面(按拓扑序)。"""
|
||||
|
||||
total_duration: float
|
||||
"""整次运行的 wall-clock 耗时(秒)。"""
|
||||
|
||||
critical_path_duration: float
|
||||
"""关键路径耗时(秒):从最早任务开始到最晚任务结束的最长依赖路径。"""
|
||||
|
||||
critical_path: tuple[str, ...]
|
||||
"""关键路径上的任务名序列(按执行顺序)。"""
|
||||
|
||||
avg_parallelism: float
|
||||
"""平均并行度 = 任务总耗时 / wall-clock 总耗时。"""
|
||||
|
||||
peak_parallelism: int
|
||||
"""峰值并行度:任一时刻同时运行的任务数最大值。"""
|
||||
|
||||
parallelism_efficiency: float
|
||||
"""并行度效率 = 关键路径耗时 / wall-clock 总耗时。``1.0`` 表示完全串行,
|
||||
越大表示并行化收益越低(瓶颈在关键路径上)。"""
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 构建
|
||||
# ------------------------------------------------------------------ #
|
||||
@classmethod
|
||||
def from_report(cls, report: RunReport, graph: Graph) -> ProfileReport:
|
||||
"""从运行报告与图构建性能剖面。
|
||||
|
||||
参数
|
||||
----
|
||||
report:
|
||||
已完成的 :class:`RunReport`,需包含 ``started_at``/``finished_at``。
|
||||
graph:
|
||||
对应的 :class:`Graph`,用于依赖关系与关键路径分析。
|
||||
|
||||
Note
|
||||
-----
|
||||
本方法不修改 ``report`` 或 ``graph``,纯函数式计算。
|
||||
"""
|
||||
task_profiles = cls._build_task_profiles(report, graph)
|
||||
total_duration = cls._calc_total_duration(report)
|
||||
critical_path, critical_duration = cls._calc_critical_path(graph, report)
|
||||
avg_par, peak_par = cls._calc_parallelism(report)
|
||||
efficiency = critical_duration / total_duration if total_duration > 0 else 0.0
|
||||
|
||||
# 标记关键路径上的任务
|
||||
critical_set = set(critical_path)
|
||||
marked = tuple(
|
||||
TaskProfile(
|
||||
name=t.name,
|
||||
status=t.status,
|
||||
duration=t.duration,
|
||||
attempts=t.attempts,
|
||||
wait_time=t.wait_time,
|
||||
is_on_critical_path=t.name in critical_set,
|
||||
deps=t.deps,
|
||||
)
|
||||
for t in task_profiles
|
||||
)
|
||||
|
||||
return cls(
|
||||
tasks=marked,
|
||||
total_duration=total_duration,
|
||||
critical_path_duration=critical_duration,
|
||||
critical_path=critical_path,
|
||||
avg_parallelism=avg_par,
|
||||
peak_parallelism=peak_par,
|
||||
parallelism_efficiency=efficiency,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_task_profiles(report: RunReport, graph: Graph) -> tuple[TaskProfile, ...]:
|
||||
"""构建每个任务的性能剖面。"""
|
||||
profiles: list[TaskProfile] = []
|
||||
for name, result in report.results.items():
|
||||
spec = graph.specs.get(name)
|
||||
deps = tuple(spec.depends_on) if spec is not None else ()
|
||||
duration = result.duration or 0.0
|
||||
wait_time = ProfileReport._calc_wait_time(result, deps, report)
|
||||
profiles.append(
|
||||
TaskProfile(
|
||||
name=name,
|
||||
status=result.status,
|
||||
duration=duration,
|
||||
attempts=result.attempts,
|
||||
wait_time=wait_time,
|
||||
is_on_critical_path=False, # 后续标记
|
||||
deps=deps,
|
||||
)
|
||||
)
|
||||
return tuple(profiles)
|
||||
|
||||
@staticmethod
|
||||
def _calc_wait_time(
|
||||
result: TaskResult[Any],
|
||||
deps: tuple[str, ...],
|
||||
report: RunReport,
|
||||
) -> float:
|
||||
"""计算等待时间:从最早依赖完成到本任务开始。
|
||||
|
||||
无硬依赖、SKIPPED 任务或时间戳缺失时返回 0.0。
|
||||
"""
|
||||
if not deps or result.started_at is None or result.status == TaskStatus.SKIPPED:
|
||||
return 0.0
|
||||
# 找出所有已完成依赖的最晚完成时间
|
||||
dep_end_times: list[datetime] = []
|
||||
for dep in deps:
|
||||
dep_result = report.results.get(dep)
|
||||
if dep_result is not None and dep_result.finished_at is not None:
|
||||
dep_end_times.append(dep_result.finished_at)
|
||||
if not dep_end_times:
|
||||
return 0.0
|
||||
latest_dep_end = max(dep_end_times)
|
||||
delta = (result.started_at - latest_dep_end).total_seconds()
|
||||
return max(0.0, delta)
|
||||
|
||||
@staticmethod
|
||||
def _calc_total_duration(report: RunReport) -> float:
|
||||
"""计算 wall-clock 总耗时:最早开始到最晚结束。"""
|
||||
starts: list[datetime] = []
|
||||
ends: list[datetime] = []
|
||||
for r in report.results.values():
|
||||
if r.started_at is not None:
|
||||
starts.append(r.started_at)
|
||||
if r.finished_at is not None:
|
||||
ends.append(r.finished_at)
|
||||
if not starts or not ends:
|
||||
return 0.0
|
||||
return (max(ends) - min(starts)).total_seconds()
|
||||
|
||||
@staticmethod
|
||||
def _calc_critical_path(graph: Graph, report: RunReport) -> tuple[tuple[str, ...], float]:
|
||||
"""计算关键路径:DAG 最长路径(按实际执行耗时)。
|
||||
|
||||
使用拓扑排序 + 动态规划,O(V+E)。SKIPPED 任务耗时按 0 计。
|
||||
"""
|
||||
# 构建耗时映射
|
||||
durations: dict[str, float] = {}
|
||||
for name, result in report.results.items():
|
||||
durations[name] = result.duration or 0.0
|
||||
|
||||
# 拓扑序(使用 graph.layers 保证与分层一致)
|
||||
try:
|
||||
layers = graph.layers()
|
||||
except Exception:
|
||||
# 图校验失败时回退为空
|
||||
return (), 0.0
|
||||
|
||||
# earliest_finish[name] = duration[name] + max(earliest_finish[dep] for dep in deps)
|
||||
earliest_finish: dict[str, float] = {}
|
||||
predecessor: dict[str, str | None] = {}
|
||||
|
||||
for layer in layers:
|
||||
for name in layer:
|
||||
spec = graph.specs.get(name)
|
||||
deps = spec.depends_on if spec is not None else ()
|
||||
if not deps:
|
||||
earliest_finish[name] = durations.get(name, 0.0)
|
||||
predecessor[name] = None
|
||||
else:
|
||||
best_dep: str | None = None
|
||||
best_ef = 0.0
|
||||
for dep in deps:
|
||||
ef = earliest_finish.get(dep, 0.0)
|
||||
if ef >= best_ef:
|
||||
best_ef = ef
|
||||
best_dep = dep
|
||||
earliest_finish[name] = best_ef + durations.get(name, 0.0)
|
||||
predecessor[name] = best_dep
|
||||
|
||||
if not earliest_finish:
|
||||
return (), 0.0
|
||||
|
||||
# 找到 earliest_finish 最大的节点作为终点
|
||||
end_node = max(earliest_finish, key=lambda n: earliest_finish[n])
|
||||
total = earliest_finish[end_node]
|
||||
|
||||
# 回溯关键路径
|
||||
path: list[str] = []
|
||||
node: str | None = end_node
|
||||
while node is not None:
|
||||
path.append(node)
|
||||
node = predecessor.get(node)
|
||||
path.reverse()
|
||||
|
||||
return tuple(path), total
|
||||
|
||||
@staticmethod
|
||||
def _calc_parallelism(report: RunReport) -> tuple[float, int]:
|
||||
"""计算平均并行度与峰值并行度。
|
||||
|
||||
基于时间线扫描:将每个任务的 [started_at, finished_at] 区间
|
||||
转为事件点(+1/-1),排序后扫描得到瞬时并行度序列。
|
||||
|
||||
返回 (avg_parallelism, peak_parallelism)。
|
||||
无有效时间戳时返回 (0.0, 0)。
|
||||
"""
|
||||
events: list[tuple[float, int]] = [] # (timestamp, delta)
|
||||
for r in report.results.values():
|
||||
if r.started_at is None or r.finished_at is None:
|
||||
continue
|
||||
if r.status == TaskStatus.SKIPPED:
|
||||
continue
|
||||
start_ts = r.started_at.timestamp()
|
||||
end_ts = r.finished_at.timestamp()
|
||||
if end_ts <= start_ts:
|
||||
continue
|
||||
events.append((start_ts, 1))
|
||||
events.append((end_ts, -1))
|
||||
|
||||
if not events:
|
||||
return 0.0, 0
|
||||
|
||||
# 排序:同一时间点先处理结束(-1)再处理开始(+1),避免虚假峰值
|
||||
events.sort(key=lambda e: (e[0], e[1]))
|
||||
|
||||
current = 0
|
||||
peak = 0
|
||||
# 加权面积用于计算平均并行度
|
||||
area = 0.0
|
||||
prev_ts = events[0][0]
|
||||
for ts, delta in events:
|
||||
if ts > prev_ts:
|
||||
area += current * (ts - prev_ts)
|
||||
current += delta
|
||||
peak = max(peak, current)
|
||||
prev_ts = ts
|
||||
|
||||
total_span = events[-1][0] - events[0][0]
|
||||
avg = area / total_span if total_span > 0 else 0.0
|
||||
return avg, peak
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 查询
|
||||
# ------------------------------------------------------------------ #
|
||||
def task(self, name: str) -> TaskProfile:
|
||||
"""返回指定任务的剖面。不存在则 ``KeyError``。"""
|
||||
for t in self.tasks:
|
||||
if t.name == name:
|
||||
return t
|
||||
raise KeyError(name)
|
||||
|
||||
def top_bottlenecks(self, n: int = 5) -> tuple[TaskProfile, ...]:
|
||||
"""返回耗时最长的 Top-N 任务(按 duration 降序)。
|
||||
|
||||
参数
|
||||
----
|
||||
n:
|
||||
返回数量。``n <= 0`` 返回空元组。
|
||||
"""
|
||||
if n <= 0:
|
||||
return ()
|
||||
return tuple(sorted(self.tasks, key=lambda t: t.duration, reverse=True)[:n])
|
||||
|
||||
def critical_tasks(self) -> tuple[TaskProfile, ...]:
|
||||
"""返回关键路径上的所有任务(按路径顺序)。"""
|
||||
critical_set = set(self.critical_path)
|
||||
# 保持关键路径顺序
|
||||
order = {name: i for i, name in enumerate(self.critical_path)}
|
||||
return tuple(sorted((t for t in self.tasks if t.name in critical_set), key=lambda t: order[t.name]))
|
||||
|
||||
def failed_tasks(self) -> tuple[TaskProfile, ...]:
|
||||
"""返回 FAILED 状态的任务。"""
|
||||
return tuple(t for t in self.tasks if t.status == TaskStatus.FAILED)
|
||||
|
||||
def skipped_tasks(self) -> tuple[TaskProfile, ...]:
|
||||
"""返回 SKIPPED 状态的任务。"""
|
||||
return tuple(t for t in self.tasks if t.status == TaskStatus.SKIPPED)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 输出
|
||||
# ------------------------------------------------------------------ #
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""转为 JSON 友好的字典。"""
|
||||
return {
|
||||
"tasks": [t.to_dict() for t in self.tasks],
|
||||
"total_duration_seconds": round(self.total_duration, 6),
|
||||
"critical_path_duration_seconds": round(self.critical_path_duration, 6),
|
||||
"critical_path": list(self.critical_path),
|
||||
"avg_parallelism": round(self.avg_parallelism, 4),
|
||||
"peak_parallelism": self.peak_parallelism,
|
||||
"parallelism_efficiency": round(self.parallelism_efficiency, 4),
|
||||
"bottlenecks": [t.to_dict() for t in self.top_bottlenecks(5)],
|
||||
}
|
||||
|
||||
def to_html(self) -> str:
|
||||
"""生成自包含的 HTML 报告(含 CSS,无外部依赖)。
|
||||
|
||||
报告含:图级指标卡片、关键路径、时间线甘特图、Top 瓶颈表格、
|
||||
全部任务表格。适合直接用浏览器打开查看。
|
||||
"""
|
||||
return _render_html(self)
|
||||
|
||||
def describe(self) -> str:
|
||||
lines: list[str] = []
|
||||
lines.append("=" * 70)
|
||||
lines.append("PyFlowX 性能剖面报告")
|
||||
lines.append("=" * 70)
|
||||
lines.append("")
|
||||
lines.append("【图级指标】")
|
||||
lines.append(f" 总耗时 (wall-clock): {self.total_duration:.3f}s")
|
||||
lines.append(f" 关键路径耗时: {self.critical_path_duration:.3f}s")
|
||||
lines.append(f" 平均并行度: {self.avg_parallelism:.2f}")
|
||||
lines.append(f" 峰值并行度: {self.peak_parallelism}")
|
||||
lines.append(f" 并行度效率: {self.parallelism_efficiency:.2%}")
|
||||
lines.append(f" 任务总数: {len(self.tasks)}")
|
||||
lines.append("")
|
||||
|
||||
# 关键路径
|
||||
lines.append("【关键路径】")
|
||||
if self.critical_path:
|
||||
lines.append(f" {' -> '.join(self.critical_path)}")
|
||||
else:
|
||||
lines.append(" (无)")
|
||||
lines.append("")
|
||||
|
||||
# Top 瓶颈
|
||||
bottlenecks = self.top_bottlenecks(5)
|
||||
lines.append(f"【Top {len(bottlenecks)} 瓶颈任务】")
|
||||
if bottlenecks:
|
||||
lines.append(f" {'任务':<30} {'耗时':>10} {'等待':>10} {'尝试':>6} {'关键路径':>8} {'状态':>8}")
|
||||
lines.append(f" {'-' * 30} {'-' * 10} {'-' * 10} {'-' * 6} {'-' * 8} {'-' * 8}")
|
||||
for t in bottlenecks:
|
||||
critical_flag = "✓" if t.is_on_critical_path else ""
|
||||
lines.append(
|
||||
f" {t.name:<30} {t.duration:>9.3f}s {t.wait_time:>9.3f}s {t.attempts:>6} "
|
||||
f"{critical_flag:>8} {t.status.value:>8}",
|
||||
)
|
||||
else:
|
||||
lines.append(" (无)")
|
||||
lines.append("")
|
||||
|
||||
# 全部任务详情
|
||||
lines.append("【全部任务】")
|
||||
if self.tasks:
|
||||
lines.append(f" {'任务':<30} {'耗时':>10} {'等待':>10} {'尝试':>6} {'关键路径':>8} {'状态':>8}")
|
||||
lines.append(f" {'-' * 30} {'-' * 10} {'-' * 10} {'-' * 6} {'-' * 8} {'-' * 8}")
|
||||
for t in self.tasks:
|
||||
critical_flag = "✓" if t.is_on_critical_path else ""
|
||||
lines.append(
|
||||
f" {t.name:<30} {t.duration:>9.3f}s {t.wait_time:>9.3f}s {t.attempts:>6} "
|
||||
f"{critical_flag:>8} {t.status.value:>8}",
|
||||
)
|
||||
else:
|
||||
lines.append(" (无)")
|
||||
lines.append("")
|
||||
lines.append("=" * 70)
|
||||
return "\n".join(lines)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"ProfileReport(tasks={len(self.tasks)}, "
|
||||
f"total={self.total_duration:.3f}s, "
|
||||
f"critical={self.critical_path_duration:.3f}s, "
|
||||
f"avg_par={self.avg_parallelism:.2f}, "
|
||||
f"peak_par={self.peak_parallelism})"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# HTML 渲染(私有,零依赖)
|
||||
# ---------------------------------------------------------------------- #
|
||||
_HTML_TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>PyFlowX 性能剖面报告</title>
|
||||
<style>
|
||||
:root {{
|
||||
--bg: #f5f5f7;
|
||||
--card: #ffffff;
|
||||
--border: #d2d2d7;
|
||||
--text: #1d1d1f;
|
||||
--muted: #6e6e73;
|
||||
--accent: #0071e3;
|
||||
--success: #34c759;
|
||||
--warning: #ff9f0a;
|
||||
--danger: #ff3b30;
|
||||
--critical: #af52de;
|
||||
}}
|
||||
* {{ box-sizing: border-box; }}
|
||||
body {{
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
margin: 0;
|
||||
padding: 24px;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
line-height: 1.5;
|
||||
}}
|
||||
h1 {{ margin: 0 0 8px; font-size: 28px; }}
|
||||
h2 {{ margin: 32px 0 12px; font-size: 20px; border-bottom: 1px solid var(--border); padding-bottom: 6px; }}
|
||||
.subtitle {{ color: var(--muted); margin: 0 0 24px; font-size: 14px; }}
|
||||
.cards {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 12px; margin-bottom: 8px; }}
|
||||
.card {{
|
||||
background: var(--card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 10px;
|
||||
padding: 16px;
|
||||
}}
|
||||
.card .label {{ font-size: 12px; color: var(--muted); margin-bottom: 4px; text-transform: uppercase; letter-spacing: 0.5px; }}
|
||||
.card .value {{ font-size: 22px; font-weight: 600; }}
|
||||
.card .unit {{ font-size: 13px; color: var(--muted); margin-left: 2px; }}
|
||||
.critical-path {{
|
||||
background: var(--card);
|
||||
border: 1px solid var(--border);
|
||||
border-left: 4px solid var(--critical);
|
||||
border-radius: 10px;
|
||||
padding: 16px;
|
||||
margin-bottom: 8px;
|
||||
}}
|
||||
.critical-path .label {{ font-size: 12px; color: var(--muted); margin-bottom: 8px; text-transform: uppercase; letter-spacing: 0.5px; }}
|
||||
.critical-path .chain {{ font-family: ui-monospace, "SF Mono", Menlo, monospace; font-size: 13px; word-break: break-all; }}
|
||||
.critical-path .arrow {{ color: var(--critical); margin: 0 6px; font-weight: 600; }}
|
||||
/* 甘特图 */
|
||||
.gantt {{
|
||||
background: var(--card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 10px;
|
||||
padding: 16px;
|
||||
overflow-x: auto;
|
||||
}}
|
||||
.gantt-row {{ display: flex; align-items: center; margin-bottom: 6px; min-width: 600px; }}
|
||||
.gantt-label {{ width: 200px; flex-shrink: 0; font-size: 13px; font-family: ui-monospace, monospace; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }}
|
||||
.gantt-track {{ flex: 1; height: 22px; background: #f0f0f3; border-radius: 4px; position: relative; }}
|
||||
.gantt-bar {{ position: absolute; height: 100%; border-radius: 4px; min-width: 2px; }}
|
||||
.gantt-bar.success {{ background: var(--success); }}
|
||||
.gantt-bar.failed {{ background: var(--danger); }}
|
||||
.gantt-bar.skipped {{ background: var(--muted); }}
|
||||
.gantt-bar.critical {{ box-shadow: 0 0 0 2px var(--critical) inset; }}
|
||||
.gantt-bar:hover {{ opacity: 0.85; }}
|
||||
.gantt-tooltip {{ position: absolute; bottom: 100%; left: 50%; transform: translateX(-50%); background: #1d1d1f; color: #fff; padding: 4px 8px; border-radius: 4px; font-size: 11px; white-space: nowrap; opacity: 0; pointer-events: none; transition: opacity 0.15s; }}
|
||||
.gantt-bar:hover .gantt-tooltip {{ opacity: 1; }}
|
||||
/* 表格 */
|
||||
table {{ width: 100%; border-collapse: collapse; background: var(--card); border-radius: 10px; overflow: hidden; border: 1px solid var(--border); }}
|
||||
th, td {{ padding: 10px 12px; text-align: left; font-size: 13px; }}
|
||||
th {{ background: #fafafa; font-weight: 600; color: var(--muted); text-transform: uppercase; font-size: 11px; letter-spacing: 0.5px; }}
|
||||
tbody tr {{ border-top: 1px solid var(--border); }}
|
||||
tbody tr:hover {{ background: #fafafa; }}
|
||||
td.num {{ font-family: ui-monospace, monospace; text-align: right; }}
|
||||
.badge {{ display: inline-block; padding: 2px 8px; border-radius: 10px; font-size: 11px; font-weight: 500; }}
|
||||
.badge.success {{ background: rgba(52,199,89,0.15); color: var(--success); }}
|
||||
.badge.failed {{ background: rgba(255,59,48,0.15); color: var(--danger); }}
|
||||
.badge.skipped {{ background: rgba(110,110,115,0.15); color: var(--muted); }}
|
||||
.star {{ color: var(--critical); font-weight: 700; }}
|
||||
.footer {{ margin-top: 32px; color: var(--muted); font-size: 12px; text-align: center; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>PyFlowX 性能剖面报告</h1>
|
||||
<p class="subtitle">由 <code>pxp</code> 生成 · {generated_at}</p>
|
||||
|
||||
<h2>图级指标</h2>
|
||||
<div class="cards">
|
||||
<div class="card"><div class="label">总耗时</div><div class="value">{total_duration:.3f}<span class="unit">s</span></div></div>
|
||||
<div class="card"><div class="label">关键路径耗时</div><div class="value">{critical_duration:.3f}<span class="unit">s</span></div></div>
|
||||
<div class="card"><div class="label">平均并行度</div><div class="value">{avg_par:.2f}</div></div>
|
||||
<div class="card"><div class="label">峰值并行度</div><div class="value">{peak_par}</div></div>
|
||||
<div class="card"><div class="label">并行度效率</div><div class="value">{efficiency:.1f}<span class="unit">%</span></div></div>
|
||||
<div class="card"><div class="label">任务总数</div><div class="value">{task_count}</div></div>
|
||||
</div>
|
||||
|
||||
<h2>关键路径</h2>
|
||||
<div class="critical-path">
|
||||
<div class="label">最长依赖路径(串行瓶颈)</div>
|
||||
<div class="chain">{critical_chain}</div>
|
||||
</div>
|
||||
|
||||
<h2>任务时间线</h2>
|
||||
<div class="gantt">
|
||||
{gantt_rows}
|
||||
</div>
|
||||
|
||||
<h2>Top 瓶颈任务</h2>
|
||||
<table>
|
||||
<thead><tr><th>任务</th><th class="num">耗时</th><th class="num">等待</th><th class="num">尝试</th><th>关键路径</th><th>状态</th></tr></thead>
|
||||
<tbody>
|
||||
{bottleneck_rows}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<h2>全部任务</h2>
|
||||
<table>
|
||||
<thead><tr><th>任务</th><th class="num">耗时</th><th class="num">等待</th><th class="num">尝试</th><th>关键路径</th><th>状态</th><th>依赖</th></tr></thead>
|
||||
<tbody>
|
||||
{all_task_rows}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<div class="footer">由 PyFlowX · pxp 生成</div>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
def _status_badge(status: TaskStatus) -> str:
|
||||
"""生成状态徽章 HTML。"""
|
||||
cls = status.value
|
||||
return f'<span class="badge {cls}">{cls}</span>'
|
||||
|
||||
|
||||
def _format_critical_chain(path: tuple[str, ...]) -> str:
|
||||
"""格式化关键路径为 HTML 链。"""
|
||||
if not path:
|
||||
return '<em style="color:var(--muted)">(无)</em>'
|
||||
arrow = '<span class="arrow">→</span>'
|
||||
return arrow.join(f"<strong>{name}</strong>" for name in path)
|
||||
|
||||
|
||||
def _render_gantt(profile: ProfileReport) -> str:
|
||||
"""渲染甘特图行 HTML。
|
||||
|
||||
每个任务一行:标签 + 时间条。时间条位置基于 wait_time + 依赖关系
|
||||
重建相对开始时间(相对最早任务起点),归一化到 0-100% 宽度。
|
||||
SKIPPED 任务不显示(无时间戳)。
|
||||
"""
|
||||
visible = [t for t in profile.tasks if t.status != TaskStatus.SKIPPED and t.duration > 0]
|
||||
if not visible:
|
||||
return '<div style="color:var(--muted);padding:12px;">(无时间线数据)</div>'
|
||||
|
||||
# 重建相对开始时间:start[name] = max(end[dep]) + wait_time
|
||||
# profile.tasks 已是拓扑序,可直接按序计算
|
||||
start: dict[str, float] = {}
|
||||
end: dict[str, float] = {}
|
||||
for t in profile.tasks:
|
||||
if t.status == TaskStatus.SKIPPED:
|
||||
continue
|
||||
dep_end = 0.0
|
||||
for dep in t.deps:
|
||||
dep_end = max(dep_end, end.get(dep, 0.0))
|
||||
s = dep_end + t.wait_time
|
||||
start[t.name] = s
|
||||
end[t.name] = s + t.duration
|
||||
|
||||
# 归一化:以最早开始时间为 0,最晚结束为 100%
|
||||
min_start = min(start.get(t.name, 0.0) for t in visible)
|
||||
max_end = max(end.get(t.name, 0.0) for t in visible)
|
||||
span = max_end - min_start
|
||||
if span <= 0:
|
||||
span = 1.0
|
||||
|
||||
rows: list[str] = []
|
||||
for t in visible:
|
||||
s = start.get(t.name, 0.0) - min_start
|
||||
left_pct = (s / span) * 100
|
||||
width_pct = (t.duration / span) * 100
|
||||
cls = t.status.value
|
||||
critical_cls = " critical" if t.is_on_critical_path else ""
|
||||
tooltip = f"{t.name}: {t.duration:.3f}s @ +{s:.3f}s ({t.status.value})"
|
||||
rows.append(
|
||||
f' <div class="gantt-row">'
|
||||
f'<div class="gantt-label" title="{t.name}">{t.name}</div>'
|
||||
f'<div class="gantt-track">'
|
||||
f'<div class="gantt-bar {cls}{critical_cls}" style="left:{left_pct:.2f}%;width:{width_pct:.2f}%">'
|
||||
f'<span class="gantt-tooltip">{tooltip}</span>'
|
||||
f"</div></div></div>"
|
||||
)
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def _render_task_row(t: TaskProfile, show_deps: bool = False) -> str:
|
||||
"""渲染任务表格行 HTML。"""
|
||||
star = '<span class="star">★</span>' if t.is_on_critical_path else ""
|
||||
deps = ", ".join(t.deps) if show_deps and t.deps else ""
|
||||
deps_cell = f"<td>{deps}</td>" if show_deps else ""
|
||||
return (
|
||||
f" <tr>"
|
||||
f"<td><code>{t.name}</code></td>"
|
||||
f'<td class="num">{t.duration:.3f}s</td>'
|
||||
f'<td class="num">{t.wait_time:.3f}s</td>'
|
||||
f'<td class="num">{t.attempts}</td>'
|
||||
f"<td>{star}</td>"
|
||||
f"<td>{_status_badge(t.status)}</td>"
|
||||
f"{deps_cell}"
|
||||
f"</tr>"
|
||||
)
|
||||
|
||||
|
||||
def _render_html(profile: ProfileReport) -> str:
|
||||
"""渲染完整 HTML 报告。"""
|
||||
from datetime import datetime as _dt
|
||||
|
||||
bottlenecks = profile.top_bottlenecks(5)
|
||||
bottleneck_rows = (
|
||||
"\n".join(_render_task_row(t) for t in bottlenecks)
|
||||
or ' <tr><td colspan="6" style="color:var(--muted);">(无)</td></tr>'
|
||||
)
|
||||
all_task_rows = (
|
||||
"\n".join(_render_task_row(t, show_deps=True) for t in profile.tasks)
|
||||
or ' <tr><td colspan="7" style="color:var(--muted);">(无)</td></tr>'
|
||||
)
|
||||
|
||||
return _HTML_TEMPLATE.format(
|
||||
generated_at=_dt.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
total_duration=profile.total_duration,
|
||||
critical_duration=profile.critical_path_duration,
|
||||
avg_par=profile.avg_parallelism,
|
||||
peak_par=profile.peak_parallelism,
|
||||
efficiency=profile.parallelism_efficiency * 100,
|
||||
task_count=len(profile.tasks),
|
||||
critical_chain=_format_critical_chain(profile.critical_path),
|
||||
gantt_rows=_render_gantt(profile),
|
||||
bottleneck_rows=bottleneck_rows,
|
||||
all_task_rows=all_task_rows,
|
||||
)
|
||||
@@ -69,6 +69,22 @@ class RunReport:
|
||||
"""以 FAILED 状态结束的任务名列表。"""
|
||||
return [name for name, r in self.results.items() if r.status == TaskStatus.FAILED]
|
||||
|
||||
def succeeded_tasks(self) -> list[str]:
|
||||
"""以 SUCCESS 状态结束的任务名列表。"""
|
||||
return [name for name, r in self.results.items() if r.status == TaskStatus.SUCCESS]
|
||||
|
||||
def skipped_tasks(self) -> list[str]:
|
||||
"""以 SKIPPED 状态结束的任务名列表。"""
|
||||
return [name for name, r in self.results.items() if r.status == TaskStatus.SKIPPED]
|
||||
|
||||
def tasks_by_status(self, status: TaskStatus) -> list[str]:
|
||||
"""返回指定状态的任务名列表。"""
|
||||
return [name for name, r in self.results.items() if r.status == status]
|
||||
|
||||
def durations(self) -> dict[str, float]:
|
||||
"""任务名 -> 执行时长(秒)。无时长记录的为 0.0。"""
|
||||
return {name: (r.duration or 0.0) for name, r in self.results.items()}
|
||||
|
||||
def describe(self) -> str:
|
||||
"""用于调试的人类可读多行报告。"""
|
||||
lines: list[str] = [f"RunReport(success={self.success})"]
|
||||
|
||||
+101
-79
@@ -14,12 +14,14 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import enum
|
||||
import sys
|
||||
from dataclasses import dataclass, field, replace
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Sequence, get_args
|
||||
|
||||
from .compose import GraphComposer
|
||||
from .errors import PyFlowXError
|
||||
from .executors import Strategy, run
|
||||
from .graph import Graph, GraphComposer
|
||||
from .graph import Graph
|
||||
from .task import TaskSpec
|
||||
|
||||
__all__ = ["CliExitCode", "CliRunner"]
|
||||
@@ -33,113 +35,137 @@ class CliExitCode(enum.IntEnum):
|
||||
INTERRUPTED = 130 # 与 POSIX 信号中断一致
|
||||
|
||||
|
||||
def _apply_verbose_to_graph(graph: Graph, verbose: bool) -> Graph:
|
||||
"""创建新图, 其中所有 TaskSpec 的 verbose 字段被设置为指定值.
|
||||
|
||||
使用 ``dataclasses.replace`` 在不可变的 TaskSpec 上创建带 verbose 标记的副本.
|
||||
依赖关系、标签等元数据全部保留.
|
||||
|
||||
Note
|
||||
-----
|
||||
自 ``_wrap_cmd`` 不再闭包捕获 ``verbose`` 后,此函数不再是必需的——
|
||||
直接翻转 ``spec.verbose`` 即可生效。保留是为了向后兼容现有调用与测试。
|
||||
TaskSpec 仍是 frozen dataclass,故仍用 ``replace`` 创建副本。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph : Graph
|
||||
原始图.
|
||||
verbose : bool
|
||||
要设置的 verbose 值.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Graph
|
||||
所有 spec 的 verbose 字段已更新的新图.
|
||||
"""
|
||||
new_specs: list[TaskSpec[Any]] = []
|
||||
for spec in graph.all_specs().values():
|
||||
if spec.verbose == verbose:
|
||||
new_specs.append(spec)
|
||||
else:
|
||||
new_specs.append(replace(spec, verbose=verbose))
|
||||
return Graph.from_specs(new_specs)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CliRunner:
|
||||
"""命令行运行器: 根据用户输入执行对应的任务流图.
|
||||
|
||||
将命令名映射到 Graph 实例.
|
||||
通过 ``sys.argv`` 解析用户输入的命令, 执行对应的图.
|
||||
将命令别名映射到 Graph 实例. 通过 ``sys.argv`` 解析用户输入的命令,
|
||||
执行对应的图.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
aliases : dict[str, str | list[str] | Graph]
|
||||
命令别名到任务引用的映射. 每个值可以是:
|
||||
* ``str`` —— 单个任务名 (引用 ``tasks`` 中注册的任务),
|
||||
生成单任务图.
|
||||
* ``list[str]`` —— 任务名列表, 自动 :meth:`Graph.chain` 建立链式依赖,
|
||||
即后一个任务依赖前一个.
|
||||
* :class:`~pyflowx.graph.Graph` —— 直接使用该图 (用于复杂场景, 如
|
||||
自定义 ``conditions``、并行分支等).
|
||||
tasks : list[TaskSpec]
|
||||
扁平注册的任务列表. ``aliases`` 中的字符串引用这些任务名.
|
||||
未被任何 alias 引用的任务不会被执行.
|
||||
strategy : str | Strategy
|
||||
默认执行策略 (``Strategy.SEQUENTIAL`` / ``Strategy.THREAD`` /
|
||||
``Strategy.ASYNC`` 或对应字符串). 可被命令行 ``--strategy`` 覆盖.
|
||||
默认执行策略. 可被命令行 ``--strategy`` 覆盖.
|
||||
description : str
|
||||
CLI 帮助文本.
|
||||
verbose : bool
|
||||
是否显示详细执行过程. ``True`` 时打印任务生命周期和 subprocess 输出.
|
||||
默认 ``True``. 可被命令行 ``--quiet`` 关闭.
|
||||
**graphs : Graph
|
||||
命令名到图的映射. 每个 key 是一个命令名, value 是对应的
|
||||
:class:`~pyflowx.graph.Graph`.
|
||||
是否显示详细执行过程. 默认 ``True``, 可被命令行 ``--quiet`` 关闭.
|
||||
|
||||
Examples
|
||||
--------
|
||||
基本用法::
|
||||
简单场景 (tasks + aliases)::
|
||||
|
||||
runner = px.CliRunner(
|
||||
clean=px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("cargo_clean", cmd=["cargo", "clean"]),
|
||||
]
|
||||
),
|
||||
build=px.Graph.from_specs(
|
||||
[
|
||||
px.TaskSpec("uv_build", cmd=["uv", "build"]),
|
||||
]
|
||||
),
|
||||
tasks=[
|
||||
px.cmd(["uv", "build"]), # name="uv_build"
|
||||
px.cmd(["maturin", "build"], name="maturin_build"),
|
||||
px.cmd(["ruff", "check", "--fix"], name="lint"),
|
||||
],
|
||||
aliases={
|
||||
"b": "uv_build",
|
||||
"ba": ["uv_build", "maturin_build"], # chain: maturin 依赖 uv
|
||||
"lint": "lint",
|
||||
},
|
||||
)
|
||||
runner.run() # 解析 sys.argv
|
||||
runner.run()
|
||||
|
||||
指定策略与描述::
|
||||
复杂场景 (直接用 Graph)::
|
||||
|
||||
runner = px.CliRunner(
|
||||
strategy=px.Strategy.THREAD,
|
||||
aliases={
|
||||
"a": px.Graph.from_specs([
|
||||
px.TaskSpec("add", cmd=["git", "add", "."], conditions=(...)),
|
||||
px.TaskSpec("commit", cmd=["git", "commit"], depends_on=("add",)),
|
||||
]),
|
||||
},
|
||||
)
|
||||
runner.run(["test", "--strategy", "sequential"])
|
||||
"""
|
||||
|
||||
graphs: dict[str, Graph] = field(default_factory=dict)
|
||||
strategy: Strategy = field(default="sequential")
|
||||
aliases: dict[str, str | list[str | TaskSpec[Any]] | TaskSpec[Any] | Graph] = field(default_factory=dict)
|
||||
tasks: list[TaskSpec[Any]] = field(default_factory=list)
|
||||
strategy: Strategy = field(default="dependency")
|
||||
description: str = field(default_factory=str)
|
||||
verbose: bool = field(default_factory=lambda: True)
|
||||
# 解析后的命令→图映射,__post_init__ 填充
|
||||
graphs: dict[str, Graph] = field(default_factory=dict, init=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.graphs:
|
||||
raise ValueError("CliRunner 至少需要一个命令 (通过关键字参数提供)")
|
||||
if not self.aliases:
|
||||
raise ValueError("CliRunner 至少需要一个别名 (通过 aliases= 提供)")
|
||||
|
||||
# 解析并展开字符串引用,委托给 GraphComposer。
|
||||
# Graph 不再 frozen,可直接赋值,无需 object.__setattr__。
|
||||
self.graphs = GraphComposer(self.graphs).resolve_all()
|
||||
# 1. 把 tasks 注册为虚拟命令图(每个 task 一个图),加入 raw_graphs
|
||||
# 使 GraphComposer 能解析对它们的字符串引用
|
||||
raw_graphs: dict[str, Graph] = {}
|
||||
for spec in self.tasks:
|
||||
if spec.name in raw_graphs:
|
||||
raise ValueError(f"任务名重复: {spec.name!r}")
|
||||
raw_graphs[spec.name] = Graph.from_specs([spec])
|
||||
|
||||
# 2. 把每个 alias 转为 Graph(alias 名可与 task 名相同,覆盖 task 注册)
|
||||
for alias, value in self.aliases.items():
|
||||
raw_graphs[alias] = self._alias_to_graph(alias, value)
|
||||
|
||||
# 3. 解析图间字符串引用(str / list[str] 引用其他 alias 或任务)
|
||||
self.graphs = GraphComposer(raw_graphs).resolve_all()
|
||||
|
||||
@staticmethod
|
||||
def _alias_to_graph(
|
||||
alias: str,
|
||||
value: str | list[str | TaskSpec[Any]] | TaskSpec[Any] | Graph,
|
||||
) -> Graph:
|
||||
"""把 alias 的值转换为 Graph.
|
||||
|
||||
* ``str`` —— 对其他 alias 或已注册任务名的引用, 由 GraphComposer 展开.
|
||||
* ``TaskSpec`` —— 单个内联任务, 生成单任务图.
|
||||
* ``list[str | TaskSpec]`` —— 引用/任务混合列表, GraphComposer 展开时
|
||||
自动让后续引用依赖前面 (chain 语义). 元素为 alias 名、任务名或
|
||||
:class:`TaskSpec` 对象 (内联任务).
|
||||
* ``Graph`` —— 原样返回 (用于复杂场景: conditions、并行分支等).
|
||||
"""
|
||||
if isinstance(value, Graph):
|
||||
return value
|
||||
if isinstance(value, TaskSpec):
|
||||
return Graph.from_specs([value])
|
||||
if isinstance(value, str):
|
||||
# 字符串引用,用 _pending_refs 占位,GraphComposer 后续展开
|
||||
return Graph.from_specs([value]) # type: ignore[arg-type]
|
||||
if isinstance(value, list):
|
||||
if not value:
|
||||
raise ValueError(f"别名 {alias!r} 的任务列表为空")
|
||||
for item in value:
|
||||
if not isinstance(item, (str, TaskSpec)):
|
||||
raise TypeError(f"别名 {alias!r} 的列表元素类型无效: {type(item).__name__}, 预期 str 或 TaskSpec")
|
||||
# str/TaskSpec 混合列表,由 GraphComposer 展开(自动建立 chain 依赖)
|
||||
return Graph.from_specs(value)
|
||||
raise TypeError(
|
||||
f"别名 {alias!r} 的值类型无效: {type(value).__name__}, 预期 str/TaskSpec/list[str|TaskSpec]/Graph"
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 内省
|
||||
# ------------------------------------------------------------------ #
|
||||
@property
|
||||
def commands(self) -> list[str]:
|
||||
"""可用的命令列表 (按插入顺序)."""
|
||||
return list(self.graphs.keys())
|
||||
"""可用的命令列表 (按 aliases 定义顺序, 不含 tasks 中未引用的任务)."""
|
||||
return list(self.aliases.keys())
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 参数解析
|
||||
# ------------------------------------------------------------------ #
|
||||
def _prog_name(self) -> str:
|
||||
"""从 sys.argv[0] 推导程序名."""
|
||||
import os
|
||||
|
||||
return os.path.basename(sys.argv[0]) if sys.argv else "pyflowx"
|
||||
return Path(sys.argv[0]).name if sys.argv else "pyflowx"
|
||||
|
||||
def create_parser(self) -> argparse.ArgumentParser:
|
||||
"""创建参数解析器.
|
||||
@@ -225,9 +251,9 @@ class CliRunner:
|
||||
parser.print_help()
|
||||
return CliExitCode.FAILURE.value
|
||||
|
||||
# 验证命令
|
||||
if parsed.command not in self.graphs:
|
||||
available = ", ".join(self.graphs.keys())
|
||||
# 验证命令(必须是已注册的 alias,不接受裸任务名)
|
||||
if parsed.command not in self.aliases:
|
||||
available = ", ".join(self.commands)
|
||||
print(
|
||||
f"错误: 未知命令 {parsed.command!r} (可用命令: {available})",
|
||||
file=sys.stderr,
|
||||
@@ -237,12 +263,8 @@ class CliRunner:
|
||||
# 确定是否 verbose: --quiet 覆盖默认值
|
||||
verbose = self.verbose and not parsed.quiet
|
||||
|
||||
# 对图应用 verbose 设置 (重建带 verbose 标记的 spec)
|
||||
# 执行对应的图 (verbose 标记由 run() 统一应用到各 spec)
|
||||
graph = self.graphs[parsed.command]
|
||||
if verbose:
|
||||
graph = _apply_verbose_to_graph(graph, verbose=True)
|
||||
|
||||
# 执行对应的图
|
||||
try:
|
||||
report = run(
|
||||
graph,
|
||||
|
||||
+139
-44
@@ -17,13 +17,15 @@ import json
|
||||
import sys
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
from typing import Any, ContextManager, Mapping
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override
|
||||
else:
|
||||
from typing_extensions import override
|
||||
from typing_extensions import override # pragma: no cover
|
||||
|
||||
from .errors import StorageError
|
||||
|
||||
@@ -54,8 +56,91 @@ class StateBackend(ABC):
|
||||
def clear(self) -> None:
|
||||
"""清除所有存储状态。"""
|
||||
|
||||
def flush(self) -> None: # noqa: B027
|
||||
"""将内存中暂存的状态持久化到外部介质。
|
||||
|
||||
class MemoryBackend(StateBackend):
|
||||
默认无操作(如 :class:`MemoryBackend` 无需落盘)。
|
||||
:class:`JSONBackend` 在 :meth:`batch` 期间会延迟落盘,需在退出时调用。
|
||||
"""
|
||||
|
||||
def batch(self) -> ContextManager[None]:
|
||||
"""返回一个上下文管理器,期间 :meth:`save` 可延迟 :meth:`flush`。
|
||||
|
||||
默认实现为 no-op(如 :class:`MemoryBackend`)。:class:`JSONBackend`
|
||||
覆盖为:进入时标记延迟,退出时统一 flush 一次,将每任务一次落盘
|
||||
(N 次写入)降为整次运行一次(O(N) 而非 O(N²))。
|
||||
"""
|
||||
return nullcontext()
|
||||
|
||||
|
||||
class _TTLStateBackendMixin(StateBackend):
|
||||
"""TTL 状态后端共享逻辑。
|
||||
|
||||
将 ``has`` / ``get`` / ``load`` / ``save`` / ``clear`` 的统一实现
|
||||
委托给四个原始存取原语::meth:`_get_raw`、:meth:`_put_raw`、
|
||||
:meth:`_iter_raw`、:meth:`_clear_raw`,并基于 :meth:`_now` 与
|
||||
``self._ttl`` 提供统一的过期判断 :meth:`_is_expired`。
|
||||
|
||||
子类需设置 ``self._ttl`` 并实现上述四个原语;如需自定义时间源
|
||||
(如 ``time.monotonic``)可覆盖 :meth:`_now`。
|
||||
"""
|
||||
|
||||
_ttl: float | None
|
||||
|
||||
# ---- 原语:由子类实现 ---- #
|
||||
@abstractmethod
|
||||
def _get_raw(self, key: str) -> tuple[Any, float] | None:
|
||||
"""返回 ``(value, ts)``;键不存在时返回 ``None``。"""
|
||||
|
||||
@abstractmethod
|
||||
def _put_raw(self, key: str, value: Any, ts: float) -> None:
|
||||
"""写入一条记录。"""
|
||||
|
||||
@abstractmethod
|
||||
def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
|
||||
"""迭代所有记录(不做过期过滤),yield ``(key, value, ts)``。"""
|
||||
|
||||
@abstractmethod
|
||||
def _clear_raw(self) -> None:
|
||||
"""清空所有记录。"""
|
||||
|
||||
# ---- 共享实现 ---- #
|
||||
def _now(self) -> float:
|
||||
"""当前时间戳,默认为 wall-clock 秒。"""
|
||||
return time.time()
|
||||
|
||||
def _is_expired(self, ts: float) -> bool:
|
||||
"""时间戳 ``ts`` 是否已过期。"""
|
||||
if self._ttl is None:
|
||||
return False
|
||||
return (self._now() - ts) > self._ttl
|
||||
|
||||
@override
|
||||
def load(self) -> Mapping[str, Any]:
|
||||
return {k: v for k, v, ts in self._iter_raw() if not self._is_expired(ts)}
|
||||
|
||||
@override
|
||||
def save(self, key: str, value: Any) -> None:
|
||||
self._put_raw(key, value, self._now())
|
||||
|
||||
@override
|
||||
def has(self, key: str) -> bool:
|
||||
entry = self._get_raw(key)
|
||||
return entry is not None and not self._is_expired(entry[1])
|
||||
|
||||
@override
|
||||
def get(self, key: str) -> Any:
|
||||
entry = self._get_raw(key)
|
||||
if entry is None or self._is_expired(entry[1]):
|
||||
raise KeyError(key)
|
||||
return entry[0]
|
||||
|
||||
@override
|
||||
def clear(self) -> None:
|
||||
self._clear_raw()
|
||||
|
||||
|
||||
class MemoryBackend(_TTLStateBackendMixin):
|
||||
"""进程内 dict 后端。进程退出即丢失。
|
||||
|
||||
Parameters
|
||||
@@ -70,35 +155,28 @@ class MemoryBackend(StateBackend):
|
||||
self._ttl = ttl
|
||||
|
||||
@override
|
||||
def load(self) -> Mapping[str, Any]:
|
||||
return {k: v for k, (v, _ts) in self._store.items() if not self._expired(k)}
|
||||
def _now(self) -> float:
|
||||
return time.monotonic()
|
||||
|
||||
@override
|
||||
def save(self, key: str, value: Any) -> None:
|
||||
self._store[key] = (value, time.monotonic())
|
||||
def _get_raw(self, key: str) -> tuple[Any, float] | None:
|
||||
return self._store.get(key)
|
||||
|
||||
@override
|
||||
def has(self, key: str) -> bool:
|
||||
return key in self._store and not self._expired(key)
|
||||
def _put_raw(self, key: str, value: Any, ts: float) -> None:
|
||||
self._store[key] = (value, ts)
|
||||
|
||||
@override
|
||||
def get(self, key: str) -> Any:
|
||||
if key not in self._store or self._expired(key):
|
||||
raise KeyError(key)
|
||||
return self._store[key][0]
|
||||
def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
|
||||
for k, (v, ts) in self._store.items():
|
||||
yield k, v, ts
|
||||
|
||||
@override
|
||||
def clear(self) -> None:
|
||||
def _clear_raw(self) -> None:
|
||||
self._store.clear()
|
||||
|
||||
def _expired(self, key: str) -> bool:
|
||||
if self._ttl is None or key not in self._store:
|
||||
return False
|
||||
_value, ts = self._store[key]
|
||||
return (time.monotonic() - ts) > self._ttl
|
||||
|
||||
|
||||
class JSONBackend(StateBackend):
|
||||
class JSONBackend(_TTLStateBackendMixin):
|
||||
"""基于文件的 JSON 存储,用于跨进程续跑。
|
||||
|
||||
存储格式:``{key: {"value": v, "ts": epoch_seconds}}``。
|
||||
@@ -116,6 +194,7 @@ class JSONBackend(StateBackend):
|
||||
self._path: str = path
|
||||
self._ttl = ttl
|
||||
self._store: dict[str, dict[str, Any]] = {}
|
||||
self._defer_flush: bool = False
|
||||
self._load()
|
||||
|
||||
def _load(self) -> None:
|
||||
@@ -131,7 +210,6 @@ class JSONBackend(StateBackend):
|
||||
if isinstance(v, dict) and "value" in v and "ts" in v:
|
||||
self._store[k] = v
|
||||
else:
|
||||
# 旧格式:纯值
|
||||
self._store[k] = {"value": v, "ts": time.time()}
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise StorageError(f"cannot read state file {self._path!r}", exc) from exc
|
||||
@@ -145,17 +223,30 @@ class JSONBackend(StateBackend):
|
||||
except (OSError, TypeError) as exc:
|
||||
raise StorageError(f"cannot write state file {self._path!r}", exc) from exc
|
||||
|
||||
def _now(self) -> float:
|
||||
return time.time()
|
||||
|
||||
def _expired(self, entry: dict[str, Any]) -> bool:
|
||||
if self._ttl is None:
|
||||
return False
|
||||
return (self._now() - float(entry.get("ts", 0))) > self._ttl
|
||||
@override
|
||||
def _get_raw(self, key: str) -> tuple[Any, float] | None:
|
||||
entry = self._store.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
return entry["value"], float(entry.get("ts", 0))
|
||||
|
||||
@override
|
||||
def load(self) -> Mapping[str, Any]:
|
||||
return {k: v["value"] for k, v in self._store.items() if not self._expired(v)}
|
||||
def _put_raw(self, key: str, value: Any, ts: float) -> None:
|
||||
self._store[key] = {"value": value, "ts": ts}
|
||||
|
||||
@override
|
||||
def _iter_raw(self) -> Iterator[tuple[str, Any, float]]:
|
||||
for k, entry in self._store.items():
|
||||
yield k, entry["value"], float(entry.get("ts", 0))
|
||||
|
||||
@override
|
||||
def _clear_raw(self) -> None:
|
||||
self._store.clear()
|
||||
|
||||
@override
|
||||
def clear(self) -> None:
|
||||
super().clear()
|
||||
self._flush()
|
||||
|
||||
@override
|
||||
def save(self, key: str, value: Any) -> None:
|
||||
@@ -163,23 +254,27 @@ class JSONBackend(StateBackend):
|
||||
_ = json.dumps(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise StorageError(f"result of key {key!r} is not JSON-serialisable", exc) from exc
|
||||
self._store[key] = {"value": value, "ts": self._now()}
|
||||
super().save(key, value)
|
||||
if not self._defer_flush:
|
||||
self._flush()
|
||||
|
||||
@override
|
||||
def flush(self) -> None:
|
||||
self._flush()
|
||||
|
||||
@override
|
||||
def has(self, key: str) -> bool:
|
||||
return key in self._store and not self._expired(self._store[key])
|
||||
@contextmanager
|
||||
def batch(self) -> Iterator[None]:
|
||||
"""进入批量模式:``save`` 暂不落盘,退出时统一 flush 一次。
|
||||
|
||||
@override
|
||||
def get(self, key: str) -> Any:
|
||||
if key not in self._store or self._expired(self._store[key]):
|
||||
raise KeyError(key)
|
||||
return self._store[key]["value"]
|
||||
|
||||
@override
|
||||
def clear(self) -> None:
|
||||
self._store.clear()
|
||||
self._flush()
|
||||
将整次运行 N 个任务的 N 次全量落盘降为 1 次。
|
||||
"""
|
||||
self._defer_flush = True
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._defer_flush = False
|
||||
self._flush()
|
||||
|
||||
|
||||
def resolve_backend(backend: StateBackend | None) -> StateBackend:
|
||||
|
||||
+208
-108
@@ -17,22 +17,24 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
ContextManager,
|
||||
Coroutine,
|
||||
Generator,
|
||||
Generic,
|
||||
Iterator,
|
||||
List,
|
||||
Mapping,
|
||||
Union,
|
||||
@@ -42,7 +44,7 @@ from typing import (
|
||||
if sys.version_info >= (3, 13):
|
||||
from typing import TypeVar
|
||||
else:
|
||||
from typing_extensions import TypeVar
|
||||
from typing_extensions import TypeVar # pragma: no cover
|
||||
|
||||
T = TypeVar("T", default=Any)
|
||||
|
||||
@@ -67,6 +69,8 @@ TaskCmd = Union[
|
||||
Strategy = Union[str, "StrategyKind"]
|
||||
StrategyKind = Any # 占位,避免循环;executors 模块用 Literal 约束
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 条件判断函数类型:接收依赖上下文(可能为空映射),返回是否应执行。
|
||||
Condition = Callable[[Context], bool]
|
||||
|
||||
@@ -74,6 +78,13 @@ Condition = Callable[[Context], bool]
|
||||
CacheKeyFn = Callable[[Context], str]
|
||||
|
||||
|
||||
def _format_skip_reason(failed_conditions: list[str]) -> str:
|
||||
"""格式化跳过原因:≤2 个全展示,>2 个仅展示前 2 个并附总数。"""
|
||||
if len(failed_conditions) <= 2:
|
||||
return f"条件不满足: {', '.join(failed_conditions)}"
|
||||
return f"条件不满足: {', '.join(failed_conditions[:2])} 等{len(failed_conditions)}个条件"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 重试策略
|
||||
# ---------------------------------------------------------------------- #
|
||||
@@ -243,6 +254,10 @@ class TaskSpec(Generic[T]):
|
||||
存取状态后端,使不同输入产生独立缓存条目。``None`` 表示用任务名。
|
||||
hooks:
|
||||
:class:`TaskHooks` 生命周期钩子。
|
||||
executor:
|
||||
同步任务的执行器:``"thread"``(默认,线程池)/ ``"process"``
|
||||
(进程池,绕过 GIL,适合 CPU 密集型;``fn`` 须可 pickle)/
|
||||
``"inline"``(直接在事件循环线程调用,最快但会阻塞循环)。
|
||||
"""
|
||||
|
||||
name: str
|
||||
@@ -268,6 +283,7 @@ class TaskSpec(Generic[T]):
|
||||
continue_on_error: bool = False
|
||||
cache_key: CacheKeyFn | None = None
|
||||
hooks: TaskHooks = field(default_factory=TaskHooks)
|
||||
executor: str = "thread" # "thread" | "process" | "inline"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.name:
|
||||
@@ -284,13 +300,16 @@ class TaskSpec(Generic[T]):
|
||||
if self.fn is None and self.cmd is None:
|
||||
raise ValueError(f"TaskSpec '{self.name}': 必须提供 fn 或 cmd 参数。")
|
||||
|
||||
@property
|
||||
@cached_property
|
||||
def effective_fn(self) -> TaskFn[T]:
|
||||
"""获取有效的执行函数。
|
||||
|
||||
若提供 ``cmd``,返回包装后的命令执行函数;否则返回 ``fn``。
|
||||
包装函数在每次调用时从 ``self`` 读取 ``verbose``/``cwd``/``env``/
|
||||
``timeout``,避免闭包捕获运行期参数,使翻转字段无需重建 spec。
|
||||
|
||||
结果按实例缓存(:func:`functools.cached_property`):frozen dataclass
|
||||
字段不可变,``_wrap_cmd`` 生成的闭包稳定,无需每次访问重建。
|
||||
"""
|
||||
if self.cmd is not None:
|
||||
return self._wrap_cmd()
|
||||
@@ -299,11 +318,17 @@ class TaskSpec(Generic[T]):
|
||||
raise ValueError(f"TaskSpec '{self.name}': 没有可执行的函数或命令。") # pragma: no cover
|
||||
|
||||
def _wrap_cmd(self) -> TaskFn[Any]:
|
||||
"""将 cmd 包装为可执行函数。"""
|
||||
"""将 cmd 包装为可执行函数。
|
||||
|
||||
实际执行逻辑位于 :mod:`pyflowx.command`,避免 :class:`TaskSpec`
|
||||
作为纯数据结构混入命令执行逻辑。
|
||||
"""
|
||||
from .command import run_command
|
||||
|
||||
spec = self
|
||||
|
||||
def _run() -> T:
|
||||
return cast(T, _run_command(spec))
|
||||
return cast(T, run_command(spec))
|
||||
|
||||
_run.__name__ = spec.name
|
||||
return _run # type: ignore[return-value]
|
||||
@@ -315,6 +340,7 @@ class TaskSpec(Generic[T]):
|
||||
-------
|
||||
(should_run, skip_reason)
|
||||
``should_run`` 为 False 时 ``skip_reason`` 描述跳过原因。
|
||||
失败条件超过 2 个时仅展示前 2 个并附总数。
|
||||
"""
|
||||
# 逐个求值条件,记录失败项。
|
||||
failed_conditions: list[str] = []
|
||||
@@ -323,14 +349,19 @@ class TaskSpec(Generic[T]):
|
||||
ok = condition(context)
|
||||
except Exception:
|
||||
ok = False
|
||||
name = getattr(condition, "__name__", None) or "匿名条件(执行错误)"
|
||||
failed_conditions.append(name)
|
||||
failed_conditions.append("匿名条件(执行错误)")
|
||||
continue
|
||||
if not ok:
|
||||
failed_conditions.append(getattr(condition, "__name__", None) or "匿名条件")
|
||||
reason = getattr(condition, "_reason", None)
|
||||
if reason is not None:
|
||||
failed_conditions.append(
|
||||
", ".join(str(r) for r in reason) if isinstance(reason, list) else str(reason),
|
||||
)
|
||||
else:
|
||||
failed_conditions.append(getattr(condition, "__name__", None) or "匿名条件")
|
||||
|
||||
if failed_conditions:
|
||||
return False, f"条件不满足: {', '.join(failed_conditions)}"
|
||||
return False, _format_skip_reason(failed_conditions)
|
||||
|
||||
if self.skip_if_missing and not self._is_cmd_available():
|
||||
cmd_name = self.cmd[0] if isinstance(self.cmd, list) and self.cmd else "unknown"
|
||||
@@ -355,118 +386,187 @@ class TaskSpec(Generic[T]):
|
||||
|
||||
def storage_key(self, context: Context) -> str:
|
||||
"""计算状态后端存储键。"""
|
||||
if self.cache_key is not None:
|
||||
try:
|
||||
return f"{self.name}:{self.cache_key(context)}"
|
||||
except Exception:
|
||||
return self.name
|
||||
return self.name
|
||||
if self.cache_key is None:
|
||||
return self.name
|
||||
try:
|
||||
return f"{self.name}:{self.cache_key(context)}"
|
||||
except (TypeError, ValueError, KeyError, AttributeError) as exc:
|
||||
# cache_key 抛出预期内的数据/类型异常时回退到 name,但仍记录警告
|
||||
# 以便用户发现 cache_key 实现中的 bug。
|
||||
logger.warning(
|
||||
"task %r: cache_key 回退到 name(%s: %s)",
|
||||
self.name,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
return self.name
|
||||
|
||||
|
||||
# 全局锁:序列化对进程级状态(os.environ / os.chdir)的临时修改。
|
||||
# ``fn`` 任务在 thread/async 策略下并发执行时,若各自配置了不同的
|
||||
# ``cwd``/``env``,会相互覆盖(os.chdir 与 os.environ 均为进程全局)。
|
||||
# 该锁仅包裹"切换→执行→恢复"区间,保证正确性;不使用 cwd/env 的任务不受影响。
|
||||
_env_cwd_lock = threading.RLock()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _env_and_cwd(
|
||||
env: Mapping[str, str] | None,
|
||||
cwd: Path | None,
|
||||
) -> Iterator[None]:
|
||||
"""临时设置环境变量与工作目录。"""
|
||||
saved_env: dict[str, str] = {}
|
||||
saved_cwd: str | None = None
|
||||
if env:
|
||||
for k, v in env.items():
|
||||
if k in os.environ:
|
||||
saved_env[k] = os.environ[k]
|
||||
os.environ[k] = v
|
||||
if cwd is not None:
|
||||
saved_cwd = str(Path.cwd())
|
||||
os.chdir(cwd)
|
||||
try:
|
||||
) -> Generator[None, None, None]:
|
||||
"""临时设置环境变量与工作目录。
|
||||
|
||||
``os.environ`` 与 ``os.chdir`` 是进程级全局状态,在 thread/async 策略下
|
||||
并发执行多个带 ``env``/``cwd`` 的 ``fn`` 任务时会相互覆盖。本函数通过
|
||||
模块级 :data:`_env_cwd_lock` 串行化"切换→执行→恢复"区间,确保正确性。
|
||||
无 ``env`` 且无 ``cwd`` 时直接 yield,不获取锁。
|
||||
"""
|
||||
if not env and cwd is None:
|
||||
yield
|
||||
finally:
|
||||
if saved_cwd is not None:
|
||||
os.chdir(saved_cwd)
|
||||
# 恢复环境变量
|
||||
return
|
||||
with _env_cwd_lock:
|
||||
saved_env: dict[str, str] = {}
|
||||
saved_cwd: str | None = None
|
||||
if env:
|
||||
for k in env:
|
||||
if k in saved_env:
|
||||
os.environ[k] = saved_env[k]
|
||||
else:
|
||||
os.environ.pop(k, None)
|
||||
|
||||
|
||||
def _run_command(spec: TaskSpec[Any]) -> Any: # noqa: PLR0912
|
||||
"""执行 ``spec.cmd`` 指定的命令(list / shell 字符串 / 可调用对象)。"""
|
||||
cmd = spec.cmd
|
||||
verbose = spec.verbose
|
||||
cwd = spec.cwd
|
||||
timeout = spec.timeout
|
||||
env_override = spec.env
|
||||
|
||||
# 可调用对象:直接调用,返回其结果。
|
||||
if callable(cmd) and not isinstance(cmd, (list, str)):
|
||||
name = getattr(cmd, "__name__", "callable")
|
||||
if verbose:
|
||||
print(f"[verbose] 执行可调用命令: {name}", flush=True)
|
||||
if cwd is not None:
|
||||
print(f"[verbose] 工作目录: {cwd}", flush=True)
|
||||
try:
|
||||
return cmd()
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"可调用命令执行异常: {name}: {e}") from e
|
||||
|
||||
is_list = isinstance(cmd, list)
|
||||
if is_list:
|
||||
cmd_str = " ".join(arg for arg in cmd) # type: ignore[union-attr]
|
||||
verb = "执行命令"
|
||||
label = "命令"
|
||||
else:
|
||||
cmd_str = cast(str, cmd)
|
||||
verb = "执行 Shell"
|
||||
label = "Shell 命令"
|
||||
|
||||
if verbose:
|
||||
print(f"[verbose] {verb}: {cmd_str}", flush=True)
|
||||
for k, v in env.items():
|
||||
if k in os.environ:
|
||||
saved_env[k] = os.environ[k]
|
||||
os.environ[k] = v
|
||||
if cwd is not None:
|
||||
print(f"[verbose] 工作目录: {cwd}", flush=True)
|
||||
|
||||
# 合并环境变量
|
||||
run_env: dict[str, str] | None = None
|
||||
if env_override:
|
||||
run_env = dict(os.environ)
|
||||
run_env.update(env_override)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cast(Union[str, List[str]], cmd),
|
||||
shell=not is_list,
|
||||
cwd=cwd,
|
||||
env=run_env,
|
||||
timeout=timeout,
|
||||
capture_output=not verbose,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError(f"{label}未找到: {cmd_str}") from None
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError(f"{label}执行超时: {cmd_str} ({timeout}s)") from None
|
||||
except OSError as e:
|
||||
raise RuntimeError(f"{label}执行异常: {cmd_str}: {e}") from e
|
||||
|
||||
if verbose:
|
||||
print(f"[verbose] 返回码: {result.returncode}", flush=True)
|
||||
|
||||
if result.returncode == 0:
|
||||
return None
|
||||
|
||||
err_msg = f"{label}执行失败: `{cmd_str}`, 返回码: {result.returncode}"
|
||||
if not verbose and result.stderr.strip():
|
||||
err_msg += f"\n{result.stderr.strip()}"
|
||||
raise RuntimeError(err_msg)
|
||||
saved_cwd = str(Path.cwd())
|
||||
os.chdir(cwd)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if saved_cwd is not None:
|
||||
os.chdir(saved_cwd)
|
||||
# 恢复环境变量
|
||||
if env:
|
||||
for k in env:
|
||||
if k in saved_env:
|
||||
os.environ[k] = saved_env[k]
|
||||
else:
|
||||
os.environ.pop(k, None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 任务模板:批量生成相似 TaskSpec 的工厂
|
||||
# ---------------------------------------------------------------------- #
|
||||
def _task_noop() -> None:
|
||||
"""task(cmd=...) 形式下的占位 fn(cmd 任务执行期不调用 fn)。"""
|
||||
return None
|
||||
|
||||
|
||||
def task(
|
||||
fn: TaskFn[Any] | None = None,
|
||||
*,
|
||||
cmd: TaskCmd | None = None,
|
||||
depends_on: tuple[str, ...] = (),
|
||||
soft_depends_on: tuple[str, ...] = (),
|
||||
defaults: Mapping[str, Any] | None = None,
|
||||
args: tuple[Any, ...] = (),
|
||||
kwargs: Mapping[str, Any] | None = None,
|
||||
retry: RetryPolicy | None = None,
|
||||
timeout: float | None = None,
|
||||
tags: tuple[str, ...] = (),
|
||||
conditions: tuple[Condition, ...] = (),
|
||||
cwd: str | Path | None = None,
|
||||
env: Mapping[str, str] | None = None,
|
||||
verbose: bool = False,
|
||||
skip_if_missing: bool = False,
|
||||
allow_upstream_skip: bool = False,
|
||||
strategy: str | None = None,
|
||||
priority: int = 0,
|
||||
concurrency_key: str | None = None,
|
||||
continue_on_error: bool = False,
|
||||
cache_key: CacheKeyFn | None = None,
|
||||
hooks: TaskHooks | None = None,
|
||||
name: str | None = None,
|
||||
) -> Any:
|
||||
"""装饰器:将函数转为 :class:`TaskSpec`。
|
||||
|
||||
``name`` 默认取 ``fn.__name__``。可直接装饰函数,或带参数使用。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> @px.task
|
||||
... def extract(): return [1, 2, 3]
|
||||
>>> @px.task(depends_on=("extract",))
|
||||
... def double(extract): return [x * 2 for x in extract]
|
||||
>>> graph = px.Graph.from_specs([extract, double])
|
||||
"""
|
||||
|
||||
def _decorate(func: TaskFn[Any]) -> TaskSpec[Any]:
|
||||
spec_name = name or func.__name__
|
||||
return TaskSpec(
|
||||
name=spec_name,
|
||||
fn=func,
|
||||
cmd=cmd,
|
||||
depends_on=depends_on,
|
||||
soft_depends_on=soft_depends_on,
|
||||
defaults=dict(defaults) if defaults else {},
|
||||
args=args,
|
||||
kwargs=dict(kwargs) if kwargs else {},
|
||||
retry=retry if retry is not None else RetryPolicy(),
|
||||
timeout=timeout,
|
||||
tags=tags,
|
||||
conditions=conditions,
|
||||
cwd=Path(cwd) if isinstance(cwd, str) else cwd,
|
||||
env=dict(env) if env else None,
|
||||
verbose=verbose,
|
||||
skip_if_missing=skip_if_missing,
|
||||
allow_upstream_skip=allow_upstream_skip,
|
||||
strategy=strategy,
|
||||
priority=priority,
|
||||
concurrency_key=concurrency_key,
|
||||
continue_on_error=continue_on_error,
|
||||
cache_key=cache_key,
|
||||
hooks=hooks if hooks is not None else TaskHooks(),
|
||||
)
|
||||
|
||||
if fn is None and cmd is None:
|
||||
# 带参数调用:@task(depends_on=...),等待被装饰函数
|
||||
return _decorate
|
||||
if fn is None:
|
||||
# task(cmd=..., name=...) 直接构造,无被装饰函数
|
||||
if name is None:
|
||||
raise ValueError("task(cmd=...) 需要显式提供 name")
|
||||
return _decorate(_task_noop)
|
||||
return _decorate(fn)
|
||||
|
||||
|
||||
def cmd(
|
||||
command: list[str],
|
||||
*,
|
||||
name: str | None = None,
|
||||
depends_on: tuple[str, ...] = (),
|
||||
**kwargs: Any,
|
||||
) -> TaskSpec[Any]:
|
||||
"""从命令列表快速创建 :class:`TaskSpec`。
|
||||
|
||||
``name`` 默认为 ``"_".join(command[:2])``(如 ``["uv", "build"]`` → ``"uv_build"``)。
|
||||
若命令不足两个元素则用 ``"_".join(command)``。
|
||||
|
||||
其余关键字参数透传给 :class:`TaskSpec`(如 ``depends_on``、``tags`` 等)。
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> uv_build = px.cmd(["uv", "build"])
|
||||
>>> uv_build.name
|
||||
'uv_build'
|
||||
>>> lint = px.cmd(["ruff", "check", "--fix"], name="lint")
|
||||
>>> lint.name
|
||||
'lint'
|
||||
"""
|
||||
spec_name = name or "_".join(command[:2]) if len(command) >= 2 else "_".join(command)
|
||||
return TaskSpec(
|
||||
name=spec_name,
|
||||
cmd=command,
|
||||
depends_on=depends_on,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def task_template(
|
||||
fn: TaskFn[Any] | None = None,
|
||||
cmd: TaskCmd | None = None,
|
||||
|
||||
@@ -6,6 +6,15 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = [
|
||||
"clr",
|
||||
"reset_icon_cache",
|
||||
"setenv",
|
||||
"setenv_group",
|
||||
"which",
|
||||
"write_file",
|
||||
]
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
@@ -66,7 +75,7 @@ def reset_icon_cache() -> list[px.TaskSpec]:
|
||||
]
|
||||
|
||||
|
||||
def setenv(name: str, value: str, default: bool = False):
|
||||
def setenv(name: str, value: str, default: bool = False) -> px.TaskSpec:
|
||||
"""设置环境变量任务."""
|
||||
|
||||
def set_env():
|
||||
@@ -78,7 +87,12 @@ def setenv(name: str, value: str, default: bool = False):
|
||||
return px.TaskSpec(f"setenv_{name.lower()}", fn=set_env, verbose=True)
|
||||
|
||||
|
||||
def which(cmd: str):
|
||||
def setenv_group(envs: dict[str, str], default: bool = False) -> list[px.TaskSpec]:
|
||||
"""设置环境变量组任务."""
|
||||
return [setenv(name, value, default) for name, value in envs.items()]
|
||||
|
||||
|
||||
def which(cmd: str) -> px.TaskSpec:
|
||||
"""查找命令路径任务."""
|
||||
which_cmd = "where" if Constants.IS_WINDOWS else "which"
|
||||
|
||||
@@ -95,4 +109,11 @@ def which(cmd: str):
|
||||
return px.TaskSpec(f"which_{cmd}", fn=find_command)
|
||||
|
||||
|
||||
__all__ = ["clr", "setenv", "which"]
|
||||
def write_file(path: str, content: str, encoding: str = "utf-8") -> px.TaskSpec:
|
||||
"""写入文件任务."""
|
||||
|
||||
def write():
|
||||
p = Path(path)
|
||||
p.write_text(content, encoding=encoding)
|
||||
|
||||
return px.TaskSpec(f"write_file_{path}", fn=write, verbose=True)
|
||||
|
||||
@@ -0,0 +1,599 @@
|
||||
"""@px.tool 装饰器: 函数签名驱动 CLI 生成 + DAG 编排.
|
||||
|
||||
替代 YAML 配置模式, 用 .py 装饰器统一描述工具.
|
||||
函数签名 → argparse 自动生成, 函数体即任务逻辑, needs/strategy 表达 DAG.
|
||||
|
||||
示例
|
||||
----
|
||||
::
|
||||
|
||||
@px.tool("pdftool", subcommand="m", help="合并 PDF")
|
||||
def pdf_merge(
|
||||
inputs: list[Path], # positional, nargs="+"
|
||||
output: Path = Path("merged.pdf"), # --output 选项
|
||||
) -> None:
|
||||
... # 函数体即逻辑
|
||||
|
||||
# CLI: pf pdftool m a.pdf b.pdf --output merged.pdf
|
||||
|
||||
cmd 任务 (执行外部命令, 函数体不执行)::
|
||||
|
||||
@px.tool("pymake", subcommand="b", help="构建", cmd=["uv", "build"])
|
||||
def b(cwd: Path = Path(".")) -> None:
|
||||
pass # 签名仅驱动 CLI, cwd 注入到 cmd 工作目录
|
||||
|
||||
聚合任务 (有 needs 无 cmd 无函数逻辑)::
|
||||
|
||||
@px.tool("pymake", subcommand="tc", help="类型检查",
|
||||
needs=["c", "pyrefly_check", "lint"], strategy="thread")
|
||||
def tc(cwd: Path = Path(".")) -> None:
|
||||
pass # 仅作依赖聚合点
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = [
|
||||
"ToolExitCode",
|
||||
"ToolSpec",
|
||||
"clear_tool_registry",
|
||||
"get_tool",
|
||||
"list_subcommands",
|
||||
"list_tools",
|
||||
"run_tool",
|
||||
"tool",
|
||||
]
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import enum
|
||||
import inspect
|
||||
import sys
|
||||
import textwrap
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Literal, Mapping, Sequence, Union, cast, get_args, get_origin, get_type_hints
|
||||
|
||||
from pyflowx.errors import PyFlowXError
|
||||
from pyflowx.executors import run
|
||||
from pyflowx.graph import Graph
|
||||
from pyflowx.task import RetryPolicy, TaskSpec
|
||||
|
||||
|
||||
class ToolExitCode(enum.IntEnum):
|
||||
"""工具执行退出码."""
|
||||
|
||||
SUCCESS = 0
|
||||
FAILURE = 1
|
||||
INTERRUPTED = 130 # 与 POSIX 信号中断一致
|
||||
|
||||
|
||||
def _noop() -> None:
|
||||
"""聚合任务的占位函数."""
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# ToolSpec: 工具描述符
|
||||
# ---------------------------------------------------------------------- #
|
||||
@dataclass(frozen=True)
|
||||
class ToolSpec:
|
||||
"""工具描述符: 由 ``@px.tool`` 装饰器注册.
|
||||
|
||||
封装函数 + CLI 元数据 + DAG 编排参数, 运行时映射为 :class:`TaskSpec`.
|
||||
"""
|
||||
|
||||
name: str # 工具名 (如 "pdftool")
|
||||
subcommand: str | None # 子命令名 (None=单命令工具)
|
||||
func: Callable[..., Any] # 函数 (签名驱动 CLI, 体即逻辑)
|
||||
help: str = "" # 帮助文本
|
||||
description: str = "" # 工具描述 (pf 列表用)
|
||||
cmd: tuple[str, ...] | str | None = None # 命令 (有则执行 cmd, 无则执行函数体)
|
||||
needs: tuple[str, ...] = () # 依赖任务 (引用同 tool 的 subcommand)
|
||||
strategy: Literal["sequential", "thread", "async", "dependency"] | None = None # 执行策略
|
||||
cwd: str | Path | None = None # 工作目录 (cmd 任务, 装饰器级默认)
|
||||
allow_upstream_skip: bool = False
|
||||
hidden: bool = False # 不暴露为 subcommand (内部 job)
|
||||
env: Mapping[str, str] | None = None
|
||||
retry: RetryPolicy | None = None
|
||||
timeout: float | None = None
|
||||
|
||||
|
||||
# 全局工具注册表: {tool_name: {subcommand: ToolSpec}}
|
||||
_TOOL_REGISTRY: dict[str, dict[str | None, ToolSpec]] = {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# @tool 装饰器 + 注册表
|
||||
# ---------------------------------------------------------------------- #
|
||||
def tool(
|
||||
name: str,
|
||||
*,
|
||||
subcommand: str | None = None,
|
||||
help: str = "",
|
||||
description: str = "",
|
||||
cmd: Sequence[str] | str | None = None,
|
||||
needs: Sequence[str] | None = None,
|
||||
strategy: Literal["sequential", "thread", "async", "dependency"] | None = None,
|
||||
cwd: str | Path | None = None,
|
||||
allow_upstream_skip: bool = False,
|
||||
hidden: bool = False,
|
||||
env: Mapping[str, str] | None = None,
|
||||
retry: RetryPolicy | None = None,
|
||||
timeout: float | None = None,
|
||||
) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
|
||||
"""装饰器: 将函数注册为 ``@px.tool`` 工具.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
工具名 (如 "pdftool"); 多个 ``@px.tool`` 共用同名即多 subcommand 工具
|
||||
subcommand : str | None
|
||||
子命令名; ``None`` 表示单命令工具 (整个工具仅一个函数)
|
||||
help : str
|
||||
子命令帮助文本; 默认用函数 docstring
|
||||
description : str
|
||||
工具描述, 用于 pf 工具列表
|
||||
cmd : Sequence[str] | str | None
|
||||
命令列表或 shell 字符串; 有 ``cmd`` 执行命令, 函数体不执行 (签名仅驱动 CLI)
|
||||
needs : Sequence[str] | None
|
||||
依赖任务名 (引用同 tool 的其他 subcommand)
|
||||
strategy : str | None
|
||||
执行策略: ``sequential`` / ``thread`` / ``async`` / ``dependency``
|
||||
cwd : str | Path | None
|
||||
工作目录 (cmd 任务装饰器级默认); 若函数签名有 ``cwd`` 参数则被 CLI 值覆盖
|
||||
allow_upstream_skip : bool
|
||||
上游 SKIPPED 时本任务仍执行
|
||||
hidden : bool
|
||||
不暴露为 subcommand (内部 job, 仅被 needs 引用)
|
||||
env / retry / timeout
|
||||
透传 :class:`TaskSpec` 对应字段
|
||||
"""
|
||||
|
||||
def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
|
||||
spec = ToolSpec(
|
||||
name=name,
|
||||
subcommand=subcommand,
|
||||
func=func,
|
||||
help=help or inspect.getdoc(func) or "",
|
||||
description=description,
|
||||
cmd=cast("tuple[str, ...] | str | None", tuple(cmd) if isinstance(cmd, (list, tuple)) else cmd),
|
||||
needs=tuple(needs) if needs else (),
|
||||
strategy=strategy,
|
||||
cwd=cwd,
|
||||
allow_upstream_skip=allow_upstream_skip,
|
||||
hidden=hidden,
|
||||
env=dict(env) if env else None,
|
||||
retry=retry,
|
||||
timeout=timeout,
|
||||
)
|
||||
_register_tool(spec)
|
||||
return func
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def _register_tool(spec: ToolSpec) -> None:
|
||||
"""注册工具到全局注册表, 校验重复."""
|
||||
if spec.name not in _TOOL_REGISTRY:
|
||||
_TOOL_REGISTRY[spec.name] = {}
|
||||
if spec.subcommand in _TOOL_REGISTRY[spec.name]:
|
||||
raise ValueError(f"工具 {spec.name!r} 的子命令 {spec.subcommand!r} 已注册")
|
||||
_TOOL_REGISTRY[spec.name][spec.subcommand] = spec
|
||||
|
||||
|
||||
def get_tool(name: str, subcommand: str | None = None) -> ToolSpec:
|
||||
"""获取已注册工具.
|
||||
|
||||
Raises
|
||||
------
|
||||
KeyError
|
||||
工具或子命令未注册
|
||||
"""
|
||||
if name not in _TOOL_REGISTRY:
|
||||
raise KeyError(f"工具 {name!r} 未注册")
|
||||
subs = _TOOL_REGISTRY[name]
|
||||
if subcommand not in subs:
|
||||
raise KeyError(f"工具 {name!r} 没有子命令 {subcommand!r}")
|
||||
return subs[subcommand]
|
||||
|
||||
|
||||
def list_tools() -> list[str]:
|
||||
"""列出所有已注册工具名."""
|
||||
return sorted(_TOOL_REGISTRY.keys())
|
||||
|
||||
|
||||
def list_subcommands(name: str, include_hidden: bool = False) -> list[str]:
|
||||
"""列出工具的子命令 (hidden 默认排除).
|
||||
|
||||
单命令工具 (subcommand=None) 返回空列表.
|
||||
"""
|
||||
if name not in _TOOL_REGISTRY:
|
||||
return []
|
||||
return sorted(
|
||||
sc for sc, spec in _TOOL_REGISTRY[name].items() if sc is not None and (include_hidden or not spec.hidden)
|
||||
)
|
||||
|
||||
|
||||
def clear_tool_registry() -> None:
|
||||
"""清空注册表 (测试用)."""
|
||||
_TOOL_REGISTRY.clear()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# CLI 生成 (函数签名 → argparse)
|
||||
# ---------------------------------------------------------------------- #
|
||||
def _snake_to_kebab(name: str) -> str:
|
||||
"""snake_case → kebab-case."""
|
||||
return name.replace("_", "-")
|
||||
|
||||
|
||||
def _unwrap_optional(annotation: Any) -> tuple[Any, bool]:
|
||||
"""返回 (实际类型, 是否 Optional).
|
||||
|
||||
``Optional[X]`` / ``X | None`` → ``(X, True)``; 其他 → ``(annotation, False)``.
|
||||
"""
|
||||
origin = get_origin(annotation)
|
||||
is_union = origin is Union
|
||||
if not is_union and sys.version_info >= (3, 10):
|
||||
import types as _types
|
||||
|
||||
is_union = origin is _types.UnionType
|
||||
if is_union:
|
||||
args = [a for a in get_args(annotation) if a is not type(None)]
|
||||
if len(args) == 1:
|
||||
return args[0], True
|
||||
return annotation, False
|
||||
|
||||
|
||||
def _map_param_type(annotation: Any) -> dict[str, Any]:
|
||||
"""将类型注解转为 argparse add_argument 的关键字参数.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict
|
||||
含 ``type`` / ``nargs`` / ``choices`` 等; bool 不在此处理 (由调用方特殊处理)
|
||||
"""
|
||||
kwargs: dict[str, Any] = {}
|
||||
inner, _is_opt = _unwrap_optional(annotation)
|
||||
origin = get_origin(inner)
|
||||
|
||||
# list[X] / List[X] → nargs="+"
|
||||
if origin is list:
|
||||
args = get_args(inner)
|
||||
if args:
|
||||
kwargs["type"] = args[0]
|
||||
kwargs["nargs"] = "+"
|
||||
return kwargs
|
||||
|
||||
# Literal["a", "b"] → choices
|
||||
if origin is Literal:
|
||||
kwargs["choices"] = list(get_args(inner))
|
||||
return kwargs
|
||||
|
||||
# 基本类型
|
||||
if inner is int:
|
||||
kwargs["type"] = int
|
||||
elif inner is float:
|
||||
kwargs["type"] = float
|
||||
elif inner is Path:
|
||||
kwargs["type"] = Path
|
||||
elif inner is str:
|
||||
kwargs["type"] = str
|
||||
|
||||
return kwargs
|
||||
|
||||
|
||||
def _add_func_args_to_parser(parser: argparse.ArgumentParser, func: Callable[..., Any]) -> None:
|
||||
"""从函数签名向 parser 添加位置参数和选项参数.
|
||||
|
||||
- 无默认值 → positional
|
||||
- 有默认值 → ``--kebab-case`` 选项
|
||||
- ``bool`` 默认 False → ``--flag`` store_true; 默认 True → ``--no-flag`` store_false
|
||||
- ``list[X]`` → ``nargs="+"``
|
||||
- ``Literal[...]`` → ``choices``
|
||||
"""
|
||||
sig = inspect.signature(func)
|
||||
try:
|
||||
hints: dict[str, Any] = get_type_hints(func)
|
||||
except Exception:
|
||||
hints = {}
|
||||
|
||||
for pname, param in sig.parameters.items():
|
||||
if pname in ("self",):
|
||||
continue
|
||||
if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD):
|
||||
continue
|
||||
annotation = hints.get(pname, param.annotation if param.annotation is not inspect.Parameter.empty else str)
|
||||
has_default = param.default is not inspect.Parameter.empty
|
||||
default = param.default if has_default else inspect.Parameter.empty
|
||||
|
||||
# bool 特殊处理
|
||||
if annotation is bool:
|
||||
if not has_default or default is False:
|
||||
parser.add_argument(
|
||||
f"--{_snake_to_kebab(pname)}",
|
||||
dest=pname,
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=pname,
|
||||
)
|
||||
else: # 默认 True → --no-flag store_false
|
||||
parser.add_argument(
|
||||
f"--no-{_snake_to_kebab(pname)}",
|
||||
dest=pname,
|
||||
action="store_false",
|
||||
default=True,
|
||||
help=f"禁用 {pname}",
|
||||
)
|
||||
continue
|
||||
|
||||
type_kwargs = _map_param_type(annotation)
|
||||
|
||||
if has_default:
|
||||
flag = f"--{_snake_to_kebab(pname)}"
|
||||
kwargs: dict[str, Any] = {"dest": pname, "default": default, "help": pname}
|
||||
kwargs.update(type_kwargs)
|
||||
parser.add_argument(flag, **kwargs)
|
||||
else:
|
||||
# 位置参数; 若类型是 list[X] 则 nargs="+" 已在 type_kwargs
|
||||
kwargs = {"help": pname}
|
||||
kwargs.update(type_kwargs)
|
||||
parser.add_argument(pname, **kwargs)
|
||||
|
||||
|
||||
def _add_global_options(parser: argparse.ArgumentParser) -> None:
|
||||
"""向 parser (及所有 subparser) 添加全局选项."""
|
||||
parser.add_argument("--dry-run", action="store_true", help="仅打印执行计划, 不执行")
|
||||
parser.add_argument("--quiet", "-q", action="store_false", dest="verbose", help="减少输出")
|
||||
parser.add_argument("--strategy", type=str, default=None, help="执行策略 (sequential/thread/dependency)")
|
||||
parser.add_argument("--list", action="store_true", dest="list_jobs", help="列出所有任务后退出")
|
||||
# 子命令也加全局选项
|
||||
for action in parser._actions:
|
||||
if action.dest == "command" and isinstance(action.choices, dict):
|
||||
for sub in action.choices.values():
|
||||
sub.add_argument("--dry-run", action="store_true")
|
||||
sub.add_argument("--quiet", "-q", action="store_false", dest="verbose")
|
||||
sub.add_argument("--strategy", type=str, default=None)
|
||||
sub.add_argument("--list", action="store_true", dest="list_jobs")
|
||||
|
||||
|
||||
def _build_parser(name: str, all_subs: dict[str | None, ToolSpec]) -> argparse.ArgumentParser:
|
||||
"""构建 argparse 解析器.
|
||||
|
||||
单命令工具 (仅 subcommand=None) → 简单 parser;
|
||||
多 subcommand 工具 → 带 subparsers.
|
||||
"""
|
||||
first_spec = next(iter(all_subs.values()))
|
||||
description = first_spec.description or first_spec.help or name
|
||||
parser = argparse.ArgumentParser(prog=f"pf {name}", description=description)
|
||||
|
||||
if None in all_subs and len(all_subs) == 1:
|
||||
# 单命令工具
|
||||
_add_func_args_to_parser(parser, all_subs[None].func)
|
||||
else:
|
||||
# 多 subcommand 工具
|
||||
visible = [(sc, sp) for sc, sp in all_subs.items() if sc is not None and not sp.hidden]
|
||||
if visible:
|
||||
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
||||
for sc, sp in sorted(visible, key=lambda x: x[0] or ""):
|
||||
sub = subparsers.add_parser(sc, help=sp.help, description=sp.help)
|
||||
_add_func_args_to_parser(sub, sp.func)
|
||||
|
||||
_add_global_options(parser)
|
||||
return parser
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 依赖收集 + TaskSpec 构建
|
||||
# ---------------------------------------------------------------------- #
|
||||
def _collect_with_deps(name: str, target: str | None) -> list[str | None]:
|
||||
"""BFS 收集 target 及其传递依赖 (subcommand 名).
|
||||
|
||||
返回顺序: 依赖在前, target 在后 (符合 DAG 拓扑).
|
||||
"""
|
||||
if name not in _TOOL_REGISTRY:
|
||||
return [target]
|
||||
subs = _TOOL_REGISTRY[name]
|
||||
result: list[str | None] = []
|
||||
seen: set[str | None] = set()
|
||||
queue: list[str | None] = [target]
|
||||
while queue:
|
||||
sc = queue.pop(0)
|
||||
if sc in seen:
|
||||
continue
|
||||
seen.add(sc)
|
||||
result.append(sc)
|
||||
if sc in subs:
|
||||
queue.extend(subs[sc].needs)
|
||||
# 反转: 依赖在前, target 在后
|
||||
result.reverse()
|
||||
return result
|
||||
|
||||
|
||||
def _has_function_logic(func: Callable[..., Any]) -> bool:
|
||||
"""判断函数体是否有实际逻辑 (非 pass/.../docstring).
|
||||
|
||||
用 ast 分析, 避免 exec 函数.
|
||||
"""
|
||||
try:
|
||||
src = inspect.getsource(func)
|
||||
src = textwrap.dedent(src)
|
||||
tree = ast.parse(src)
|
||||
except (OSError, TypeError, SyntaxError): # pragma: no cover
|
||||
return True
|
||||
func_def = next((n for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))), None)
|
||||
if func_def is None: # pragma: no cover
|
||||
return True
|
||||
for stmt in func_def.body:
|
||||
if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Constant):
|
||||
continue # docstring
|
||||
if isinstance(stmt, ast.Pass):
|
||||
continue
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_aggregate(spec: ToolSpec) -> bool:
|
||||
"""判断是否为聚合任务 (有 needs 无 cmd 无函数逻辑)."""
|
||||
if spec.cmd is not None or not spec.needs:
|
||||
return False
|
||||
return not _has_function_logic(spec.func)
|
||||
|
||||
|
||||
def _build_task_spec(spec: ToolSpec, variables: Mapping[str, Any]) -> TaskSpec:
|
||||
"""将 ToolSpec + 解析后的变量转为 TaskSpec.
|
||||
|
||||
- cmd 任务: 执行命令, cwd 从 variables["cwd"] 或装饰器 cwd 取
|
||||
- 聚合任务 (有 needs 无 cmd 无函数逻辑): fn=noop
|
||||
- fn 任务: 执行函数, kwargs 按签名从 variables 取
|
||||
"""
|
||||
task_name = spec.subcommand if spec.subcommand is not None else spec.name
|
||||
|
||||
# cmd 任务
|
||||
if spec.cmd is not None:
|
||||
cwd_value = variables.get("cwd", spec.cwd)
|
||||
cwd = Path(cwd_value) if cwd_value is not None else None
|
||||
cmd_value: Any = list(spec.cmd) if isinstance(spec.cmd, tuple) else spec.cmd
|
||||
return TaskSpec(
|
||||
name=task_name,
|
||||
cmd=cmd_value,
|
||||
depends_on=spec.needs,
|
||||
cwd=cwd,
|
||||
env=spec.env,
|
||||
retry=spec.retry or RetryPolicy(),
|
||||
timeout=spec.timeout,
|
||||
allow_upstream_skip=spec.allow_upstream_skip,
|
||||
strategy=spec.strategy,
|
||||
)
|
||||
|
||||
# 聚合任务
|
||||
if _is_aggregate(spec):
|
||||
return TaskSpec(
|
||||
name=task_name,
|
||||
fn=_noop,
|
||||
depends_on=spec.needs,
|
||||
allow_upstream_skip=spec.allow_upstream_skip,
|
||||
strategy=spec.strategy,
|
||||
)
|
||||
|
||||
# fn 任务
|
||||
sig = inspect.signature(spec.func)
|
||||
kwargs: dict[str, Any] = {}
|
||||
for pname in sig.parameters:
|
||||
if pname in variables:
|
||||
kwargs[pname] = variables[pname]
|
||||
cwd_value = variables.get("cwd")
|
||||
cwd = Path(cwd_value) if cwd_value is not None else None
|
||||
return TaskSpec(
|
||||
name=task_name,
|
||||
fn=spec.func,
|
||||
kwargs=kwargs,
|
||||
depends_on=spec.needs,
|
||||
cwd=cwd,
|
||||
env=spec.env,
|
||||
retry=spec.retry or RetryPolicy(),
|
||||
timeout=spec.timeout,
|
||||
allow_upstream_skip=spec.allow_upstream_skip,
|
||||
strategy=spec.strategy,
|
||||
)
|
||||
|
||||
|
||||
def _extract_variables(args: argparse.Namespace, func: Callable[..., Any]) -> dict[str, Any]:
|
||||
"""从 argparse namespace 提取函数签名对应的参数."""
|
||||
sig = inspect.signature(func)
|
||||
variables: dict[str, Any] = {}
|
||||
for pname in sig.parameters:
|
||||
if hasattr(args, pname):
|
||||
variables[pname] = getattr(args, pname)
|
||||
return variables
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# 执行入口
|
||||
# ---------------------------------------------------------------------- #
|
||||
def _handle_list(name: str, all_subs: dict[str | None, ToolSpec]) -> int:
|
||||
"""处理 --list, 打印任务列表."""
|
||||
print(f"工具 {name} 的任务列表:")
|
||||
for sc, spec in sorted(all_subs.items(), key=lambda x: (x[0] is not None, x[0] or "")):
|
||||
display = sc if sc is not None else name
|
||||
deps = ", ".join(spec.needs) if spec.needs else "(无依赖)"
|
||||
hidden = " [隐藏]" if spec.hidden else ""
|
||||
cmd_tag = " [cmd]" if spec.cmd is not None else ""
|
||||
print(f" - {display} (依赖: {deps}){cmd_tag}{hidden}")
|
||||
return 0
|
||||
|
||||
|
||||
def run_tool(name: str, argv: Sequence[str] | None = None) -> int:
|
||||
"""执行工具, 返回退出码 (0 成功 / 1 失败 / 130 中断).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
工具名 (已通过 ``@px.tool`` 注册)
|
||||
argv : Sequence[str] | None
|
||||
命令行参数 (不含工具名); None 用 ``sys.argv[1:]``
|
||||
|
||||
Returns
|
||||
-------
|
||||
int
|
||||
退出码
|
||||
"""
|
||||
if name not in _TOOL_REGISTRY:
|
||||
print(f"错误: 未注册工具 {name!r}", file=sys.stderr)
|
||||
return ToolExitCode.FAILURE
|
||||
|
||||
all_subs = _TOOL_REGISTRY[name]
|
||||
argv = list(argv) if argv is not None else sys.argv[1:]
|
||||
|
||||
parser = _build_parser(name, all_subs)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
# --list 优先
|
||||
if getattr(args, "list_jobs", False):
|
||||
return _handle_list(name, all_subs)
|
||||
|
||||
# 确定要执行的 subcommand
|
||||
visible_subs = [sc for sc, sp in all_subs.items() if sc is not None and not sp.hidden]
|
||||
is_multi = len(visible_subs) > 1
|
||||
|
||||
if is_multi:
|
||||
subcommand = getattr(args, "command", None)
|
||||
if subcommand is None:
|
||||
parser.print_help()
|
||||
return ToolExitCode.SUCCESS
|
||||
else:
|
||||
# 单命令工具: subcommand 是 None 或唯一可见的具名 subcommand
|
||||
subcommand = visible_subs[0] if visible_subs else None
|
||||
|
||||
if subcommand not in all_subs:
|
||||
print(f"错误: 未知子命令 {subcommand!r}", file=sys.stderr)
|
||||
return ToolExitCode.FAILURE
|
||||
|
||||
target_spec = all_subs[subcommand]
|
||||
variables = _extract_variables(args, target_spec.func)
|
||||
|
||||
# 收集依赖 + 构建子图
|
||||
all_names = _collect_with_deps(name, subcommand)
|
||||
specs: list[TaskSpec] = []
|
||||
for sc in all_names:
|
||||
dep_spec = all_subs[sc]
|
||||
specs.append(_build_task_spec(dep_spec, variables))
|
||||
|
||||
graph = Graph.from_specs(specs)
|
||||
|
||||
print(f"[{name}] 执行: {subcommand or '全部任务'}", flush=True)
|
||||
|
||||
try:
|
||||
run(
|
||||
graph,
|
||||
strategy=target_spec.strategy or "dependency",
|
||||
dry_run=getattr(args, "dry_run", False),
|
||||
verbose=getattr(args, "verbose", True),
|
||||
)
|
||||
return ToolExitCode.SUCCESS
|
||||
except PyFlowXError as e:
|
||||
print(f"错误: {e}", file=sys.stderr)
|
||||
return ToolExitCode.FAILURE
|
||||
except KeyboardInterrupt:
|
||||
print("\n已中断", file=sys.stderr)
|
||||
return ToolExitCode.INTERRUPTED
|
||||
@@ -0,0 +1,26 @@
|
||||
"""进程池测试辅助:模块级函数(须可 pickle)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
|
||||
def cpu_heavy(n: int) -> int:
|
||||
"""CPU 密集型计算(求平方和)。"""
|
||||
return sum(i * i for i in range(n))
|
||||
|
||||
|
||||
def add(a: int, b: int) -> int:
|
||||
"""简单加法。"""
|
||||
return a + b
|
||||
|
||||
|
||||
def sub(a: int, b: int) -> int:
|
||||
"""简单减法。"""
|
||||
return a - b
|
||||
|
||||
|
||||
def slow_sleep(seconds: float) -> int:
|
||||
"""睡眠指定秒数,用于测试超时。"""
|
||||
time.sleep(seconds)
|
||||
return int(seconds)
|
||||
+20
-103
@@ -5,8 +5,7 @@ from __future__ import annotations
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.cli import autofmt
|
||||
from pyflowx.ops import autofmt as dev
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
@@ -19,14 +18,14 @@ class TestFormatWithRuff:
|
||||
"""Should format with ruff."""
|
||||
with patch("subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
autofmt.format_with_ruff(tmp_path, fix=True)
|
||||
dev.format_with_ruff(tmp_path, fix=True)
|
||||
assert mock_run.called
|
||||
|
||||
def test_format_with_ruff_no_fix(self, tmp_path: Path) -> None:
|
||||
"""Should format with ruff without fix."""
|
||||
with patch("subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
autofmt.format_with_ruff(tmp_path, fix=False)
|
||||
dev.format_with_ruff(tmp_path, fix=False)
|
||||
# Should not include --fix flag
|
||||
call_args = mock_run.call_args[0][0]
|
||||
assert "--fix" not in call_args
|
||||
@@ -42,14 +41,14 @@ class TestLintWithRuff:
|
||||
"""Should lint with ruff."""
|
||||
with patch("subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
autofmt.lint_with_ruff(tmp_path, fix=True)
|
||||
dev.lint_with_ruff(tmp_path, fix=True)
|
||||
assert mock_run.called
|
||||
|
||||
def test_lint_with_ruff_no_fix(self, tmp_path: Path) -> None:
|
||||
"""Should lint with ruff without fix."""
|
||||
with patch("subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
autofmt.lint_with_ruff(tmp_path, fix=False)
|
||||
dev.lint_with_ruff(tmp_path, fix=False)
|
||||
# Should not include --fix flag
|
||||
call_args = mock_run.call_args[0][0]
|
||||
assert "--fix" not in call_args
|
||||
@@ -66,7 +65,7 @@ class TestAddDocstring:
|
||||
py_file = tmp_path / "test.py"
|
||||
py_file.write_text("def test():\n pass\n")
|
||||
|
||||
result = autofmt.add_docstring(py_file, '"""Test module."""')
|
||||
result = dev.add_docstring(py_file, '"""Test module."""')
|
||||
assert result is True
|
||||
|
||||
def test_add_docstring_skips_files_with_docstring(self, tmp_path: Path) -> None:
|
||||
@@ -74,7 +73,7 @@ class TestAddDocstring:
|
||||
py_file = tmp_path / "test.py"
|
||||
py_file.write_text('"""Existing docstring."""\ndef test():\n pass\n')
|
||||
|
||||
result = autofmt.add_docstring(py_file, '"""New docstring."""')
|
||||
result = dev.add_docstring(py_file, '"""New docstring."""')
|
||||
assert result is False
|
||||
|
||||
def test_add_docstring_empty_file(self, tmp_path: Path) -> None:
|
||||
@@ -82,7 +81,7 @@ class TestAddDocstring:
|
||||
py_file = tmp_path / "test.py"
|
||||
py_file.write_text("")
|
||||
|
||||
result = autofmt.add_docstring(py_file, '"""Test module."""')
|
||||
result = dev.add_docstring(py_file, '"""Test module."""')
|
||||
# Should handle empty file
|
||||
assert result is True
|
||||
|
||||
@@ -98,7 +97,7 @@ class TestGenerateModuleDocstring:
|
||||
py_file = tmp_path / "test.py"
|
||||
py_file.write_text("def test():\n pass\n")
|
||||
|
||||
result = autofmt.generate_module_docstring(py_file)
|
||||
result = dev.generate_module_docstring(py_file)
|
||||
# Should contain "Tests for" since stem contains "test"
|
||||
assert "Tests for" in result
|
||||
|
||||
@@ -108,7 +107,7 @@ class TestGenerateModuleDocstring:
|
||||
py_file.parent.mkdir(parents=True)
|
||||
py_file.write_text("def test():\n pass\n")
|
||||
|
||||
result = autofmt.generate_module_docstring(py_file)
|
||||
result = dev.generate_module_docstring(py_file)
|
||||
assert "mypackage" in result
|
||||
|
||||
def test_generate_module_docstring_cli(self, tmp_path: Path) -> None:
|
||||
@@ -116,7 +115,7 @@ class TestGenerateModuleDocstring:
|
||||
py_file = tmp_path / "cli.py"
|
||||
py_file.write_text("def test():\n pass\n")
|
||||
|
||||
result = autofmt.generate_module_docstring(py_file)
|
||||
result = dev.generate_module_docstring(py_file)
|
||||
assert "Command-line interface" in result
|
||||
|
||||
def test_generate_module_docstring_util(self, tmp_path: Path) -> None:
|
||||
@@ -124,7 +123,7 @@ class TestGenerateModuleDocstring:
|
||||
py_file = tmp_path / "utils.py"
|
||||
py_file.write_text("def test():\n pass\n")
|
||||
|
||||
result = autofmt.generate_module_docstring(py_file)
|
||||
result = dev.generate_module_docstring(py_file)
|
||||
assert "Utility functions" in result
|
||||
|
||||
|
||||
@@ -139,8 +138,8 @@ class TestAutoAddDocstrings:
|
||||
py_file = tmp_path / "test.py"
|
||||
py_file.write_text("def test():\n pass\n")
|
||||
|
||||
with patch.object(autofmt, "add_docstring", return_value=True):
|
||||
count = autofmt.auto_add_docstrings(tmp_path)
|
||||
with patch.object(dev, "add_docstring", return_value=True):
|
||||
count = dev.auto_add_docstrings(tmp_path)
|
||||
assert count >= 0
|
||||
|
||||
def test_auto_add_docstrings_skips_ignored(self, tmp_path: Path) -> None:
|
||||
@@ -149,7 +148,7 @@ class TestAutoAddDocstrings:
|
||||
py_file.parent.mkdir()
|
||||
py_file.write_text("def test():\n pass\n")
|
||||
|
||||
count = autofmt.auto_add_docstrings(tmp_path)
|
||||
count = dev.auto_add_docstrings(tmp_path)
|
||||
# Should skip __pycache__
|
||||
assert count == 0
|
||||
|
||||
@@ -158,7 +157,7 @@ class TestAutoAddDocstrings:
|
||||
txt_file = tmp_path / "test.txt"
|
||||
txt_file.write_text("test content")
|
||||
|
||||
count = autofmt.auto_add_docstrings(tmp_path)
|
||||
count = dev.auto_add_docstrings(tmp_path)
|
||||
assert count == 0
|
||||
|
||||
|
||||
@@ -179,7 +178,7 @@ class TestSyncPyprojectConfig:
|
||||
|
||||
with patch("subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
autofmt.sync_pyproject_config(tmp_path)
|
||||
dev.sync_pyproject_config(tmp_path)
|
||||
assert mock_run.called
|
||||
|
||||
def test_sync_pyproject_config_updates_file(self, tmp_path: Path) -> None:
|
||||
@@ -193,7 +192,7 @@ class TestSyncPyprojectConfig:
|
||||
|
||||
with patch("subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
autofmt.sync_pyproject_config(tmp_path)
|
||||
dev.sync_pyproject_config(tmp_path)
|
||||
assert mock_run.called
|
||||
|
||||
|
||||
@@ -207,95 +206,13 @@ class TestFormatAll:
|
||||
"""Should run ruff format."""
|
||||
with patch("subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
autofmt.format_all(tmp_path)
|
||||
dev.format_all(tmp_path)
|
||||
assert mock_run.called
|
||||
|
||||
def test_format_all_runs_ruff_check(self, tmp_path: Path) -> None:
|
||||
"""Should run ruff check."""
|
||||
with patch("subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
autofmt.format_all(tmp_path)
|
||||
dev.format_all(tmp_path)
|
||||
# Should call ruff format and ruff check
|
||||
assert mock_run.call_count == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# main function
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestMain:
|
||||
"""Test main function."""
|
||||
|
||||
def test_main_fmt_default_target(self) -> None:
|
||||
"""main() should handle fmt command with default target."""
|
||||
with patch("sys.argv", ["autofmt", "fmt"]), patch.object(px, "run") as mock_run:
|
||||
autofmt.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_fmt_custom_target(self) -> None:
|
||||
"""main() should handle fmt command with custom target."""
|
||||
with patch("sys.argv", ["autofmt", "fmt", "--target", "src"]), patch.object(px, "run") as mock_run:
|
||||
autofmt.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_lint_default_target(self) -> None:
|
||||
"""main() should handle lint command with default target."""
|
||||
with patch("sys.argv", ["autofmt", "lint"]), patch.object(px, "run") as mock_run:
|
||||
autofmt.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_lint_with_fix(self) -> None:
|
||||
"""main() should handle lint command with fix."""
|
||||
with patch("sys.argv", ["autofmt", "lint", "--fix"]), patch.object(px, "run") as mock_run:
|
||||
autofmt.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_lint_custom_target(self) -> None:
|
||||
"""main() should handle lint command with custom target."""
|
||||
with patch("sys.argv", ["autofmt", "lint", "--target", "src"]), patch.object(px, "run") as mock_run:
|
||||
autofmt.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_doc_default_root(self) -> None:
|
||||
"""main() should handle doc command with default root."""
|
||||
with patch("sys.argv", ["autofmt", "doc"]), patch.object(px, "run") as mock_run:
|
||||
autofmt.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_doc_custom_root(self) -> None:
|
||||
"""main() should handle doc command with custom root."""
|
||||
with patch("sys.argv", ["autofmt", "doc", "--root-dir", "src"]), patch.object(px, "run") as mock_run:
|
||||
autofmt.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_sync_default_root(self) -> None:
|
||||
"""main() should handle sync command with default root."""
|
||||
with patch("sys.argv", ["autofmt", "sync"]), patch.object(px, "run") as mock_run:
|
||||
autofmt.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_sync_custom_root(self) -> None:
|
||||
"""main() should handle sync command with custom root."""
|
||||
with patch("sys.argv", ["autofmt", "sync", "--root-dir", "."]), patch.object(px, "run") as mock_run:
|
||||
autofmt.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_with_no_args_shows_help(self) -> None:
|
||||
"""main() with no args should show help."""
|
||||
with patch("sys.argv", ["autofmt"]), patch.object(autofmt, "main"):
|
||||
# Just call main, it should show help and return
|
||||
autofmt.main()
|
||||
# main() should return without calling px.run
|
||||
assert True
|
||||
|
||||
def test_main_creates_task_specs_with_verbose(self) -> None:
|
||||
"""main() should create TaskSpecs with verbose=True."""
|
||||
with patch("sys.argv", ["autofmt", "fmt"]), patch.object(px, "run") as mock_run:
|
||||
autofmt.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_uses_thread_strategy(self) -> None:
|
||||
"""main() should use thread strategy."""
|
||||
with patch("sys.argv", ["autofmt", "fmt"]), patch.object(px, "run") as mock_run:
|
||||
autofmt.main()
|
||||
# Check that strategy="thread" was used
|
||||
assert mock_run.called
|
||||
|
||||
+144
-270
@@ -1,14 +1,13 @@
|
||||
"""Tests for cli.bumpversion module."""
|
||||
"""Tests for ops.bumpversion module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.cli import bumpversion
|
||||
from pyflowx.ops import bumpversion
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -23,296 +22,171 @@ def auto_use_tmp_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
class TestBumpFileVersion:
|
||||
"""Test bump_file_version function."""
|
||||
|
||||
def test_bump_patch_version(self, tmp_path: Path) -> None:
|
||||
"""Should bump patch version correctly."""
|
||||
test_file = tmp_path / "pyproject.toml"
|
||||
test_file.write_text('version = "1.2.3"', encoding="utf-8")
|
||||
@pytest.mark.parametrize(
|
||||
("part", "expected"),
|
||||
[("patch", "1.2.4"), ("minor", "1.3.0"), ("major", "2.0.0")],
|
||||
)
|
||||
def test_bump_pyproject(self, tmp_path: Path, part: str, expected: str) -> None:
|
||||
"""pyproject.toml 三种 part 递增."""
|
||||
f = tmp_path / "pyproject.toml"
|
||||
f.write_text('version = "1.2.3"', encoding="utf-8")
|
||||
assert bumpversion.bump_file_version(f, part) == expected # type: ignore[arg-type]
|
||||
assert f'version = "{expected}"' in f.read_text(encoding="utf-8")
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "patch")
|
||||
@pytest.mark.parametrize(
|
||||
("part", "expected"),
|
||||
[("patch", "1.2.4"), ("minor", "1.3.0"), ("major", "2.0.0")],
|
||||
)
|
||||
def test_bump_init_py(self, tmp_path: Path, part: str, expected: str) -> None:
|
||||
"""__init__.py 三种 part 递增."""
|
||||
f = tmp_path / "__init__.py"
|
||||
f.write_text('__version__ = "1.2.3"', encoding="utf-8")
|
||||
assert bumpversion.bump_file_version(f, part) == expected # type: ignore[arg-type]
|
||||
assert f'__version__ = "{expected}"' in f.read_text(encoding="utf-8")
|
||||
|
||||
assert result == "1.2.4"
|
||||
assert test_file.read_text(encoding="utf-8") == 'version = "1.2.4"'
|
||||
|
||||
def test_bump_minor_version(self, tmp_path: Path) -> None:
|
||||
"""Should bump minor version correctly."""
|
||||
test_file = tmp_path / "pyproject.toml"
|
||||
test_file.write_text('version = "1.2.3"', encoding="utf-8")
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "minor")
|
||||
|
||||
assert result == "1.3.0"
|
||||
assert test_file.read_text(encoding="utf-8") == 'version = "1.3.0"'
|
||||
|
||||
def test_bump_major_version(self, tmp_path: Path) -> None:
|
||||
"""Should bump major version correctly."""
|
||||
test_file = tmp_path / "pyproject.toml"
|
||||
test_file.write_text('version = "1.2.3"', encoding="utf-8")
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "major")
|
||||
|
||||
assert result == "2.0.0"
|
||||
assert test_file.read_text(encoding="utf-8") == 'version = "2.0.0"'
|
||||
|
||||
def test_version_pattern_with_prerelease(self, tmp_path: Path) -> None:
|
||||
"""Should handle version with prerelease suffix."""
|
||||
test_file = tmp_path / "pyproject.toml"
|
||||
test_file.write_text('version = "1.2.3-alpha.1"', encoding="utf-8")
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "patch")
|
||||
|
||||
assert result == "1.2.4"
|
||||
# 预发布版本应该被清除
|
||||
content = test_file.read_text(encoding="utf-8")
|
||||
def test_prerelease_and_build_metadata_stripped(self, tmp_path: Path) -> None:
|
||||
"""prerelease 和 build metadata 应被清除."""
|
||||
f = tmp_path / "pyproject.toml"
|
||||
f.write_text('version = "1.2.3-alpha.1+build.123"', encoding="utf-8")
|
||||
assert bumpversion.bump_file_version(f, "patch") == "1.2.4"
|
||||
content = f.read_text(encoding="utf-8")
|
||||
assert "alpha" not in content
|
||||
|
||||
def test_version_pattern_with_build_metadata(self, tmp_path: Path) -> None:
|
||||
"""Should handle version with build metadata."""
|
||||
test_file = tmp_path / "pyproject.toml"
|
||||
test_file.write_text('version = "1.2.3+build.123"', encoding="utf-8")
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "patch")
|
||||
|
||||
assert result == "1.2.4"
|
||||
# 构建元数据应该被清除
|
||||
content = test_file.read_text(encoding="utf-8")
|
||||
assert "build" not in content
|
||||
|
||||
def test_no_version_found(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""Should return None when no version pattern found."""
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("no version here", encoding="utf-8")
|
||||
def test_dependencies_not_modified(self, tmp_path: Path) -> None:
|
||||
"""只更新 project version, 不动 dependencies 中的版本号."""
|
||||
f = tmp_path / "pyproject.toml"
|
||||
f.write_text(
|
||||
'[project]\nversion = "1.0.0"\ndependencies = ["lib >= 2.0.0", "other >= 3.0.0"]\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert bumpversion.bump_file_version(f, "patch") == "1.0.1"
|
||||
content = f.read_text(encoding="utf-8")
|
||||
assert 'version = "1.0.1"' in content
|
||||
assert "lib >= 2.0.0" in content
|
||||
assert "other >= 3.0.0" in content
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "patch")
|
||||
def test_no_version_pattern_returns_none(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""未匹配到版本号模式返回 None (支持类型但无版本 / 不支持的文件类型)."""
|
||||
f1 = tmp_path / "__init__.py"
|
||||
f1.write_text("# no version here", encoding="utf-8")
|
||||
assert bumpversion.bump_file_version(f1, "patch") is None
|
||||
assert "未找到版本号模式" in capsys.readouterr().out
|
||||
|
||||
assert result is None
|
||||
captured = capsys.readouterr()
|
||||
assert "未找到版本号模式" in captured.out
|
||||
|
||||
def test_utf8_encoding(self, tmp_path: Path) -> None:
|
||||
"""Should handle UTF-8 encoded files correctly."""
|
||||
test_file = tmp_path / "__init__.py"
|
||||
test_file.write_text('__version__ = "1.2.3"', encoding="utf-8")
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "patch")
|
||||
|
||||
assert result == "1.2.4"
|
||||
assert test_file.read_text(encoding="utf-8") == '__version__ = "1.2.4"'
|
||||
|
||||
def test_pyproject_toml_format(self, tmp_path: Path) -> None:
|
||||
"""Should handle pyproject.toml format correctly."""
|
||||
test_file = tmp_path / "pyproject.toml"
|
||||
content = """
|
||||
[project]
|
||||
name = "test"
|
||||
version = "0.1.0"
|
||||
description = "Test project"
|
||||
"""
|
||||
test_file.write_text(content, encoding="utf-8")
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "minor")
|
||||
|
||||
assert result == "0.2.0"
|
||||
updated = test_file.read_text(encoding="utf-8")
|
||||
assert 'version = "0.2.0"' in updated
|
||||
assert 'name = "test"' in updated
|
||||
|
||||
def test_init_py_format(self, tmp_path: Path) -> None:
|
||||
"""Should handle __init__.py format correctly."""
|
||||
test_file = tmp_path / "__init__.py"
|
||||
content = '''"""Package info."""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
'''
|
||||
test_file.write_text(content, encoding="utf-8")
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "major")
|
||||
|
||||
assert result == "2.0.0"
|
||||
updated = test_file.read_text(encoding="utf-8")
|
||||
assert '__version__ = "2.0.0"' in updated
|
||||
|
||||
def test_multiple_versions_in_file(self, tmp_path: Path) -> None:
|
||||
"""Should only bump the project version, not dependencies."""
|
||||
test_file = tmp_path / "pyproject.toml"
|
||||
content = """
|
||||
[project]
|
||||
version = "1.0.0"
|
||||
dependencies = ["lib >= 2.0.0", "other >= 3.0.0"]
|
||||
"""
|
||||
test_file.write_text(content, encoding="utf-8")
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "patch")
|
||||
|
||||
assert result == "1.0.1"
|
||||
updated = test_file.read_text(encoding="utf-8")
|
||||
assert 'version = "1.0.1"' in updated
|
||||
# 确保 dependencies 中的版本没有被更新
|
||||
assert "lib >= 2.0.0" in updated
|
||||
assert "other >= 3.0.0" in updated
|
||||
|
||||
def test_file_read_error(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""Should handle file read errors."""
|
||||
# 创建一个目录而不是文件
|
||||
test_file = tmp_path / "test_dir"
|
||||
test_file.mkdir()
|
||||
f2 = tmp_path / "test.txt"
|
||||
f2.write_text("no version here", encoding="utf-8")
|
||||
assert bumpversion.bump_file_version(f2, "patch") is None
|
||||
|
||||
def test_read_directory_raises(self, tmp_path: Path) -> None:
|
||||
"""读取目录 (名为 __init__.py) 应抛异常."""
|
||||
f = tmp_path / "__init__.py"
|
||||
f.mkdir()
|
||||
with pytest.raises(Exception): # noqa: B017
|
||||
bumpversion.bump_file_version(test_file, "patch")
|
||||
bumpversion.bump_file_version(f, "patch")
|
||||
|
||||
def test_file_write_error(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""Should handle file write errors."""
|
||||
# 在只读目录中创建文件(这个测试在某些系统上可能不适用)
|
||||
test_file = tmp_path / "readonly.toml"
|
||||
test_file.write_text('version = "1.0.0"', encoding="utf-8")
|
||||
# 设置为只读
|
||||
test_file.chmod(0o444)
|
||||
def test_write_failure_raises(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""写入失败应抛 OSError."""
|
||||
f = tmp_path / "__init__.py"
|
||||
f.write_text('__version__ = "1.0.0"', encoding="utf-8")
|
||||
|
||||
try:
|
||||
with pytest.raises(Exception): # noqa: B017
|
||||
bumpversion.bump_file_version(test_file, "patch")
|
||||
finally:
|
||||
# 恢复权限以便清理
|
||||
test_file.chmod(0o644)
|
||||
def raise_oserror(*_args: object, **_kwargs: object) -> None:
|
||||
raise OSError("write failed")
|
||||
|
||||
monkeypatch.setattr(Path, "write_text", raise_oserror)
|
||||
with pytest.raises(OSError, match="write failed"):
|
||||
bumpversion.bump_file_version(f, "patch")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# Version pattern tests
|
||||
# bump_project_version (核心 bug 修复: 不同步文件统一同步)
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestVersionPattern:
|
||||
"""Test version pattern matching."""
|
||||
class TestBumpProjectVersion:
|
||||
"""Test bump_project_version function."""
|
||||
|
||||
def test_simple_version(self, tmp_path: Path) -> None:
|
||||
"""Should match simple version."""
|
||||
test_file = tmp_path / "__init__.py"
|
||||
test_file.write_text('__version__ = "1.0.0"', encoding="utf-8")
|
||||
@staticmethod
|
||||
def _mock_subprocess(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]:
|
||||
"""Mock subprocess.run, 返回调用记录列表."""
|
||||
calls: list[list[str]] = []
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "patch")
|
||||
def fake_run(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[bytes]:
|
||||
calls.append(cmd)
|
||||
return subprocess.CompletedProcess(cmd, 0, b"", b"")
|
||||
|
||||
assert result == "1.0.1"
|
||||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||||
return calls
|
||||
|
||||
def test_version_with_zeros(self, tmp_path: Path) -> None:
|
||||
"""Should handle versions with zeros correctly."""
|
||||
test_file = tmp_path / "__init__.py"
|
||||
test_file.write_text('__version__ = "0.0.0"', encoding="utf-8")
|
||||
def test_unsynced_files_synchronized(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
"""核心 bug 修复: 不同步的文件应统一同步到同一新版本号.
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "patch")
|
||||
场景: __init__.py = 0.4.5, pyproject.toml = 0.3.5 (历史不同步)
|
||||
期望: bump patch 后两者都变为 0.4.6 (取最大值 0.4.5 作为基准 +1)
|
||||
"""
|
||||
init_file = tmp_path / "src" / "pkg" / "__init__.py"
|
||||
init_file.parent.mkdir(parents=True)
|
||||
init_file.write_text('__version__ = "0.4.5"', encoding="utf-8")
|
||||
pyproj = tmp_path / "pyproject.toml"
|
||||
pyproj.write_text('version = "0.3.5"', encoding="utf-8")
|
||||
|
||||
assert result == "0.0.1"
|
||||
calls = self._mock_subprocess(monkeypatch)
|
||||
|
||||
def test_large_version_numbers(self, tmp_path: Path) -> None:
|
||||
"""Should handle large version numbers."""
|
||||
test_file = tmp_path / "__init__.py"
|
||||
test_file.write_text('__version__ = "10.20.30"', encoding="utf-8")
|
||||
result = bumpversion.bump_project_version("patch")
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "minor")
|
||||
assert result == "0.4.6"
|
||||
assert '__version__ = "0.4.6"' in init_file.read_text(encoding="utf-8")
|
||||
assert 'version = "0.4.6"' in pyproj.read_text(encoding="utf-8")
|
||||
out = capsys.readouterr().out
|
||||
assert "基准版本: 0.4.5" in out
|
||||
assert "新版本: 0.4.6" in out
|
||||
|
||||
assert result == "10.21.0"
|
||||
add_calls = [c for c in calls if c[:2] == ["git", "add"]]
|
||||
assert len(add_calls) == 1
|
||||
# 跨平台: Windows 上 Path 转换为反斜杠, 统一用正斜杠比较
|
||||
init_path = str(init_file.relative_to(tmp_path)).replace("\\", "/")
|
||||
assert init_path in [arg.replace("\\", "/") for arg in add_calls[0]]
|
||||
assert "pyproject.toml" in add_calls[0]
|
||||
assert "." not in add_calls[0][2:]
|
||||
|
||||
def test_version_in_url(self, tmp_path: Path) -> None:
|
||||
"""Should not match version in URL or other contexts."""
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("https://example.com/v1.2.3/download", encoding="utf-8")
|
||||
tag_calls = [c for c in calls if c[:2] == ["git", "tag"]]
|
||||
assert len(tag_calls) == 1
|
||||
assert "v0.4.6" in tag_calls[0]
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "patch")
|
||||
|
||||
# 不应该匹配 URL 中的版本号
|
||||
assert result is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# Edge cases
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestEdgeCases:
|
||||
"""Test edge cases and error handling."""
|
||||
|
||||
def test_empty_file(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""Should handle empty file."""
|
||||
test_file = tmp_path / "empty.txt"
|
||||
test_file.write_text("", encoding="utf-8")
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "patch")
|
||||
|
||||
assert result is None
|
||||
captured = capsys.readouterr()
|
||||
assert "未找到版本号模式" in captured.out
|
||||
|
||||
def test_file_with_special_chars(self, tmp_path: Path) -> None:
|
||||
"""Should handle file with special characters."""
|
||||
test_file = tmp_path / "__init__.py"
|
||||
content = '# 中文注释\n__version__ = "1.0.0"\n# 特殊字符: @#$%'
|
||||
test_file.write_text(content, encoding="utf-8")
|
||||
|
||||
result = bumpversion.bump_file_version(test_file, "patch")
|
||||
|
||||
assert result == "1.0.1"
|
||||
updated = test_file.read_text(encoding="utf-8")
|
||||
assert "# 中文注释" in updated
|
||||
assert "# 特殊字符: @#$%" in updated
|
||||
|
||||
def test_consecutive_bumps(self, tmp_path: Path) -> None:
|
||||
"""Should handle consecutive version bumps correctly."""
|
||||
test_file = tmp_path / "__init__.py"
|
||||
test_file.write_text('__version__ = "1.0.0"', encoding="utf-8")
|
||||
|
||||
# 第一次 bump
|
||||
result1 = bumpversion.bump_file_version(test_file, "patch")
|
||||
assert result1 == "1.0.1"
|
||||
|
||||
# 第二次 bump
|
||||
result2 = bumpversion.bump_file_version(test_file, "minor")
|
||||
assert result2 == "1.1.0"
|
||||
|
||||
# 第三次 bump
|
||||
result3 = bumpversion.bump_file_version(test_file, "major")
|
||||
assert result3 == "2.0.0"
|
||||
|
||||
# 验证最终结果
|
||||
assert test_file.read_text(encoding="utf-8") == '__version__ = "2.0.0"'
|
||||
|
||||
|
||||
class TestBumpVersionCli:
|
||||
"""Test bumpversion CLI."""
|
||||
|
||||
def test_minor(self, tmp_path: Path) -> None:
|
||||
"""Should handle minor version bump."""
|
||||
test_file = tmp_path / "__init__.py"
|
||||
test_file.write_text('__version__ = "1.0.0"', encoding="utf-8")
|
||||
|
||||
# Mock px.run: 只真正执行第一次调用(版本更新),其余返回空 dict
|
||||
with patch("sys.argv", ["bumpversion", "minor", "--no-tag"]), patch("pyflowx.run") as mock_run:
|
||||
|
||||
def run_side_effect(graph: px.Graph, strategy: str | None = None):
|
||||
# 执行实际版本更新任务
|
||||
results = {}
|
||||
for spec in graph.specs.values():
|
||||
if spec.fn is not None and spec.args:
|
||||
results[spec.name] = spec.fn(*spec.args)
|
||||
return results
|
||||
|
||||
mock_run.side_effect = run_side_effect
|
||||
bumpversion.main()
|
||||
|
||||
# 验证版本号已更新
|
||||
assert test_file.read_text(encoding="utf-8") == '__version__ = "1.1.0"'
|
||||
|
||||
def test_no_valid_files(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""Should handle no valid files."""
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("这是一个测试文件", encoding="utf-8")
|
||||
|
||||
with patch("sys.argv", ["bumpversion", "minor", "--no-tag"]), patch("pyflowx.run") as mock_run:
|
||||
|
||||
def run_side_effect(graph: px.Graph, strategy: str | None = None):
|
||||
# 执行实际版本更新任务
|
||||
results = {}
|
||||
for spec in graph.specs.values():
|
||||
if spec.fn is not None and spec.args:
|
||||
results[spec.name] = spec.fn(*spec.args)
|
||||
return results
|
||||
|
||||
mock_run.side_effect = run_side_effect
|
||||
bumpversion.main()
|
||||
|
||||
# 验证未更新任何文件
|
||||
assert test_file.read_text(encoding="utf-8") == "这是一个测试文件"
|
||||
def test_no_files_returns_none(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""无版本号文件返回 None."""
|
||||
assert bumpversion.bump_project_version("patch") is None
|
||||
assert "未找到包含版本号的文件" in capsys.readouterr().out
|
||||
|
||||
def test_files_without_version_returns_none(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
"""有文件但所有文件都无版本号返回 None."""
|
||||
f = tmp_path / "__init__.py"
|
||||
f.write_text("# no version here", encoding="utf-8")
|
||||
self._mock_subprocess(monkeypatch)
|
||||
|
||||
assert bumpversion.bump_project_version("patch") is None
|
||||
assert "未能从任何文件读取版本号" in capsys.readouterr().out
|
||||
|
||||
def test_no_tag_skips_tag_creation(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""no_tag=True 跳过 tag 创建."""
|
||||
pyproj = tmp_path / "pyproject.toml"
|
||||
pyproj.write_text('version = "1.0.0"', encoding="utf-8")
|
||||
|
||||
calls = self._mock_subprocess(monkeypatch)
|
||||
|
||||
assert bumpversion.bump_project_version("patch", no_tag=True) == "1.0.1"
|
||||
assert not any(c[:2] == ["git", "tag"] for c in calls)
|
||||
|
||||
def test_ignored_dirs_excluded(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
""".venv 等忽略目录中的版本号文件不被处理."""
|
||||
venv_init = tmp_path / ".venv" / "lib" / "pkg" / "__init__.py"
|
||||
venv_init.parent.mkdir(parents=True)
|
||||
venv_init.write_text('__version__ = "0.1.0"', encoding="utf-8")
|
||||
pyproj = tmp_path / "pyproject.toml"
|
||||
pyproj.write_text('version = "1.0.0"', encoding="utf-8")
|
||||
|
||||
self._mock_subprocess(monkeypatch)
|
||||
|
||||
assert bumpversion.bump_project_version("patch") == "1.0.1"
|
||||
assert venv_init.read_text(encoding="utf-8") == '__version__ = "0.1.0"'
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
"""Tests for cli.clearscreen module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.cli import clearscreen
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# main function
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestMain:
|
||||
"""Test main function."""
|
||||
|
||||
def test_main_creates_graph_and_runs(self) -> None:
|
||||
"""main() should create a Graph and run it."""
|
||||
with patch.object(px, "run") as mock_run:
|
||||
clearscreen.main()
|
||||
assert mock_run.called
|
||||
+210
-182
@@ -125,19 +125,21 @@ class TestEmailDatabase:
|
||||
|
||||
# Insert test emails
|
||||
for i in range(5):
|
||||
db.insert_email({
|
||||
"file_path": f"/test/path{i}.eml",
|
||||
"file_hash": f"hash{i}",
|
||||
"subject": f"Subject {i}",
|
||||
"sender": f"sender{i}@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": f"Mon, {i + 1} Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": f"2024-01-0{i + 1}T12:00:00",
|
||||
"body_text": f"Body {i}",
|
||||
"body_html": f"<p>Body {i}</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
})
|
||||
db.insert_email(
|
||||
{
|
||||
"file_path": f"/test/path{i}.eml",
|
||||
"file_hash": f"hash{i}",
|
||||
"subject": f"Subject {i}",
|
||||
"sender": f"sender{i}@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": f"Mon, {i + 1} Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": f"2024-01-0{i + 1}T12:00:00",
|
||||
"body_text": f"Body {i}",
|
||||
"body_html": f"<p>Body {i}</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
}
|
||||
)
|
||||
|
||||
results = db.search_emails(limit=3)
|
||||
assert len(results) == 3
|
||||
@@ -148,33 +150,37 @@ class TestEmailDatabase:
|
||||
db_path = tmp_path / "test.db"
|
||||
db = emlmanager.EmailDatabase(db_path)
|
||||
|
||||
db.insert_email({
|
||||
"file_path": "/test/path1.eml",
|
||||
"file_hash": "hash1",
|
||||
"subject": "Important Meeting",
|
||||
"sender": "sender1@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Mon, 1 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-01T12:00:00",
|
||||
"body_text": "Meeting body",
|
||||
"body_html": "<p>Meeting body</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
})
|
||||
db.insert_email(
|
||||
{
|
||||
"file_path": "/test/path1.eml",
|
||||
"file_hash": "hash1",
|
||||
"subject": "Important Meeting",
|
||||
"sender": "sender1@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Mon, 1 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-01T12:00:00",
|
||||
"body_text": "Meeting body",
|
||||
"body_html": "<p>Meeting body</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
}
|
||||
)
|
||||
|
||||
db.insert_email({
|
||||
"file_path": "/test/path2.eml",
|
||||
"file_hash": "hash2",
|
||||
"subject": "Casual Chat",
|
||||
"sender": "sender2@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Tue, 2 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-02T12:00:00",
|
||||
"body_text": "Chat body",
|
||||
"body_html": "<p>Chat body</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
})
|
||||
db.insert_email(
|
||||
{
|
||||
"file_path": "/test/path2.eml",
|
||||
"file_hash": "hash2",
|
||||
"subject": "Casual Chat",
|
||||
"sender": "sender2@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Tue, 2 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-02T12:00:00",
|
||||
"body_text": "Chat body",
|
||||
"body_html": "<p>Chat body</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
}
|
||||
)
|
||||
|
||||
results = db.search_emails(keyword="Meeting", field="subject")
|
||||
assert len(results) == 1
|
||||
@@ -186,33 +192,37 @@ class TestEmailDatabase:
|
||||
db_path = tmp_path / "test.db"
|
||||
db = emlmanager.EmailDatabase(db_path)
|
||||
|
||||
db.insert_email({
|
||||
"file_path": "/test/path1.eml",
|
||||
"file_hash": "hash1",
|
||||
"subject": "Test",
|
||||
"sender": "alice@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Mon, 1 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-01T12:00:00",
|
||||
"body_text": "Body",
|
||||
"body_html": "<p>Body</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
})
|
||||
db.insert_email(
|
||||
{
|
||||
"file_path": "/test/path1.eml",
|
||||
"file_hash": "hash1",
|
||||
"subject": "Test",
|
||||
"sender": "alice@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Mon, 1 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-01T12:00:00",
|
||||
"body_text": "Body",
|
||||
"body_html": "<p>Body</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
}
|
||||
)
|
||||
|
||||
db.insert_email({
|
||||
"file_path": "/test/path2.eml",
|
||||
"file_hash": "hash2",
|
||||
"subject": "Test",
|
||||
"sender": "bob@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Tue, 2 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-02T12:00:00",
|
||||
"body_text": "Body",
|
||||
"body_html": "<p>Body</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
})
|
||||
db.insert_email(
|
||||
{
|
||||
"file_path": "/test/path2.eml",
|
||||
"file_hash": "hash2",
|
||||
"subject": "Test",
|
||||
"sender": "bob@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Tue, 2 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-02T12:00:00",
|
||||
"body_text": "Body",
|
||||
"body_html": "<p>Body</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
}
|
||||
)
|
||||
|
||||
results = db.search_emails(keyword="alice", field="sender")
|
||||
assert len(results) == 1
|
||||
@@ -224,19 +234,21 @@ class TestEmailDatabase:
|
||||
db_path = tmp_path / "test.db"
|
||||
db = emlmanager.EmailDatabase(db_path)
|
||||
|
||||
db.insert_email({
|
||||
"file_path": "/test/path1.eml",
|
||||
"file_hash": "hash1",
|
||||
"subject": "Project Update",
|
||||
"sender": "manager@example.com",
|
||||
"recipients": "team@example.com",
|
||||
"date": "Mon, 1 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-01T12:00:00",
|
||||
"body_text": "Please review the quarterly report",
|
||||
"body_html": "<p>Please review the quarterly report</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
})
|
||||
db.insert_email(
|
||||
{
|
||||
"file_path": "/test/path1.eml",
|
||||
"file_hash": "hash1",
|
||||
"subject": "Project Update",
|
||||
"sender": "manager@example.com",
|
||||
"recipients": "team@example.com",
|
||||
"date": "Mon, 1 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-01T12:00:00",
|
||||
"body_text": "Please review the quarterly report",
|
||||
"body_html": "<p>Please review the quarterly report</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
}
|
||||
)
|
||||
|
||||
# Search for keyword in subject
|
||||
results = db.search_emails(keyword="Project", field="all")
|
||||
@@ -253,47 +265,53 @@ class TestEmailDatabase:
|
||||
db = emlmanager.EmailDatabase(db_path)
|
||||
|
||||
# Insert emails with same subject (different prefixes)
|
||||
db.insert_email({
|
||||
"file_path": "/test/path1.eml",
|
||||
"file_hash": "hash1",
|
||||
"subject": "Meeting Tomorrow",
|
||||
"sender": "sender1@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Mon, 1 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-01T12:00:00",
|
||||
"body_text": "Body 1",
|
||||
"body_html": "<p>Body 1</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
})
|
||||
db.insert_email(
|
||||
{
|
||||
"file_path": "/test/path1.eml",
|
||||
"file_hash": "hash1",
|
||||
"subject": "Meeting Tomorrow",
|
||||
"sender": "sender1@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Mon, 1 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-01T12:00:00",
|
||||
"body_text": "Body 1",
|
||||
"body_html": "<p>Body 1</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
}
|
||||
)
|
||||
|
||||
db.insert_email({
|
||||
"file_path": "/test/path2.eml",
|
||||
"file_hash": "hash2",
|
||||
"subject": "Re: Meeting Tomorrow",
|
||||
"sender": "sender2@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Tue, 2 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-02T12:00:00",
|
||||
"body_text": "Body 2",
|
||||
"body_html": "<p>Body 2</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
})
|
||||
db.insert_email(
|
||||
{
|
||||
"file_path": "/test/path2.eml",
|
||||
"file_hash": "hash2",
|
||||
"subject": "Re: Meeting Tomorrow",
|
||||
"sender": "sender2@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Tue, 2 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-02T12:00:00",
|
||||
"body_text": "Body 2",
|
||||
"body_html": "<p>Body 2</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
}
|
||||
)
|
||||
|
||||
db.insert_email({
|
||||
"file_path": "/test/path3.eml",
|
||||
"file_hash": "hash3",
|
||||
"subject": "Different Topic",
|
||||
"sender": "sender3@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Wed, 3 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-03T12:00:00",
|
||||
"body_text": "Body 3",
|
||||
"body_html": "<p>Body 3</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
})
|
||||
db.insert_email(
|
||||
{
|
||||
"file_path": "/test/path3.eml",
|
||||
"file_hash": "hash3",
|
||||
"subject": "Different Topic",
|
||||
"sender": "sender3@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Wed, 3 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-03T12:00:00",
|
||||
"body_text": "Body 3",
|
||||
"body_html": "<p>Body 3</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
}
|
||||
)
|
||||
|
||||
grouped = db.get_grouped_emails()
|
||||
# Should have 2 groups: "Meeting Tomorrow" and "Different Topic"
|
||||
@@ -322,19 +340,21 @@ class TestEmailDatabase:
|
||||
assert db.get_email_count() == 0
|
||||
|
||||
for i in range(3):
|
||||
db.insert_email({
|
||||
"file_path": f"/test/path{i}.eml",
|
||||
"file_hash": f"hash{i}",
|
||||
"subject": f"Subject {i}",
|
||||
"sender": f"sender{i}@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": f"Mon, {i + 1} Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": f"2024-01-0{i + 1}T12:00:00",
|
||||
"body_text": f"Body {i}",
|
||||
"body_html": f"<p>Body {i}</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
})
|
||||
db.insert_email(
|
||||
{
|
||||
"file_path": f"/test/path{i}.eml",
|
||||
"file_hash": f"hash{i}",
|
||||
"subject": f"Subject {i}",
|
||||
"sender": f"sender{i}@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": f"Mon, {i + 1} Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": f"2024-01-0{i + 1}T12:00:00",
|
||||
"body_text": f"Body {i}",
|
||||
"body_html": f"<p>Body {i}</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
}
|
||||
)
|
||||
|
||||
assert db.get_email_count() == 3
|
||||
db.close()
|
||||
@@ -346,19 +366,21 @@ class TestEmailDatabase:
|
||||
|
||||
# Insert some emails
|
||||
for i in range(3):
|
||||
db.insert_email({
|
||||
"file_path": f"/test/path{i}.eml",
|
||||
"file_hash": f"hash{i}",
|
||||
"subject": f"Subject {i}",
|
||||
"sender": f"sender{i}@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": f"Mon, {i + 1} Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": f"2024-01-0{i + 1}T12:00:00",
|
||||
"body_text": f"Body {i}",
|
||||
"body_html": f"<p>Body {i}</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
})
|
||||
db.insert_email(
|
||||
{
|
||||
"file_path": f"/test/path{i}.eml",
|
||||
"file_hash": f"hash{i}",
|
||||
"subject": f"Subject {i}",
|
||||
"sender": f"sender{i}@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": f"Mon, {i + 1} Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": f"2024-01-0{i + 1}T12:00:00",
|
||||
"body_text": f"Body {i}",
|
||||
"body_html": f"<p>Body {i}</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
}
|
||||
)
|
||||
|
||||
assert db.get_email_count() == 3
|
||||
|
||||
@@ -687,19 +709,21 @@ class TestEmlManagerHandler:
|
||||
|
||||
# Insert some emails
|
||||
for i in range(3):
|
||||
db.insert_email({
|
||||
"file_path": f"/test/path{i}.eml",
|
||||
"file_hash": f"hash{i}",
|
||||
"subject": f"Subject {i}",
|
||||
"sender": f"sender{i}@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": f"Mon, {i + 1} Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": f"2024-01-0{i + 1}T12:00:00",
|
||||
"body_text": f"Body {i}",
|
||||
"body_html": f"<p>Body {i}</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
})
|
||||
db.insert_email(
|
||||
{
|
||||
"file_path": f"/test/path{i}.eml",
|
||||
"file_hash": f"hash{i}",
|
||||
"subject": f"Subject {i}",
|
||||
"sender": f"sender{i}@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": f"Mon, {i + 1} Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": f"2024-01-0{i + 1}T12:00:00",
|
||||
"body_text": f"Body {i}",
|
||||
"body_html": f"<p>Body {i}</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
}
|
||||
)
|
||||
|
||||
# Create a mock handler instance without calling __init__
|
||||
handler = Mock(spec=emlmanager.EmlManagerHandler)
|
||||
@@ -721,19 +745,21 @@ class TestEmlManagerHandler:
|
||||
db = emlmanager.EmailDatabase(db_path)
|
||||
|
||||
# Insert test email
|
||||
db.insert_email({
|
||||
"file_path": "/test/path.eml",
|
||||
"file_hash": "hash",
|
||||
"subject": "Test Subject",
|
||||
"sender": "sender@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Mon, 1 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-01T12:00:00",
|
||||
"body_text": "Test body",
|
||||
"body_html": "<p>Test body</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
})
|
||||
db.insert_email(
|
||||
{
|
||||
"file_path": "/test/path.eml",
|
||||
"file_hash": "hash",
|
||||
"subject": "Test Subject",
|
||||
"sender": "sender@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Mon, 1 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-01T12:00:00",
|
||||
"body_text": "Test body",
|
||||
"body_html": "<p>Test body</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
}
|
||||
)
|
||||
|
||||
# Create a mock handler instance without calling __init__
|
||||
handler = Mock(spec=emlmanager.EmlManagerHandler)
|
||||
@@ -756,19 +782,21 @@ class TestEmlManagerHandler:
|
||||
db = emlmanager.EmailDatabase(db_path)
|
||||
|
||||
# Insert test email
|
||||
db.insert_email({
|
||||
"file_path": "/test/path.eml",
|
||||
"file_hash": "hash",
|
||||
"subject": "Test Subject",
|
||||
"sender": "sender@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Mon, 1 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-01T12:00:00",
|
||||
"body_text": "Test body",
|
||||
"body_html": "<p>Test body</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
})
|
||||
db.insert_email(
|
||||
{
|
||||
"file_path": "/test/path.eml",
|
||||
"file_hash": "hash",
|
||||
"subject": "Test Subject",
|
||||
"sender": "sender@example.com",
|
||||
"recipients": "recipient@example.com",
|
||||
"date": "Mon, 1 Jan 2024 12:00:00 +0000",
|
||||
"date_parsed": "2024-01-01T12:00:00",
|
||||
"body_text": "Test body",
|
||||
"body_html": "<p>Test body</p>",
|
||||
"has_attachments": 0,
|
||||
"file_size": 1024,
|
||||
}
|
||||
)
|
||||
|
||||
assert db.get_email_count() == 1
|
||||
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
"""Tests for ops.envdev / ops.dockercmd 模块 (镜像源配置/Docker 登录)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from pyflowx.conditions import Constants
|
||||
from pyflowx.ops.dockercmd import docker_login_tencent
|
||||
from pyflowx.ops.envdev import (
|
||||
download_rustup_script,
|
||||
install_linux_docker,
|
||||
install_linux_fonts,
|
||||
install_linux_qt_libs,
|
||||
install_rust_toolchain,
|
||||
setup_conda_mirror,
|
||||
setup_linux_system_mirror,
|
||||
setup_python_mirror,
|
||||
setup_rust_mirror,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# setup_python_mirror
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestSetupPythonMirror:
|
||||
"""``setup_python_mirror`` 函数测试."""
|
||||
|
||||
def test_unknown_mirror_skips(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""未知镜像源应打印提示并跳过."""
|
||||
monkeypatch.setattr(subprocess, "run", lambda *_, **__: MagicMock())
|
||||
setup_python_mirror("unknown_mirror")
|
||||
captured = capsys.readouterr()
|
||||
assert "未知 Python 镜像源" in captured.out
|
||||
|
||||
def test_known_mirror_writes_config(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""已知镜像源应写入配置文件."""
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
setup_python_mirror("tsinghua")
|
||||
# Linux 平台默认配置路径
|
||||
config_path = tmp_path / ".pip" / "pip.conf"
|
||||
if not config_path.exists():
|
||||
config_path = tmp_path / "pip" / "pip.ini"
|
||||
assert config_path.exists()
|
||||
content = config_path.read_text(encoding="utf-8")
|
||||
assert "pypi.tuna.tsinghua.edu.cn" in content
|
||||
|
||||
def test_linux_uses_pip_conf(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""Linux 平台应写入 ~/.pip/pip.conf."""
|
||||
monkeypatch.setattr(Constants, "IS_LINUX", True)
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
setup_python_mirror("tsinghua")
|
||||
config_path = tmp_path / ".pip" / "pip.conf"
|
||||
assert config_path.exists()
|
||||
|
||||
def test_non_linux_uses_pip_ini(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""非 Linux 平台应写入 ~/pip/pip.ini."""
|
||||
monkeypatch.setattr(Constants, "IS_LINUX", False)
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
setup_python_mirror("tsinghua")
|
||||
config_path = tmp_path / "pip" / "pip.ini"
|
||||
assert config_path.exists()
|
||||
|
||||
def test_sets_env_vars(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""应设置 PIP_INDEX_URL 等环境变量."""
|
||||
import os
|
||||
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
monkeypatch.setattr(os, "environ", {})
|
||||
|
||||
setup_python_mirror("aliyun")
|
||||
|
||||
assert "PIP_INDEX_URL" in os.environ
|
||||
assert "mirrors.aliyun.com" in os.environ["PIP_INDEX_URL"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# setup_conda_mirror
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestSetupCondaMirror:
|
||||
"""``setup_conda_mirror`` 函数测试."""
|
||||
|
||||
def test_unknown_mirror_skips(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""未知镜像源应跳过."""
|
||||
setup_conda_mirror("unknown")
|
||||
captured = capsys.readouterr()
|
||||
assert "未知 Conda 镜像源" in captured.out
|
||||
|
||||
def test_known_mirror_writes_condarc(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""已知镜像源应写入 ~/.condarc."""
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
setup_conda_mirror("tsinghua")
|
||||
condarc = tmp_path / ".condarc"
|
||||
assert condarc.exists()
|
||||
content = condarc.read_text(encoding="utf-8")
|
||||
assert "tsinghua" in content
|
||||
assert "channels:" in content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# setup_rust_mirror
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestSetupRustMirror:
|
||||
"""``setup_rust_mirror`` 函数测试."""
|
||||
|
||||
def test_unknown_mirror_skips(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""未知镜像源应跳过."""
|
||||
setup_rust_mirror("unknown")
|
||||
captured = capsys.readouterr()
|
||||
assert "未知 Rust 镜像源" in captured.out
|
||||
|
||||
def test_known_mirror_writes_config(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""已知镜像源应写入 ~/.cargo/config.toml."""
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
setup_rust_mirror("ustc", "nightly")
|
||||
config = tmp_path / ".cargo" / "config.toml"
|
||||
assert config.exists()
|
||||
content = config.read_text(encoding="utf-8")
|
||||
assert "ustc" in content
|
||||
|
||||
def test_creates_sccache_dir(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""应创建 sccache 缓存目录."""
|
||||
from pyflowx.ops import envdev as envdev_module
|
||||
|
||||
fake_sccache = tmp_path / ".cargo" / "sccache"
|
||||
monkeypatch.setattr(envdev_module, "_RUST_SCCACHE_DIR", fake_sccache)
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
setup_rust_mirror("tsinghua")
|
||||
assert fake_sccache.exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# docker_login_tencent
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestDockerLoginTencent:
|
||||
"""``docker_login_tencent`` 函数测试."""
|
||||
|
||||
def test_default_username_uses_getpass(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""未提供 username 时应使用 getpass.getuser."""
|
||||
monkeypatch.setattr("getpass.getuser", lambda: "testuser")
|
||||
ran_cmds: list[list[str]] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: ran_cmds.append(cmd) or MagicMock())
|
||||
|
||||
docker_login_tencent()
|
||||
|
||||
assert ran_cmds[0][0] == "docker"
|
||||
assert ran_cmds[0][1] == "login"
|
||||
assert "testuser" in ran_cmds[0]
|
||||
assert "ccr.ccs.tencentyun.com" in ran_cmds[0]
|
||||
|
||||
def test_custom_username(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""提供 username 时应使用自定义用户名."""
|
||||
ran_cmds: list[list[str]] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: ran_cmds.append(cmd) or MagicMock())
|
||||
|
||||
docker_login_tencent("myuser")
|
||||
|
||||
assert "myuser" in ran_cmds[0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# setup_linux_system_mirror
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestSetupLinuxSystemMirror:
|
||||
"""``setup_linux_system_mirror`` 函数测试."""
|
||||
|
||||
def test_non_linux_skips(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""非 Linux 平台应跳过."""
|
||||
monkeypatch.setattr(Constants, "IS_LINUX", False)
|
||||
called: list[str] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: called.append(cmd))
|
||||
|
||||
setup_linux_system_mirror()
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert "仅在 Linux 上执行" in captured.out
|
||||
assert called == []
|
||||
|
||||
def test_linux_already_configured_skips(
|
||||
self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
"""Linux 上已配置国内镜像时应跳过."""
|
||||
monkeypatch.setattr(Constants, "IS_LINUX", True)
|
||||
|
||||
def fake_read_text(self: Path, encoding: str = "utf-8") -> str:
|
||||
return "tsinghua mirror configured"
|
||||
|
||||
monkeypatch.setattr(Path, "read_text", fake_read_text)
|
||||
called: list[str] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: called.append(cmd))
|
||||
|
||||
setup_linux_system_mirror()
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert "已配置" in captured.out
|
||||
assert called == []
|
||||
|
||||
def test_linux_not_configured_runs_script(
|
||||
self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
"""Linux 上未配置镜像时应执行下载与安装脚本."""
|
||||
monkeypatch.setattr(Constants, "IS_LINUX", True)
|
||||
|
||||
def fake_read_text(self: Path, encoding: str = "utf-8") -> str:
|
||||
raise OSError("file not found")
|
||||
|
||||
monkeypatch.setattr(Path, "read_text", fake_read_text)
|
||||
ran_cmds: list[str] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: ran_cmds.append(cmd) or MagicMock())
|
||||
|
||||
setup_linux_system_mirror()
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert "下载" in captured.out
|
||||
assert "安装" in captured.out
|
||||
assert len(ran_cmds) == 2
|
||||
|
||||
def test_linux_content_without_mirror_runs_script(
|
||||
self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
"""Linux 上文件存在但不包含镜像关键词时应执行脚本."""
|
||||
monkeypatch.setattr(Constants, "IS_LINUX", True)
|
||||
|
||||
def fake_read_text(self: Path, encoding: str = "utf-8") -> str:
|
||||
return "deb http://archive.ubuntu.com/ubuntu/ jammy main"
|
||||
|
||||
monkeypatch.setattr(Path, "read_text", fake_read_text)
|
||||
ran_cmds: list[str] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: ran_cmds.append(cmd) or MagicMock())
|
||||
|
||||
setup_linux_system_mirror()
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert "下载" in captured.out
|
||||
assert len(ran_cmds) == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# install_linux_qt_libs / install_linux_fonts / install_linux_docker
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestLinuxInstallers:
|
||||
"""Linux 专用安装函数测试."""
|
||||
|
||||
def test_qt_libs_non_linux_skips(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""非 Linux 上 install_linux_qt_libs 应跳过."""
|
||||
monkeypatch.setattr(Constants, "IS_LINUX", False)
|
||||
called: list[list[str]] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: called.append(cmd))
|
||||
install_linux_qt_libs()
|
||||
captured = capsys.readouterr()
|
||||
assert "仅在 Linux" in captured.out
|
||||
assert called == []
|
||||
|
||||
def test_qt_libs_linux_runs_apt(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Linux 上应执行 apt install."""
|
||||
monkeypatch.setattr(Constants, "IS_LINUX", True)
|
||||
ran_cmds: list[list[str]] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: ran_cmds.append(cmd) or MagicMock())
|
||||
install_linux_qt_libs()
|
||||
assert ran_cmds[0][0] == "sudo"
|
||||
assert "apt" in ran_cmds[0]
|
||||
assert "install" in ran_cmds[0]
|
||||
|
||||
def test_fonts_non_linux_skips(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""非 Linux 上 install_linux_fonts 应跳过."""
|
||||
monkeypatch.setattr(Constants, "IS_LINUX", False)
|
||||
called: list[list[str]] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: called.append(cmd))
|
||||
install_linux_fonts()
|
||||
captured = capsys.readouterr()
|
||||
assert "仅在 Linux" in captured.out
|
||||
assert called == []
|
||||
|
||||
def test_fonts_linux_runs_apt(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Linux 上应执行 apt install 字体包."""
|
||||
monkeypatch.setattr(Constants, "IS_LINUX", True)
|
||||
ran_cmds: list[list[str]] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: ran_cmds.append(cmd) or MagicMock())
|
||||
install_linux_fonts()
|
||||
assert "fonts-noto-cjk" in ran_cmds[0]
|
||||
|
||||
def test_docker_non_linux_skips(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""非 Linux 上 install_linux_docker 应跳过."""
|
||||
monkeypatch.setattr(Constants, "IS_LINUX", False)
|
||||
called: list[list[str]] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: called.append(cmd))
|
||||
install_linux_docker()
|
||||
captured = capsys.readouterr()
|
||||
assert "仅在 Linux" in captured.out
|
||||
assert called == []
|
||||
|
||||
def test_docker_linux_runs_install_and_usermod(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Linux 上应执行 apt install docker-compose-v2 和 usermod."""
|
||||
monkeypatch.setattr(Constants, "IS_LINUX", True)
|
||||
monkeypatch.setattr("getpass.getuser", lambda: "testuser")
|
||||
ran_cmds: list[list[str]] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: ran_cmds.append(cmd) or MagicMock())
|
||||
|
||||
install_linux_docker()
|
||||
|
||||
assert any("docker-compose-v2" in cmd for cmd in ran_cmds)
|
||||
assert any("usermod" in cmd and "docker" in cmd for cmd in ran_cmds)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# download_rustup_script
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestDownloadRustupScript:
|
||||
"""``download_rustup_script`` 函数测试."""
|
||||
|
||||
def test_rustup_installed_skips(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""rustup 已安装时应跳过."""
|
||||
monkeypatch.setattr("shutil.which", lambda _cmd: "/usr/bin/rustup")
|
||||
called: list[list[str]] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: called.append(cmd))
|
||||
|
||||
download_rustup_script()
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert "已安装" in captured.out
|
||||
assert called == []
|
||||
|
||||
def test_windows_downloads_exe(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Windows 上应下载 rustup-init.exe."""
|
||||
monkeypatch.setattr("shutil.which", lambda _cmd: None)
|
||||
monkeypatch.setattr(Constants, "IS_WINDOWS", True)
|
||||
ran_cmds: list[list[str]] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: ran_cmds.append(cmd) or MagicMock())
|
||||
|
||||
download_rustup_script()
|
||||
|
||||
assert "powershell" in ran_cmds[0]
|
||||
assert "rustup-init.exe" in ran_cmds[0]
|
||||
|
||||
def test_non_windows_downloads_sh(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""非 Windows 上应下载 rustup-init.sh."""
|
||||
monkeypatch.setattr("shutil.which", lambda _cmd: None)
|
||||
monkeypatch.setattr(Constants, "IS_WINDOWS", False)
|
||||
ran_cmds: list[list[str]] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: ran_cmds.append(cmd) or MagicMock())
|
||||
|
||||
download_rustup_script()
|
||||
|
||||
assert "curl" in ran_cmds[0]
|
||||
assert "rustup-init.sh" in ran_cmds[0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# install_rust_toolchain
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestInstallRustToolchain:
|
||||
"""``install_rust_toolchain`` 函数测试."""
|
||||
|
||||
def test_rustup_not_installed_skips(
|
||||
self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
"""rustup 未安装时应跳过."""
|
||||
monkeypatch.setattr("shutil.which", lambda _cmd: None)
|
||||
called: list[list[str]] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: called.append(cmd))
|
||||
|
||||
install_rust_toolchain()
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert "未安装" in captured.out
|
||||
assert called == []
|
||||
|
||||
def test_rustup_installed_runs_toolchain_install(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""rustup 已安装时应执行 toolchain install."""
|
||||
monkeypatch.setattr("shutil.which", lambda _cmd: "/usr/bin/rustup")
|
||||
ran_cmds: list[list[str]] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: ran_cmds.append(cmd) or MagicMock())
|
||||
|
||||
install_rust_toolchain("nightly")
|
||||
|
||||
assert ran_cmds == [["rustup", "toolchain", "install", "nightly"]]
|
||||
|
||||
def test_default_version_stable(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""默认版本应为 stable."""
|
||||
monkeypatch.setattr("shutil.which", lambda _cmd: "/usr/bin/rustup")
|
||||
ran_cmds: list[list[str]] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda cmd, **_: ran_cmds.append(cmd) or MagicMock())
|
||||
|
||||
install_rust_toolchain()
|
||||
|
||||
assert "stable" in ran_cmds[0]
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Tests for ops.envdev / sglang / msdownload 模块 (@px.tool 注册验证)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.ops import envdev, msdownload, sglang # noqa: F401 -- 触发 @px.tool 注册
|
||||
from pyflowx.tools import _TOOL_REGISTRY, get_tool
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# envdev 注册验证
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestEnvdevRegistration:
|
||||
"""``envdev`` 模块 ``@px.tool`` 注册验证."""
|
||||
|
||||
def test_envdev_registered(self) -> None:
|
||||
"""envdev 应在 _TOOL_REGISTRY 中注册."""
|
||||
assert "envdev" in _TOOL_REGISTRY
|
||||
|
||||
def test_visible_subcommands(self) -> None:
|
||||
"""可见子命令应包含核心镜像源配置命令."""
|
||||
subs = px.list_subcommands("envdev")
|
||||
assert "setup-python" in subs
|
||||
assert "setup-conda" in subs
|
||||
assert "setup-rust" in subs
|
||||
assert "setup-linux-mirror" in subs
|
||||
|
||||
def test_install_subcommands_visible(self) -> None:
|
||||
"""安装类子命令应可见."""
|
||||
subs = px.list_subcommands("envdev")
|
||||
assert "download-rustup" in subs
|
||||
assert "install-rust" in subs
|
||||
assert "install-qt-libs" in subs
|
||||
assert "install-fonts" in subs
|
||||
assert "install-docker" in subs
|
||||
|
||||
def test_setup_python_is_fn(self) -> None:
|
||||
"""setup-python 应为 fn 任务 (非 cmd)."""
|
||||
spec = get_tool("envdev", "setup-python")
|
||||
assert spec.func is not None
|
||||
assert spec.cmd is None
|
||||
|
||||
def test_setup_conda_is_fn(self) -> None:
|
||||
"""setup-conda 应为 fn 任务."""
|
||||
spec = get_tool("envdev", "setup-conda")
|
||||
assert spec.func is not None
|
||||
|
||||
def test_setup_rust_is_fn(self) -> None:
|
||||
"""setup-rust 应为 fn 任务."""
|
||||
spec = get_tool("envdev", "setup-rust")
|
||||
assert spec.func is not None
|
||||
|
||||
def test_setup_linux_mirror_is_fn(self) -> None:
|
||||
"""setup-linux-mirror 应为 fn 任务."""
|
||||
spec = get_tool("envdev", "setup-linux-mirror")
|
||||
assert spec.func is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# sglang 注册验证
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestSglangRegistration:
|
||||
"""``sglang`` 模块 ``@px.tool`` 注册验证."""
|
||||
|
||||
def test_sglang_registered(self) -> None:
|
||||
"""sglang 应在 _TOOL_REGISTRY 中注册."""
|
||||
assert "sglang" in _TOOL_REGISTRY
|
||||
|
||||
def test_visible_subcommands(self) -> None:
|
||||
"""可见子命令: install, run."""
|
||||
subs = px.list_subcommands("sglang")
|
||||
assert "install" in subs
|
||||
assert "run" in subs
|
||||
|
||||
def test_run_needs_install(self) -> None:
|
||||
"""run 应依赖 install."""
|
||||
spec = get_tool("sglang", "run")
|
||||
assert "install" in spec.needs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# msdownload 注册验证
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestMsdownloadRegistration:
|
||||
"""``msdownload`` 模块 ``@px.tool`` 注册验证."""
|
||||
|
||||
def test_msdownload_registered(self) -> None:
|
||||
"""msdownload 应在 _TOOL_REGISTRY 中注册."""
|
||||
assert "msdownload" in _TOOL_REGISTRY
|
||||
|
||||
def test_msdownload_default_spec(self) -> None:
|
||||
"""msdownload 默认 spec 应可获取且为 fn 任务."""
|
||||
spec = get_tool("msdownload")
|
||||
assert spec.func is not None
|
||||
assert spec.cmd is None
|
||||
|
||||
def test_msdownload_run_callable(self) -> None:
|
||||
"""msdownload_run 应为可调用函数."""
|
||||
assert callable(msdownload.msdownload_run)
|
||||
@@ -1,110 +0,0 @@
|
||||
"""Tests for cli.envpy module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.cli import envpy
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# set_pip_mirror
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestSetPipMirror:
|
||||
"""Test set_pip_mirror function."""
|
||||
|
||||
def test_set_pip_mirror_tsinghua(self, tmp_path: Path) -> None:
|
||||
"""Should set tsinghua mirror."""
|
||||
with patch.object(Path, "home", return_value=tmp_path):
|
||||
envpy.set_pip_mirror("tsinghua")
|
||||
# Check pip config
|
||||
pip_config = tmp_path / "pip" / "pip.ini"
|
||||
if envpy.Constants.IS_WINDOWS:
|
||||
assert pip_config.exists() or (tmp_path / "pip" / "pip.conf").exists()
|
||||
|
||||
def test_set_pip_mirror_aliyun(self, tmp_path: Path) -> None:
|
||||
"""Should set aliyun mirror."""
|
||||
with patch.object(Path, "home", return_value=tmp_path):
|
||||
envpy.set_pip_mirror("aliyun")
|
||||
# Check pip config
|
||||
pip_dir = tmp_path / "pip"
|
||||
assert pip_dir.exists()
|
||||
|
||||
def test_set_pip_mirror_with_token(self, tmp_path: Path) -> None:
|
||||
"""Should set mirror with token."""
|
||||
with patch.object(Path, "home", return_value=tmp_path):
|
||||
envpy.set_pip_mirror("tsinghua", token="test_token")
|
||||
# Check that token is set
|
||||
|
||||
def test_set_pip_mirror_creates_pip_dir(self, tmp_path: Path) -> None:
|
||||
"""Should create pip directory if it doesn't exist."""
|
||||
pip_dir = tmp_path / "pip"
|
||||
with patch.object(Path, "home", return_value=tmp_path):
|
||||
envpy.set_pip_mirror("tsinghua")
|
||||
assert pip_dir.exists()
|
||||
assert pip_dir.is_dir()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# main function
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestMain:
|
||||
"""Test main function."""
|
||||
|
||||
def test_main_mirror_tsinghua(self) -> None:
|
||||
"""main() should handle mirror tsinghua command."""
|
||||
with patch("sys.argv", ["envpy", "mirror", "tsinghua"]), patch.object(px, "run") as mock_run, patch.object(
|
||||
envpy, "set_pip_mirror"
|
||||
):
|
||||
envpy.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_mirror_aliyun(self) -> None:
|
||||
"""main() should handle mirror aliyun command."""
|
||||
with patch("sys.argv", ["envpy", "mirror", "aliyun"]), patch.object(px, "run") as mock_run, patch.object(
|
||||
envpy, "set_pip_mirror"
|
||||
):
|
||||
envpy.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_mirror_with_token(self) -> None:
|
||||
"""main() should handle mirror with token."""
|
||||
with patch("sys.argv", ["envpy", "mirror", "tsinghua", "--token", "test_token"]), patch.object(
|
||||
px, "run"
|
||||
) as mock_run, patch.object(envpy, "set_pip_mirror"):
|
||||
envpy.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_with_no_args_shows_help(self) -> None:
|
||||
"""main() with no args should show help and return."""
|
||||
with patch("sys.argv", ["envpy"]):
|
||||
envpy.main()
|
||||
# Should print help and return
|
||||
|
||||
def test_main_invalid_mirror_shows_error(self) -> None:
|
||||
"""main() with invalid mirror should show error."""
|
||||
with patch("sys.argv", ["envpy", "mirror", "invalid"]), pytest.raises(SystemExit) as exc_info:
|
||||
envpy.main()
|
||||
assert exc_info.value.code == 2
|
||||
|
||||
def test_main_creates_task_spec_with_correct_name(self) -> None:
|
||||
"""main() should create TaskSpec with correct name."""
|
||||
with patch("sys.argv", ["envpy", "mirror", "tsinghua"]), patch.object(px, "run") as mock_run, patch.object(
|
||||
envpy, "set_pip_mirror"
|
||||
):
|
||||
envpy.main()
|
||||
graph = mock_run.call_args[0][0]
|
||||
task_names = list(graph.all_specs().keys())
|
||||
assert "set_pip_mirror" in task_names
|
||||
|
||||
def test_main_uses_thread_strategy(self) -> None:
|
||||
"""main() should use thread strategy."""
|
||||
with patch("sys.argv", ["envpy", "mirror", "tsinghua"]), patch.object(px, "run") as mock_run, patch.object(
|
||||
envpy, "set_pip_mirror"
|
||||
):
|
||||
envpy.main()
|
||||
assert mock_run.call_args[1]["strategy"] == "thread"
|
||||
@@ -1,210 +0,0 @@
|
||||
"""Tests for cli.envrs module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.cli import envrs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# set_rust_mirror
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestSetRustMirror:
|
||||
"""Test set_rust_mirror function."""
|
||||
|
||||
def test_set_rust_mirror_aliyun(self, tmp_path: Path) -> None:
|
||||
"""Should set aliyun mirror."""
|
||||
with patch.object(Path, "home", return_value=tmp_path):
|
||||
envrs.set_rust_mirror("aliyun")
|
||||
# Check environment variables
|
||||
assert os.environ.get("RUSTUP_DIST_SERVER") == "https://mirrors.aliyun.com/rustup"
|
||||
assert os.environ.get("RUSTUP_UPDATE_ROOT") == "https://mirrors.aliyun.com/rustup/rustup"
|
||||
# Check cargo config
|
||||
cargo_config = tmp_path / ".cargo" / "config.toml"
|
||||
assert cargo_config.exists()
|
||||
content = cargo_config.read_text()
|
||||
assert "aliyun" in content
|
||||
|
||||
def test_set_rust_mirror_ustc(self, tmp_path: Path) -> None:
|
||||
"""Should set ustc mirror."""
|
||||
with patch.object(Path, "home", return_value=tmp_path):
|
||||
envrs.set_rust_mirror("ustc")
|
||||
assert os.environ.get("RUSTUP_DIST_SERVER") == "https://mirrors.ustc.edu.cn/rust-static"
|
||||
assert os.environ.get("RUSTUP_UPDATE_ROOT") == "https://mirrors.ustc.edu.cn/rust-static/rustup"
|
||||
|
||||
def test_set_rust_mirror_tsinghua(self, tmp_path: Path) -> None:
|
||||
"""Should set tsinghua mirror."""
|
||||
with patch.object(Path, "home", return_value=tmp_path):
|
||||
envrs.set_rust_mirror("tsinghua")
|
||||
assert os.environ.get("RUSTUP_DIST_SERVER") == "https://mirrors.tuna.tsinghua.edu.cn/rustup"
|
||||
assert os.environ.get("RUSTUP_UPDATE_ROOT") == "https://mirrors.tuna.tsinghua.edu.cn/rustup/rustup"
|
||||
|
||||
def test_set_rust_mirror_unknown_uses_default(self, tmp_path: Path) -> None:
|
||||
"""Should use default mirror for unknown mirror name."""
|
||||
with patch.object(Path, "home", return_value=tmp_path):
|
||||
# pyrefly: ignore [bad-argument-type]
|
||||
envrs.set_rust_mirror("unknown")
|
||||
# Should use default mirror (tsinghua)
|
||||
assert os.environ.get("RUSTUP_DIST_SERVER") == "https://mirrors.tuna.tsinghua.edu.cn/rustup"
|
||||
|
||||
def test_set_rust_mirror_creates_cargo_dir(self, tmp_path: Path) -> None:
|
||||
"""Should create .cargo directory if it doesn't exist."""
|
||||
cargo_dir = tmp_path / ".cargo"
|
||||
with patch.object(Path, "home", return_value=tmp_path):
|
||||
envrs.set_rust_mirror("aliyun")
|
||||
assert cargo_dir.exists()
|
||||
assert cargo_dir.is_dir()
|
||||
|
||||
def test_set_rust_mirror_prints_message(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""Should print mirror name."""
|
||||
with patch.object(Path, "home", return_value=tmp_path):
|
||||
envrs.set_rust_mirror("aliyun")
|
||||
captured = capsys.readouterr()
|
||||
assert "已设置 Rust 镜像源: aliyun" in captured.out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# install_rust
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestInstallRust:
|
||||
"""Test install_rust function."""
|
||||
|
||||
def test_install_rust_stable(self) -> None:
|
||||
"""Should install stable Rust."""
|
||||
with patch("subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
envrs.install_rust("stable")
|
||||
mock_run.assert_called_once_with(["rustup", "toolchain", "install", "stable"], check=True)
|
||||
|
||||
def test_install_rust_nightly(self) -> None:
|
||||
"""Should install nightly Rust."""
|
||||
with patch("subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
envrs.install_rust("nightly")
|
||||
mock_run.assert_called_once_with(["rustup", "toolchain", "install", "nightly"], check=True)
|
||||
|
||||
def test_install_rust_beta(self) -> None:
|
||||
"""Should install beta Rust."""
|
||||
with patch("subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
envrs.install_rust("beta")
|
||||
mock_run.assert_called_once_with(["rustup", "toolchain", "install", "beta"], check=True)
|
||||
|
||||
def test_install_rust_file_not_found(self) -> None:
|
||||
"""Should raise FileNotFoundError when rustup not found."""
|
||||
with patch("subprocess.run", side_effect=FileNotFoundError), pytest.raises(FileNotFoundError):
|
||||
envrs.install_rust("stable")
|
||||
|
||||
def test_install_rust_prints_message(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""Should print installation message."""
|
||||
with patch("subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
envrs.install_rust("stable")
|
||||
captured = capsys.readouterr()
|
||||
assert "已安装 Rust stable" in captured.out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# main function
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestMain:
|
||||
"""Test main function."""
|
||||
|
||||
def test_main_mirror_aliyun(self) -> None:
|
||||
"""main() should handle mirror aliyun command."""
|
||||
with patch("sys.argv", ["envrs", "mirror", "aliyun"]), patch.object(px, "run") as mock_run, patch.object(
|
||||
envrs, "set_rust_mirror"
|
||||
):
|
||||
envrs.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_mirror_ustc(self) -> None:
|
||||
"""main() should handle mirror ustc command."""
|
||||
with patch("sys.argv", ["envrs", "mirror", "ustc"]), patch.object(px, "run") as mock_run, patch.object(
|
||||
envrs, "set_rust_mirror"
|
||||
):
|
||||
envrs.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_mirror_tsinghua(self) -> None:
|
||||
"""main() should handle mirror tsinghua command."""
|
||||
with patch("sys.argv", ["envrs", "mirror", "tsinghua"]), patch.object(px, "run") as mock_run, patch.object(
|
||||
envrs, "set_rust_mirror"
|
||||
):
|
||||
envrs.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_mirror_default(self) -> None:
|
||||
"""main() should use default mirror when not specified."""
|
||||
with patch("sys.argv", ["envrs", "mirror"]), patch.object(px, "run") as mock_run, patch.object(
|
||||
envrs, "set_rust_mirror"
|
||||
):
|
||||
envrs.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_install_stable(self) -> None:
|
||||
"""main() should handle install stable command."""
|
||||
with patch("sys.argv", ["envrs", "install", "stable"]), patch.object(px, "run") as mock_run:
|
||||
envrs.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_install_nightly(self) -> None:
|
||||
"""main() should handle install nightly command."""
|
||||
with patch("sys.argv", ["envrs", "install", "nightly"]), patch.object(px, "run") as mock_run:
|
||||
envrs.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_install_beta(self) -> None:
|
||||
"""main() should handle install beta command."""
|
||||
with patch("sys.argv", ["envrs", "install", "beta"]), patch.object(px, "run") as mock_run:
|
||||
envrs.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_install_default(self) -> None:
|
||||
"""main() should use default version when not specified."""
|
||||
with patch("sys.argv", ["envrs", "install"]), patch.object(px, "run") as mock_run:
|
||||
envrs.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_with_no_args_shows_help(self) -> None:
|
||||
"""main() with no args should show help and return."""
|
||||
with patch("sys.argv", ["envrs"]):
|
||||
envrs.main()
|
||||
# Should print help and return
|
||||
|
||||
def test_main_invalid_version_shows_error(self) -> None:
|
||||
"""main() with invalid version should show error."""
|
||||
with patch("sys.argv", ["envrs", "install", "invalid"]), pytest.raises(SystemExit) as exc_info:
|
||||
envrs.main()
|
||||
assert exc_info.value.code == 2
|
||||
|
||||
def test_main_invalid_mirror_shows_error(self) -> None:
|
||||
"""main() with invalid mirror should show error."""
|
||||
with patch("sys.argv", ["envrs", "mirror", "invalid"]), pytest.raises(SystemExit) as exc_info:
|
||||
envrs.main()
|
||||
assert exc_info.value.code == 2
|
||||
|
||||
def test_main_creates_task_spec_with_verbose(self) -> None:
|
||||
"""main() should create TaskSpec with verbose=True."""
|
||||
with patch("sys.argv", ["envrs", "mirror", "aliyun"]), patch.object(px, "run") as mock_run, patch.object(
|
||||
envrs, "set_rust_mirror"
|
||||
):
|
||||
envrs.main()
|
||||
graph = mock_run.call_args[0][0]
|
||||
specs = graph.all_specs()
|
||||
for spec in specs.values():
|
||||
assert spec.verbose is True
|
||||
|
||||
def test_main_uses_thread_strategy(self) -> None:
|
||||
"""main() should use thread strategy."""
|
||||
with patch("sys.argv", ["envrs", "mirror", "aliyun"]), patch.object(px, "run") as mock_run, patch.object(
|
||||
envrs, "set_rust_mirror"
|
||||
):
|
||||
envrs.main()
|
||||
assert mock_run.call_args[1]["strategy"] == "thread"
|
||||
+10
-43
@@ -3,10 +3,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.cli import filedate
|
||||
from pyflowx.ops import filedate as files
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
@@ -20,7 +18,7 @@ class TestGetFileTimestamp:
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("test content")
|
||||
|
||||
timestamp = filedate.get_file_timestamp(test_file)
|
||||
timestamp = files.get_file_timestamp(test_file)
|
||||
assert len(timestamp) == 8 # YYYYMMDD format
|
||||
assert timestamp.isdigit()
|
||||
|
||||
@@ -36,7 +34,7 @@ class TestRemoveDatePrefix:
|
||||
test_file = tmp_path / "20240101_test.txt"
|
||||
test_file.write_text("test content")
|
||||
|
||||
new_path = filedate.remove_date_prefix(test_file)
|
||||
new_path = files.remove_date_prefix(test_file)
|
||||
assert new_path.name == "test.txt"
|
||||
|
||||
def test_remove_date_prefix_without_date(self, tmp_path: Path) -> None:
|
||||
@@ -44,7 +42,7 @@ class TestRemoveDatePrefix:
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("test content")
|
||||
|
||||
new_path = filedate.remove_date_prefix(test_file)
|
||||
new_path = files.remove_date_prefix(test_file)
|
||||
assert new_path == test_file
|
||||
|
||||
|
||||
@@ -59,7 +57,7 @@ class TestAddDatePrefix:
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("test content")
|
||||
|
||||
new_path = filedate.add_date_prefix(test_file)
|
||||
new_path = files.add_date_prefix(test_file)
|
||||
assert new_path.name.startswith("20") # Starts with year
|
||||
assert "_test.txt" in new_path.name
|
||||
|
||||
@@ -75,7 +73,7 @@ class TestProcessFileDate:
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("test content")
|
||||
|
||||
filedate.process_file_date(test_file, clear=False)
|
||||
files.process_file_date(test_file, clear=False)
|
||||
# File should be renamed with date prefix
|
||||
|
||||
def test_process_file_date_clear(self, tmp_path: Path) -> None:
|
||||
@@ -83,7 +81,7 @@ class TestProcessFileDate:
|
||||
test_file = tmp_path / "20240101_test.txt"
|
||||
test_file.write_text("test content")
|
||||
|
||||
filedate.process_file_date(test_file, clear=True)
|
||||
files.process_file_date(test_file, clear=True)
|
||||
# File should be renamed without date prefix
|
||||
|
||||
|
||||
@@ -95,42 +93,11 @@ class TestProcessFilesDate:
|
||||
|
||||
def test_process_files_date_batch(self, tmp_path: Path) -> None:
|
||||
"""Should process multiple files."""
|
||||
files = []
|
||||
file_list = []
|
||||
for i in range(3):
|
||||
test_file = tmp_path / f"test{i}.txt"
|
||||
test_file.write_text(f"content{i}")
|
||||
files.append(test_file)
|
||||
file_list.append(test_file)
|
||||
|
||||
filedate.process_files_date(files, clear=False)
|
||||
files.process_files_date(file_list, clear=False)
|
||||
# All files should be processed
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# main function
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestMain:
|
||||
"""Test main function."""
|
||||
|
||||
def test_main_add_command(self, tmp_path: Path) -> None:
|
||||
"""main() should handle add command."""
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("test content")
|
||||
|
||||
with patch("sys.argv", ["filedate", "add", str(test_file)]), patch.object(px, "run") as mock_run:
|
||||
filedate.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_clear_command(self, tmp_path: Path) -> None:
|
||||
"""main() should handle clear command."""
|
||||
test_file = tmp_path / "20240101_test.txt"
|
||||
test_file.write_text("test content")
|
||||
|
||||
with patch("sys.argv", ["filedate", "clear", str(test_file)]), patch.object(px, "run") as mock_run:
|
||||
filedate.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_with_no_args_shows_help(self) -> None:
|
||||
"""main() with no args should show help."""
|
||||
with patch("sys.argv", ["filedate"]):
|
||||
filedate.main()
|
||||
# Should print help and return
|
||||
|
||||
+12
-49
@@ -3,10 +3,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.cli import filelevel
|
||||
from pyflowx.ops import filelevel as files
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
@@ -18,19 +16,19 @@ class TestRemoveMarks:
|
||||
def test_remove_marks_single_mark(self) -> None:
|
||||
"""Should remove single mark."""
|
||||
stem = "filename(PUB)"
|
||||
result = filelevel.remove_marks(stem, ["PUB"])
|
||||
result = files.remove_marks(stem, ["PUB"])
|
||||
assert result == "filename"
|
||||
|
||||
def test_remove_marks_multiple_marks(self) -> None:
|
||||
"""Should remove multiple marks."""
|
||||
stem = "filename(PUB)(NOR)"
|
||||
result = filelevel.remove_marks(stem, ["PUB", "NOR"])
|
||||
result = files.remove_marks(stem, ["PUB", "NOR"])
|
||||
assert result == "filename"
|
||||
|
||||
def test_remove_marks_no_marks(self) -> None:
|
||||
"""Should not change stem without marks."""
|
||||
stem = "filename"
|
||||
result = filelevel.remove_marks(stem, ["PUB"])
|
||||
result = files.remove_marks(stem, ["PUB"])
|
||||
assert result == "filename"
|
||||
|
||||
|
||||
@@ -45,7 +43,7 @@ class TestProcessFileLevel:
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("test content")
|
||||
|
||||
filelevel.process_file_level(test_file, level=1)
|
||||
files.process_file_level(test_file, level=1)
|
||||
# File should be renamed with PUB level
|
||||
|
||||
def test_process_file_level_set_int(self, tmp_path: Path) -> None:
|
||||
@@ -53,7 +51,7 @@ class TestProcessFileLevel:
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("test content")
|
||||
|
||||
filelevel.process_file_level(test_file, level=2)
|
||||
files.process_file_level(test_file, level=2)
|
||||
# File should be renamed with INT level
|
||||
|
||||
def test_process_file_level_clear(self, tmp_path: Path) -> None:
|
||||
@@ -61,7 +59,7 @@ class TestProcessFileLevel:
|
||||
test_file = tmp_path / "test(PUB).txt"
|
||||
test_file.write_text("test content")
|
||||
|
||||
filelevel.process_file_level(test_file, level=0)
|
||||
files.process_file_level(test_file, level=0)
|
||||
# File should be renamed without level
|
||||
|
||||
def test_process_file_level_invalid_level(self, tmp_path: Path) -> None:
|
||||
@@ -69,14 +67,14 @@ class TestProcessFileLevel:
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("test content")
|
||||
|
||||
filelevel.process_file_level(test_file, level=5)
|
||||
files.process_file_level(test_file, level=5)
|
||||
# Should print error message
|
||||
|
||||
def test_process_file_level_nonexistent_file(self, tmp_path: Path) -> None:
|
||||
"""Should handle nonexistent file."""
|
||||
test_file = tmp_path / "nonexistent.txt"
|
||||
|
||||
filelevel.process_file_level(test_file, level=1)
|
||||
files.process_file_level(test_file, level=1)
|
||||
# Should print error message
|
||||
|
||||
|
||||
@@ -88,46 +86,11 @@ class TestProcessFilesLevel:
|
||||
|
||||
def test_process_files_level_batch(self, tmp_path: Path) -> None:
|
||||
"""Should process multiple files."""
|
||||
files = []
|
||||
file_list = []
|
||||
for i in range(3):
|
||||
test_file = tmp_path / f"test{i}.txt"
|
||||
test_file.write_text(f"content{i}")
|
||||
files.append(test_file)
|
||||
file_list.append(test_file)
|
||||
|
||||
filelevel.process_files_level(files, level=1)
|
||||
files.process_files_level(file_list, level=1)
|
||||
# All files should be processed
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# main function
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestMain:
|
||||
"""Test main function."""
|
||||
|
||||
def test_main_set_command(self, tmp_path: Path) -> None:
|
||||
"""main() should handle set command."""
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("test content")
|
||||
|
||||
with patch("sys.argv", ["filelevel", "set", str(test_file), "--level", "1"]), patch.object(
|
||||
px, "run"
|
||||
) as mock_run:
|
||||
filelevel.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_set_command_level_2(self, tmp_path: Path) -> None:
|
||||
"""main() should handle set command with level 2."""
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("test content")
|
||||
|
||||
with patch("sys.argv", ["filelevel", "set", str(test_file), "--level", "2"]), patch.object(
|
||||
px, "run"
|
||||
) as mock_run:
|
||||
filelevel.main()
|
||||
assert mock_run.called
|
||||
|
||||
def test_main_with_no_args_shows_help(self) -> None:
|
||||
"""main() with no args should show help."""
|
||||
with patch("sys.argv", ["filelevel"]):
|
||||
filelevel.main()
|
||||
# Should print help and return
|
||||
|
||||
@@ -5,8 +5,7 @@ from __future__ import annotations
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pyflowx as px
|
||||
from pyflowx.cli import folderback
|
||||
from pyflowx.ops import folderback as files
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
@@ -22,7 +21,7 @@ class TestRemoveDump:
|
||||
dst = tmp_path / "backup"
|
||||
dst.mkdir()
|
||||
|
||||
folderback.remove_dump(src, dst, 5)
|
||||
files.remove_dump(src, dst, 5)
|
||||
# Should not raise error
|
||||
|
||||
def test_remove_dump_within_limit(self, tmp_path: Path) -> None:
|
||||
@@ -37,7 +36,7 @@ class TestRemoveDump:
|
||||
zip_file = dst / f"source_20240101_12000{i}.zip"
|
||||
zip_file.write_bytes(b"ZIP content")
|
||||
|
||||
folderback.remove_dump(src, dst, 5)
|
||||
files.remove_dump(src, dst, 5)
|
||||
# All files should remain
|
||||
assert len(list(dst.glob("*.zip"))) == 3
|
||||
|
||||
@@ -53,7 +52,7 @@ class TestRemoveDump:
|
||||
zip_file = dst / f"source_20240101_12000{i}.zip"
|
||||
zip_file.write_bytes(b"ZIP content")
|
||||
|
||||
folderback.remove_dump(src, dst, 5)
|
||||
files.remove_dump(src, dst, 5)
|
||||
# Should have only 5 files
|
||||
assert len(list(dst.glob("*.zip"))) == 5
|
||||
|
||||
@@ -73,7 +72,7 @@ class TestZipTarget:
|
||||
dst.mkdir()
|
||||
|
||||
with patch("time.strftime", return_value="_20240101_120000"):
|
||||
folderback.zip_target(src, dst, 5)
|
||||
files.zip_target(src, dst, 5)
|
||||
|
||||
# Should create zip file
|
||||
zip_files = list(dst.glob("*.zip"))
|
||||
@@ -91,7 +90,7 @@ class TestZipTarget:
|
||||
dst.mkdir()
|
||||
|
||||
with patch("time.strftime", return_value="_20240101_120000"):
|
||||
folderback.zip_target(src, dst, 5)
|
||||
files.zip_target(src, dst, 5)
|
||||
|
||||
# Should create zip file
|
||||
zip_files = list(dst.glob("*.zip"))
|
||||
@@ -111,8 +110,8 @@ class TestBackupFolder:
|
||||
(source_dir / "test.txt").write_text("test content")
|
||||
backup_dir = tmp_path / "backup"
|
||||
|
||||
with patch.object(folderback, "zip_target") as mock_zip:
|
||||
folderback.backup_folder(str(source_dir), str(backup_dir), 5)
|
||||
with patch.object(files, "zip_target") as mock_zip:
|
||||
files.backup_folder(str(source_dir), str(backup_dir), 5)
|
||||
assert mock_zip.called
|
||||
|
||||
def test_backup_folder_with_max_backups(self, tmp_path: Path) -> None:
|
||||
@@ -122,8 +121,8 @@ class TestBackupFolder:
|
||||
(source_dir / "test.txt").write_text("test content")
|
||||
backup_dir = tmp_path / "backup"
|
||||
|
||||
with patch.object(folderback, "zip_target") as mock_zip:
|
||||
folderback.backup_folder(str(source_dir), str(backup_dir), 10)
|
||||
with patch.object(files, "zip_target") as mock_zip:
|
||||
files.backup_folder(str(source_dir), str(backup_dir), 10)
|
||||
assert mock_zip.called
|
||||
|
||||
def test_backup_folder_source_not_exists(self, tmp_path: Path) -> None:
|
||||
@@ -132,7 +131,7 @@ class TestBackupFolder:
|
||||
backup_dir = tmp_path / "backup"
|
||||
backup_dir.mkdir()
|
||||
|
||||
folderback.backup_folder(str(source_dir), str(backup_dir), 5)
|
||||
files.backup_folder(str(source_dir), str(backup_dir), 5)
|
||||
# Should print error message and return
|
||||
|
||||
def test_backup_folder_creates_dst(self, tmp_path: Path) -> None:
|
||||
@@ -142,32 +141,18 @@ class TestBackupFolder:
|
||||
(source_dir / "test.txt").write_text("test content")
|
||||
backup_dir = tmp_path / "backup"
|
||||
|
||||
with patch.object(folderback, "zip_target") as mock_zip:
|
||||
folderback.backup_folder(str(source_dir), str(backup_dir), 5)
|
||||
with patch.object(files, "zip_target") as mock_zip:
|
||||
files.backup_folder(str(source_dir), str(backup_dir), 5)
|
||||
assert backup_dir.exists()
|
||||
assert mock_zip.called
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# TaskSpec definitions
|
||||
# 函数注册
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestTaskSpecDefinitions:
|
||||
"""Test that all TaskSpec definitions are valid."""
|
||||
class TestRegisteredFunctions:
|
||||
"""Test that folderback functions are registered."""
|
||||
|
||||
def test_folderback_default_spec(self) -> None:
|
||||
"""folderback_default spec should be properly defined."""
|
||||
assert folderback.folderback_default.name == "folderback_default"
|
||||
assert folderback.folderback_default.fn is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- #
|
||||
# main function
|
||||
# ---------------------------------------------------------------------- #
|
||||
class TestMain:
|
||||
"""Test main function."""
|
||||
|
||||
def test_main_calls_run_cli(self) -> None:
|
||||
"""main() should create a CliRunner and call run_cli()."""
|
||||
with patch.object(px.CliRunner, "run_cli") as mock_run_cli:
|
||||
folderback.main()
|
||||
assert mock_run_cli.called
|
||||
"""folderback_default should be a callable function."""
|
||||
assert callable(files.folderback_default)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user