From 592d4a824eedccaffa6445ec3809bf29566c4472 Mon Sep 17 00:00:00 2001 From: lingyuzeng Date: Sun, 12 Oct 2025 19:39:12 +0800 Subject: [PATCH] first add --- .gitignore | 30 ++++ LICENSE | 21 +++ README.md | 138 ++++++++++++++++++ docs/best_practices.md | 41 ++++++ examples/01_basic/conf/config.yaml | 6 + examples/01_basic/train.py | 27 ++++ examples/02_ml_training/conf/config.yaml | 10 ++ .../02_ml_training/conf/dataset/cifar10.yaml | 5 + .../02_ml_training/conf/dataset/imagenet.yaml | 5 + .../02_ml_training/conf/model/resnet.yaml | 5 + .../conf/model/transformer.yaml | 7 + .../02_ml_training/conf/optimizer/adam.yaml | 4 + .../02_ml_training/conf/optimizer/sgd.yaml | 5 + examples/02_ml_training/train.py | 32 ++++ examples/04_multi_env/app.py | 30 ++++ examples/04_multi_env/conf/config.yaml | 11 ++ examples/04_multi_env/conf/env/dev.yaml | 9 ++ examples/04_multi_env/conf/env/prod.yaml | 9 ++ examples/04_multi_env/conf/env/test.yaml | 9 ++ pyproject.toml | 28 ++++ quickstart.sh | 33 +++++ requirements.txt | 4 + 22 files changed, 469 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 docs/best_practices.md create mode 100644 examples/01_basic/conf/config.yaml create mode 100644 examples/01_basic/train.py create mode 100644 examples/02_ml_training/conf/config.yaml create mode 100644 examples/02_ml_training/conf/dataset/cifar10.yaml create mode 100644 examples/02_ml_training/conf/dataset/imagenet.yaml create mode 100644 examples/02_ml_training/conf/model/resnet.yaml create mode 100644 examples/02_ml_training/conf/model/transformer.yaml create mode 100644 examples/02_ml_training/conf/optimizer/adam.yaml create mode 100644 examples/02_ml_training/conf/optimizer/sgd.yaml create mode 100644 examples/02_ml_training/train.py create mode 100644 examples/04_multi_env/app.py create mode 100644 examples/04_multi_env/conf/config.yaml create mode 100644 examples/04_multi_env/conf/env/dev.yaml create mode 100644 examples/04_multi_env/conf/env/prod.yaml create mode 100644 examples/04_multi_env/conf/env/test.yaml create mode 100644 pyproject.toml create mode 100755 quickstart.sh create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dd648c5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +# Python +__pycache__/ +*.py[cod] +*.so +.Python +*.egg-info/ + +# Virtual Environment +.venv/ +venv/ + +# Hydra +outputs/ +multirun/ +.hydra/ + +# IDEs +.vscode/ +.idea/ +.DS_Store + +# Data and Models +data/ +checkpoints/ +*.pth +*.pt + +# Logs +logs/ +*.log diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b77bf2a --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..eb57b05 --- /dev/null +++ b/README.md @@ -0,0 +1,138 @@ +# 🎯 Hydra ML Config Template + +
+ +**优雅的Python配置管理解决方案 | 告别杂乱的硬编码** + +[快速开始](#快速开始) • [使用场景](#使用场景) • [最佳实践](#最佳实践) • [常见问题](#常见问题) + +
+ +--- + +## 📖 项目简介 + +这是一个基于 [Hydra](https://hydra.cc/) 的Python配置管理模板仓库,专为机器学习和数据科学项目设计。通过分层配置、动态组合、命令行覆盖等特性,让你的项目配置管理变得简单优雅。 + +### ✨ 核心特性 + +- 🔧 **分层配置** - 模块化管理,配置文件清晰有序 +- 🎨 **动态组合** - 灵活切换不同配置组合 +- 🚀 **命令行覆盖** - 无需修改代码即可调整参数 +- 📊 **自动实验管理** - 每次运行自动创建独立输出目录 +- 🔄 **多任务运行** - 一次性运行多组超参数实验 +- 💾 **配置版本化** - 完整保存每次实验的配置 + +--- + +## 🚀 快速开始 + +### 环境安装 + +本项目使用 [uv](https://github.com/astral-sh/uv) 进行依赖管理,这是一个极速的Python包管理工具。 + +#### 1. 安装 uv + +```bash +# macOS/Linux +curl -LsSf https://astral.sh/uv/install.sh | sh + +# Windows +powershell -c "irm https://astral.sh/uv/install.ps1 | iex" + +# 或使用 pip +pip install uv +``` + +#### 2. 克隆仓库 + +```bash +git clone /hydra-ml-config-template.git +cd hydra-ml-config-template +``` + +#### 3. 创建虚拟环境并安装依赖 + +```bash +# 使用 uv 创建虚拟环境 +uv venv + +# 激活虚拟环境 +# Linux/macOS: +source .venv/bin/activate +# Windows: +.venv\Scripts\activate + +# 安装依赖 +uv pip install -e . +``` + +#### 4. 运行示例 + +```bash +# 基础示例 +python examples/01_basic/train.py + +# 使用不同配置 +python examples/02_ml_training/train.py model=transformer optimizer=sgd + +# 多任务运行 +python examples/02_ml_training/train.py -m learning_rate=0.001,0.01,0.1 +``` + +--- + +## 🎯 使用场景 + +### 场景1: 基础配置管理 + +```bash +cd examples/01_basic +python train.py learning_rate=0.01 +``` + +### 场景2: 机器学习训练配置 + +```bash +cd examples/02_ml_training +python train.py model=transformer optimizer=sgd +python train.py learning_rate=0.01 batch_size=64 +``` + +### 场景3: 多环境配置 + +```bash +cd examples/04_multi_env +python app.py env=dev # 开发环境 +python app.py env=prod # 生产环境 +``` + +--- + +## 📚 学习资源 + +- [Hydra官方文档](https://hydra.cc/) +- [项目结构说明](PROJECT_STRUCTURE.md) +- [最佳实践](docs/best_practices.md) + +--- + +## 🤝 贡献 + +欢迎提交Issue和Pull Request! + +--- + +## 📄 许可证 + +MIT License + +--- + +
+ +**如果这个模板对你有帮助,请给个 ⭐️ Star!** + +Made with ❤️ for ML Engineers + +
diff --git a/docs/best_practices.md b/docs/best_practices.md new file mode 100644 index 0000000..dcf3cc8 --- /dev/null +++ b/docs/best_practices.md @@ -0,0 +1,41 @@ +# 🎯 Hydra配置管理最佳实践 + +## 1. 配置文件组织 + +按功能模块组织配置文件： + +``` +conf/ +├── config.yaml +├── model/ +├── optimizer/ +└── dataset/ +``` + +## 2. 变量插值 + +使用变量插值避免重复： + +```yaml +data_dir: /data/project +train_dir: ${data_dir}/train +val_dir: ${data_dir}/val +``` + +## 3. 命令行覆盖 + +灵活调整参数： + +```bash +python train.py learning_rate=0.01 batch_size=64 +``` + +## 4. 多任务运行 + +批量实验： + +```bash +python train.py -m learning_rate=0.001,0.01,0.1 +``` + +更多信息请访问 [Hydra官方文档](https://hydra.cc/) diff --git a/examples/01_basic/conf/config.yaml b/examples/01_basic/conf/config.yaml new file mode 100644 index 0000000..fa1fb26 --- /dev/null +++ b/examples/01_basic/conf/config.yaml @@ -0,0 +1,6 @@ +# 基础配置示例 +learning_rate: 0.001 +batch_size: 32 +epochs: 100 +model_name: "ResNet50" +device: "cuda" diff --git a/examples/01_basic/train.py b/examples/01_basic/train.py new file mode 100644 index 0000000..3d92570 --- /dev/null +++ b/examples/01_basic/train.py @@ -0,0 +1,27 @@ +"""基础Hydra配置管理示例""" + +import hydra +from omegaconf import DictConfig +from rich.console import Console +from rich.table import Table + +console = Console() + +@hydra.main(version_base=None, config_path="conf", config_name="config") +def train(cfg: DictConfig) -> None: + console.print("\n[bold green]🚀 基础Hydra配置示例[/bold green]\n") + + table = Table(title="当前配置") + table.add_column("参数", style="cyan") + table.add_column("值", style="yellow") + + table.add_row("学习率", str(cfg.learning_rate)) + table.add_row("批次大小", str(cfg.batch_size)) + table.add_row("训练轮数", str(cfg.epochs)) + table.add_row("模型名称", cfg.model_name) + + console.print(table) + console.print("\n[bold green]✅ 训练完成！[/bold green]\n") + +if __name__ == "__main__": + train() diff --git a/examples/02_ml_training/conf/config.yaml b/examples/02_ml_training/conf/config.yaml new file mode 100644 index 0000000..e37b257 --- /dev/null +++ b/examples/02_ml_training/conf/config.yaml @@ -0,0 +1,10 @@ +defaults: + - model: resnet + - optimizer: adam + - dataset: cifar10 + - _self_ + +learning_rate: 0.001 +batch_size: 32 +epochs: 100 +device: cuda diff --git a/examples/02_ml_training/conf/dataset/cifar10.yaml b/examples/02_ml_training/conf/dataset/cifar10.yaml new file mode 100644 index 0000000..2ffff33 --- /dev/null +++ b/examples/02_ml_training/conf/dataset/cifar10.yaml @@ -0,0 +1,5 @@ +name: CIFAR-10 +num_classes: 10 +image_size: 32 +train_samples: 50000 +val_samples: 10000 diff --git a/examples/02_ml_training/conf/dataset/imagenet.yaml b/examples/02_ml_training/conf/dataset/imagenet.yaml new file mode 100644 index 0000000..1f3bc55 --- /dev/null +++ b/examples/02_ml_training/conf/dataset/imagenet.yaml @@ -0,0 +1,5 @@ +name: ImageNet +num_classes: 1000 +image_size: 224 +train_samples: 1281167 +val_samples: 50000 diff --git a/examples/02_ml_training/conf/model/resnet.yaml b/examples/02_ml_training/conf/model/resnet.yaml new file mode 100644 index 0000000..2691a74 --- /dev/null +++ b/examples/02_ml_training/conf/model/resnet.yaml @@ -0,0 +1,5 @@ +name: ResNet50 +layers: 50 +pretrained: true +num_classes: 1000 +dropout: 0.5 diff --git a/examples/02_ml_training/conf/model/transformer.yaml b/examples/02_ml_training/conf/model/transformer.yaml new file mode 100644 index 0000000..d06eec2 --- /dev/null +++ b/examples/02_ml_training/conf/model/transformer.yaml @@ -0,0 +1,7 @@ +name: ViT-B/16 +layers: 12 +pretrained: true +num_classes: 1000 +hidden_dim: 768 +num_heads: 12 +patch_size: 16 diff --git a/examples/02_ml_training/conf/optimizer/adam.yaml b/examples/02_ml_training/conf/optimizer/adam.yaml new file mode 100644 index 0000000..ca855dd --- /dev/null +++ b/examples/02_ml_training/conf/optimizer/adam.yaml @@ -0,0 +1,4 @@ +name: Adam +lr: ${learning_rate} +betas: [0.9, 0.999] +weight_decay: 0.0001 diff --git a/examples/02_ml_training/conf/optimizer/sgd.yaml b/examples/02_ml_training/conf/optimizer/sgd.yaml new file mode 100644 index 0000000..0383cb9 --- /dev/null +++ b/examples/02_ml_training/conf/optimizer/sgd.yaml @@ -0,0 +1,5 @@ +name: SGD +lr: ${learning_rate} +momentum: 0.9 +weight_decay: 0.0005 +nesterov: true diff --git a/examples/02_ml_training/train.py b/examples/02_ml_training/train.py new file mode 100644 index 0000000..4f57f8e --- /dev/null +++ b/examples/02_ml_training/train.py @@ -0,0 +1,32 @@ +"""机器学习训练配置示例""" + +import hydra +from omegaconf import DictConfig +from rich.console import Console +from rich.tree import Tree + +console = Console() + +@hydra.main(version_base=None, config_path="conf", config_name="config") +def train(cfg: DictConfig) -> None: + console.print("\n[bold green]🤖 机器学习训练配置示例[/bold green]\n") + + tree = Tree("🎯 Training Configuration") + + model_tree = tree.add("[yellow]Model") + model_tree.add(f"Name: {cfg.model.name}") + model_tree.add(f"Layers: {cfg.model.layers}") + + optimizer_tree = tree.add("[cyan]Optimizer") + optimizer_tree.add(f"Name: {cfg.optimizer.name}") + optimizer_tree.add(f"Learning Rate: {cfg.optimizer.lr}") + + dataset_tree = tree.add("[magenta]Dataset") + dataset_tree.add(f"Name: {cfg.dataset.name}") + dataset_tree.add(f"Classes: {cfg.dataset.num_classes}") + + console.print(tree) + console.print("\n[bold green]✅ 配置加载完成！[/bold green]\n") + +if __name__ == "__main__": + train() diff --git a/examples/04_multi_env/app.py b/examples/04_multi_env/app.py new file mode 100644 index 0000000..05d5c65 --- /dev/null +++ b/examples/04_multi_env/app.py @@ -0,0 +1,30 @@ +"""多环境配置管理示例""" + +import hydra +from omegaconf import DictConfig +from rich.console import Console +from rich.table import Table + +console = Console() + +@hydra.main(version_base=None, config_path="conf", config_name="config") +def run_app(cfg: DictConfig) -> None: + env_colors = {"dev": "yellow", "test": "cyan", "prod": "red"} + color = env_colors.get(cfg.env.name, "white") + + console.print(f"\n[bold {color}]🌍 {cfg.env.name.upper()} 环境配置[/bold {color}]\n") + + table = Table() + table.add_column("配置项", style="cyan") + table.add_column("值", style="yellow") + + table.add_row("环境名称", cfg.env.name) + table.add_row("调试模式", "✅" if cfg.env.debug else "❌") + table.add_row("数据库主机", cfg.database.host) + table.add_row("服务器端口", str(cfg.server.port)) + + console.print(table) + console.print() + +if __name__ == "__main__": + run_app() diff --git a/examples/04_multi_env/conf/config.yaml b/examples/04_multi_env/conf/config.yaml new file mode 100644 index 0000000..f7dbf01 --- /dev/null +++ b/examples/04_multi_env/conf/config.yaml @@ -0,0 +1,11 @@ +defaults: + - env: dev + - _self_ + +database: + host: localhost + port: 5432 + +server: + host: 0.0.0.0 + port: 8000 diff --git a/examples/04_multi_env/conf/env/dev.yaml b/examples/04_multi_env/conf/env/dev.yaml new file mode 100644 index 0000000..87a33ee --- /dev/null +++ b/examples/04_multi_env/conf/env/dev.yaml @@ -0,0 +1,9 @@ +name: dev +debug: true + +database: + host: localhost + port: 5432 + +server: + port: 8000 diff --git a/examples/04_multi_env/conf/env/prod.yaml b/examples/04_multi_env/conf/env/prod.yaml new file mode 100644 index 0000000..2907284 --- /dev/null +++ b/examples/04_multi_env/conf/env/prod.yaml @@ -0,0 +1,9 @@ +name: prod +debug: false + +database: + host: prod-db.example.com + port: 5432 + +server: + port: 8000 diff --git a/examples/04_multi_env/conf/env/test.yaml b/examples/04_multi_env/conf/env/test.yaml new file mode 100644 index 0000000..0b9c23e --- /dev/null +++ b/examples/04_multi_env/conf/env/test.yaml @@ -0,0 +1,9 @@ +name: test +debug: false + +database: + host: test-db.example.com + port: 5432 + +server: + port: 8080 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c3acd3a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "hydra-ml-config-template" +version = "1.0.0" +description = "Python Hydra配置管理最佳实践模板，专为机器学习项目设计" +authors = [ + {name = "Your Name", email = "your.email@example.com"} +] +readme = "README.md" +requires-python = ">=3.8" +license = {text = "MIT"} + +dependencies = [ + "hydra-core>=1.3.0", + "omegaconf>=2.3.0", + "pyyaml>=6.0", + "rich>=13.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/quickstart.sh b/quickstart.sh new file mode 100755 index 0000000..200a19a --- /dev/null +++ b/quickstart.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -e + +echo "🚀 Hydra ML Config Template - 快速开始" +echo "========================================" + +# 安装 uv（如果需要） +if ! command -v uv &> /dev/null; then + echo "📦 安装 uv..." + curl -LsSf https://astral.sh/uv/install.sh | sh +fi + +# 创建虚拟环境 +echo "🔧 创建虚拟环境..." +uv venv + +# 激活虚拟环境并安装依赖 +echo "📥 安装依赖..." +source .venv/bin/activate +uv pip install -e . + +# 运行示例 +echo "🎯 运行示例..." +echo "" +echo "1️⃣ 基础示例" +cd examples/01_basic && python train.py && cd ../.. + +echo "" +echo "2️⃣ 机器学习训练示例" +cd examples/02_ml_training && python train.py && cd ../.. + +echo "" +echo "✨ 完成！查看 README.md 了解更多" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4083f59 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +hydra-core>=1.3.0 +omegaconf>=2.3.0 +pyyaml>=6.0 +rich>=13.0.0