add woodpecker

2025-10-03 12:24:01 +08:00
parent 3452db86ee
commit 6491999aae
4 changed files with 329 additions and 0 deletions
--- a/woodpecker/.env
+++ b/woodpecker/.env
@@ -0,0 +1,32 @@
+# === Woodpecker 公开地址（必须带协议）===
+WOODPECKER_HOST=https://ci.jmsu.top
+WOODPECKER_HOSTNAME=ci.jmsu.top
+# === Gitea 服务器地址（必须带协议）===
+# 例如你的 Gitea 是 https://git.jmsu.top 或 https://gitea.jmsu.top
+WOODPECKER_GITEA_URL=https://gitea.jmsu.top
+
+# === gRPC 外部域名（给跨机 agent 走 Traefik TCP，用来 HostSNI）===
+WOODPECKER_GRPC_HOST=ci-agent.jmsu.top
+
+# === 其余你已给出的保持不变（示例）===
+WOODPECKER_AGENT_SECRET=3ad4d1a5fc1876bf126bafbcbd0c5b75afa944f299cdbb9e690e27db74766252
+WOODPECKER_GITEA_CLIENT=d3a2f6a1-2e99-497f-a105-743eed57b36a
+WOODPECKER_GITEA_SECRET=gto_j5kndqtf5bof7a36mr2wiemdkjogl6yplucm2jbr4wklwraznzta
+
+#https://ci.jmsu.top/authorize
+#https://100.64.0.27:8420/authorize
+
+
+LOCAL_TS_IP=100.64.0.27
+CONSUL_SERVER_IP=100.64.0.1
+CONSUL_DC=dc1
+TRAEFIK_HTTP_ENTRYPOINT=websecure
+TRAEFIK_TCP_ENTRYPOINT=tcp
+CHECK_TYPE=http
+CHECK_PATH=/
+CHECK_INTERVAL=10s
+CHECK_TIMEOUT=2s
+DEREG_AFTER=1m
+
+# （可选）管理员账号（逗号分隔）
+WOODPECKER_ADMIN=lingyuzeng
--- a/woodpecker/README.md
+++ b/woodpecker/README.md
@@ -0,0 +1,109 @@
+3) 外部 Agent 使用示例
+
+在其他机器（非同一 docker 网络）的 agent：
+
+```bash
+export WOODPECKER_SERVER=ci-agent.jmsu.top:443   # Traefik TCP 对外端口
+export WOODPECKER_AGENT_SECRET=3ad4d1a5fc1876bf126bafbcbd0c5b75afa944f299cdbb9e690e27db74766252
+docker run --rm -e WOODPECKER_SERVER -e WOODPECKER_AGENT_SECRET \
+  -v /var/run/docker.sock:/var/run/docker.sock woodpeckerci/woodpecker-agent:latest
+```
+
+## 测试解析日志
+
+```bash
+curl -s http://100.64.0.1:8500/v1/agent/services \
+  | jq '.["woodpecker-web-100.64.0.27-8420"]'
+```
+
+## woodpecker 的其他镜像
+
+1. woodpeckerci/plugin-gitea-release
+
+用途：在 Gitea 上发布 Release。
+
+典型场景：
+
+当你在流水线里构建好二进制或打包好的产物后，可以用这个插件直接把产物上传到 Gitea 的 release 页面。
+
+类似 GitHub Actions 里的 gh release create。
+
+关键参数（pipeline yaml 里用的时候要传 env）：
+
+api_key: Gitea 的个人访问令牌
+
+files: 需要上传的文件路径
+
+base_url: Gitea 实例的 URL
+
+title / note: Release 标题、描述
+
+2. woodpeckerci/woodpecker-cli
+
+用途：Woodpecker 的命令行客户端。
+
+典型场景：
+
+在 CI/CD 环境或本地 shell 中调用 Woodpecker API，触发/查询流水线。
+
+类似 gh（GitHub CLI）、glab（GitLab CLI）。
+
+功能示例：
+
+woodpecker-cli info → 查看服务器信息
+
+woodpecker-cli build start <repo> → 触发构建
+
+woodpecker-cli build logs <build_id> → 查看日志
+
+3. woodpeckerci/plugin-s3
+
+用途：将构建产物上传到 S3 存储（或兼容 S3 的对象存储，例如 MinIO、Ceph RGW、阿里云 OSS、腾讯云 COS）。
+
+典型场景：
+
+构建产物（模型文件、Docker 镜像 tar 包、静态网站文件）上传到对象存储，方便下载或后续部署。
+
+关键参数：
+
+bucket：目标存储桶
+
+access_key / secret_key：认证凭据
+
+endpoint：对象存储的 API 地址
+
+source：要上传的文件路径
+
+4. woodpeckerci/plugin-git
+
+用途：在流水线里进行 Git 操作（checkout、clone、push）。
+
+典型场景：
+
+默认情况下，Woodpecker agent 会自动 clone 对应的仓库，但如果你需要 额外操作 Git，比如 push 生成的文件回仓库、同步到另一个 repo，就会用到这个插件。
+
+常见用法：
+
+自动更新子模块
+
+构建完成后，把生成的文档推送到 gh-pages / docs 分支
+
+将版本号 tag 回写到仓库
+
+| 镜像                     | 主要功能                        | 常见用途                         |
+| ---------------------- | --------------------------- | ---------------------------- |
+| `plugin-gitea-release` | 在 Gitea 上创建 Release 并上传产物   | 发布二进制包 / 模型文件到 Gitea Release |
+| `woodpecker-cli`       | CLI 工具，管理 Woodpecker 服务器和构建 | 本地或 CI 脚本里触发/监控流水线           |
+| `plugin-s3`            | 上传产物到 S3 / 对象存储             | 存放模型、静态文件、备份                 |
+| `plugin-git`           | 执行 Git 操作（clone/push）       | 自动推送 tag、同步分支、更新文档           |
+
+
+结合你的场景（LLM 微调 + 自动部署）：
+
+plugin-s3：可以把训练好的模型权重、日志直接上传到 MinIO/OSS，方便分发。
+
+plugin-gitea-release：你可以在 Gitea release 里发一个“训练完成的模型包”。
+
+plugin-git：可以在训练完成后自动 push 版本号/配置文件回到仓库。
+
+woodpecker-cli：你本地调试流水线、或在另一台机器上触发/监控 build。
--- a/woodpecker/docker-compose.yml
+++ b/woodpecker/docker-compose.yml
@@ -0,0 +1,95 @@
+version: "3.8"
+
+services:
+  woodpecker-server:
+    image: woodpeckerci/woodpecker-server:v3.10.0
+    container_name: woodpecker-server
+    restart: unless-stopped
+    cpus: 0.5
+    mem_limit: 512m
+    networks:
+      - woodpecker
+    environment:
+      - WOODPECKER_OPEN=true
+      - WOODPECKER_HOST=${WOODPECKER_HOST}
+      - WOODPECKER_AGENT_SECRET=${WOODPECKER_AGENT_SECRET}
+      - WOODPECKER_ADMIN=${WOODPECKER_ADMIN}
+      - WOODPECKER_GITEA=true
+      - WOODPECKER_GITEA_URL=${WOODPECKER_GITEA_URL}
+      - WOODPECKER_GITEA_CLIENT=${WOODPECKER_GITEA_CLIENT}
+      - WOODPECKER_GITEA_SECRET=${WOODPECKER_GITEA_SECRET}
+      - WOODPECKER_GITEA_SKIP_VERIFY=true
+    # 只把 gRPC(容器 9000) 绑定到本机 Tailscale IP 的 8419
+    ports:
+      - "${LOCAL_TS_IP}:8419:9000"
+      - "${LOCAL_TS_IP}:8420:8000"
+    volumes:
+      - "./data:/var/lib/woodpecker"
+
+  woodpecker-agent:
+    container_name: woodpecker-agent
+    image: woodpeckerci/woodpecker-agent:v3.10.0
+    restart: unless-stopped
+    # cpus: 0.5
+    # mem_limit: 1024m
+    depends_on:
+      - woodpecker-server
+    networks:
+      - woodpecker
+    environment:
+      # 内网 agent 仍然走容器网络直连 server:9000
+      - "WOODPECKER_SERVER=woodpecker-server:9000"
+      - "WOODPECKER_AGENT_SECRET=${WOODPECKER_AGENT_SECRET}"
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock"
+
+  # === gRPC TCP 注册：HostSNI(`${WOODPECKER_GRPC_HOST}`) -> tcp -> LOCAL_TS_IP:8419 ===
+  woodpecker-grpc-registrar:
+    image: hashicorp/consul:1.21
+    container_name: woodpecker-grpc-registrar
+    restart: unless-stopped
+    networks:
+      - woodpecker
+    environment:
+      - CONSUL_HTTP_ADDR=http://${CONSUL_SERVER_IP}:8500
+      - SERVICE_NAME=woodpecker-grpc
+      - SERVICE_ADDR=${LOCAL_TS_IP}
+      - SERVICE_PORT=8419          # 对外注册用 8419
+      - ROUTE_HOST=${WOODPECKER_GRPC_HOST}
+      - SERVICE_PROTOCOL=tcp
+      - CHECK_TYPE=tcp
+      - CHECK_INTERVAL=${CHECK_INTERVAL}
+      - CHECK_TIMEOUT=${CHECK_TIMEOUT}
+      - DEREG_AFTER=${DEREG_AFTER}
+      - TRAEFIK_TCP_ENTRYPOINT=${TRAEFIK_TCP_ENTRYPOINT}
+    volumes:
+      - ./registrar.sh:/registrar.sh:ro
+    entrypoint: ["/bin/sh","/registrar.sh"]
+
+  # === 可选：Web(HTTP) 注册（默认注释掉；若需要对外暴露 Web，再开启） ===
+  woodpecker-web-registrar:
+    image: hashicorp/consul:1.21
+    container_name: woodpecker-web-registrar
+    restart: unless-stopped
+    networks:
+      - woodpecker
+    environment:
+      - CONSUL_HTTP_ADDR=http://${CONSUL_SERVER_IP}:8500
+      - SERVICE_NAME=woodpecker-web
+      - SERVICE_ADDR=${LOCAL_TS_IP}
+      - SERVICE_PORT=8420        # 若要暴露 Web，请同时在 woodpecker-server 里把 8420:8000 也映射
+      - ROUTE_HOST=${WOODPECKER_HOSTNAME}
+      - SERVICE_PROTOCOL=http
+      - CHECK_TYPE=http
+      - CHECK_PATH=${CHECK_PATH}
+      - CHECK_INTERVAL=${CHECK_INTERVAL}
+      - CHECK_TIMEOUT=${CHECK_TIMEOUT}
+      - DEREG_AFTER=${DEREG_AFTER}
+      - TRAEFIK_HTTP_ENTRYPOINT=${TRAEFIK_HTTP_ENTRYPOINT}
+    volumes:
+      - ./registrar.sh:/registrar.sh:ro
+    entrypoint: ["/bin/sh","/registrar.sh"]
+
+networks:
+  woodpecker:
+    driver: bridge
--- a/woodpecker/registrar.sh
+++ b/woodpecker/registrar.sh
@@ -0,0 +1,93 @@
+#!/bin/sh
+set -eu
+
+: "${SERVICE_NAME:?need SERVICE_NAME}"
+: "${SERVICE_ADDR:?need SERVICE_ADDR}"
+: "${SERVICE_PORT:?need SERVICE_PORT}"
+: "${ROUTE_HOST:?need ROUTE_HOST}"
+
+CONSUL="${CONSUL_HTTP_ADDR:?need CONSUL_HTTP_ADDR}"
+SERVICE_PROTOCOL="${SERVICE_PROTOCOL:-http}"          # http | tcp
+CHECK_TYPE="${CHECK_TYPE:-tcp}"                       # http | tcp
+CHECK_PATH="${CHECK_PATH:-/}"
+CHECK_INTERVAL="${CHECK_INTERVAL:-10s}"
+CHECK_TIMEOUT="${CHECK_TIMEOUT:-2s}"
+DEREG_AFTER="${DEREG_AFTER:-1m}"
+TRAEFIK_HTTP_ENTRYPOINT="${TRAEFIK_HTTP_ENTRYPOINT:-websecure}"
+TRAEFIK_TCP_ENTRYPOINT="${TRAEFIK_TCP_ENTRYPOINT:-tcp}"
+# TRAEFIK_CERT_RESOLVER="${TRAEFIK_CERT_RESOLVER:-cf}"
+
+echo "[registrar] consul: $CONSUL, service: $SERVICE_NAME@$SERVICE_ADDR:$SERVICE_PORT"
+
+# 等云端 Consul Server 可用
+for i in $(seq 1 90); do
+  if wget -qO- "$CONSUL/v1/status/leader" >/dev/null 2>&1; then
+    break
+  fi
+  sleep 1
+done
+
+ID="${SERVICE_NAME}-${SERVICE_ADDR}-${SERVICE_PORT}"
+
+# 组装 Traefik tags（按“行”累加，避免值中逗号被拆）
+NL='
+'
+TAGS="traefik.enable=true"
+
+if [ "$SERVICE_PROTOCOL" = "http" ]; then
+  TAGS="$TAGS${NL}traefik.http.routers.${SERVICE_NAME}.rule=Host(\`${ROUTE_HOST}\`)"
+  TAGS="$TAGS${NL}traefik.http.routers.${SERVICE_NAME}.entrypoints=${TRAEFIK_HTTP_ENTRYPOINT}"
+  TAGS="$TAGS${NL}traefik.http.routers.${SERVICE_NAME}.tls=true"
+  TAGS="$TAGS${NL}traefik.http.services.${SERVICE_NAME}.loadbalancer.server.scheme=http"
+  TAGS="$TAGS${NL}traefik.http.services.${SERVICE_NAME}.loadbalancer.server.port=${SERVICE_PORT}"
+  # 抢占路由：给当前 Host 的 router 设置更高优先级
+  TAGS="$TAGS${NL}traefik.http.routers.${SERVICE_NAME}.priority=10000"
+  # 可选中间件（注意：值里有逗号也安全）
+  TAGS="$TAGS${NL}traefik.http.routers.${SERVICE_NAME}.middlewares=gzip-all@file,sec-headers@file"
+  # 如需 ACME 证书解析器可再加一行（取消注释）
+  # TAGS="$TAGS${NL}traefik.http.routers.${SERVICE_NAME}.tls.certresolver=${TRAEFIK_CERT_RESOLVER}"
+elif [ "$SERVICE_PROTOCOL" = "tcp" ]; then
+  TAGS="$TAGS${NL}traefik.tcp.routers.${SERVICE_NAME}.rule=HostSNI(\`${ROUTE_HOST}\`)"
+  TAGS="$TAGS${NL}traefik.tcp.routers.${SERVICE_NAME}.entrypoints=${TRAEFIK_TCP_ENTRYPOINT}"
+  TAGS="$TAGS${NL}traefik.tcp.services.${SERVICE_NAME}.loadbalancer.server.port=${SERVICE_PORT}"
+else
+  echo "unsupported SERVICE_PROTOCOL=$SERVICE_PROTOCOL" >&2; exit 2
+fi
+
+# 转 JSON 数组（按“行”解析）
+to_json_array() {
+  # 逐行 -> trim -> "..." -> [ ... ]
+  awk 'BEGIN{RS="\n"} NF {gsub(/^[ \t]+|[ \t]+$/,""); printf "\"%s\",\n",$0}' |
+  sed '1s/^/[/' | sed '$s/,\s*$/]/'
+}
+TAGS_JSON="$(printf "%s" "$TAGS" | to_json_array)"
+
+# 健康检查 JSON
+if [ "$CHECK_TYPE" = "http" ]; then
+  CHECK_JSON=$(cat <<EOF
+{"Name":"http","HTTP":"http://${SERVICE_ADDR}:${SERVICE_PORT}${CHECK_PATH}","Interval":"${CHECK_INTERVAL}","Timeout":"${CHECK_TIMEOUT}","DeregisterCriticalServiceAfter":"${DEREG_AFTER}"}
+EOF
+)
+else
+  CHECK_JSON=$(cat <<EOF
+{"Name":"tcp","TCP":"${SERVICE_ADDR}:${SERVICE_PORT}","Interval":"${CHECK_INTERVAL}","Timeout":"${CHECK_TIMEOUT}","DeregisterCriticalServiceAfter":"${DEREG_AFTER}"}
+EOF
+)
+fi
+
+# 写 service 定义并注册到云端 Consul
+cat > /tmp/svc.json <<EOF
+{"service":{"id":"${ID}","name":"${SERVICE_NAME}","address":"${SERVICE_ADDR}","port":${SERVICE_PORT},"tags":${TAGS_JSON},"checks":[${CHECK_JSON}]}}
+EOF
+
+echo "[registrar] register ${ID} -> ${CONSUL}"
+consul services register -http-addr="$CONSUL" /tmp/svc.json
+
+term() {
+  echo "[registrar] deregister ${ID}"
+  consul services deregister -http-addr="$CONSUL" /tmp/svc.json || true
+  exit 0
+}
+trap term TERM INT
+
+tail -f /dev/null