first add

2024-06-12 14:56:55 +08:00
parent 86d1eaab86
commit cf1bc7b655
14 changed files with 436 additions and 0 deletions
--- a/Dockerfile.3bfinetune
+++ b/Dockerfile.3bfinetune
@@ -0,0 +1,45 @@
+# syntax=docker/dockerfile:1
+FROM nvidia/cuda:11.7.1-devel-ubuntu22.04
+
+ARG DEBIAN_FRONTEND="noninteractive"
+ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND}
+
+COPY src/requirements.txt /root/
+ENV MAMBA_ROOT_PREFIX=~/micromamba
+WORKDIR /root
+SHELL ["/bin/bash", "-c"]
+
+RUN <<EOT
+#!/bin/bash
+# 更新软件源
+apt-get update
+apt-get install -y wget curl git jq vim bash libaio-dev build-essential openssh-server
+# Configure SSH for password and public key authentication
+echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
+echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config
+echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config
+echo "Port 22" >> /etc/ssh/sshd_config
+mkdir /var/run/sshd
+echo 'root:C2024D513C' | chpasswd
+# Install Micromamba
+echo 1 | bash <(curl -s https://cdn.jsdelivr.net/gh/hotwa/MicroMamba_Installer@main/install.sh)
+micromamba shell init -s bash -p ~/micromamba
+micromamba create -n finetune -c conda-forge python=3.10 cudatoolkit=11.7 -y
+# Configure Python and install packages
+mkdir -p ~/.pip
+echo "
+[global]
+index-url = https://mirrors.aliyun.com/pypi/simple/
+
+[install]
+trusted-host=mirrors.aliyun.com
+" >> ~/.pip/pip.conf
+micromamba run -n finetune pip install protobuf ipython torch==1.13.1 transformers deepspeed tqdm requests peft matplotlib sentencepiece torchvision torchaudio ninja triton
+echo "micromamba activate finetune" >> ~/.bashrc
+EOT
+
+# Expose SSH port
+EXPOSE 3222
+
+# Keep the container running
+CMD ["/usr/sbin/sshd", "-D"]
--- a/Dockerfile.bgpt
+++ b/Dockerfile.bgpt
@@ -0,0 +1,47 @@
+# syntax=docker/dockerfile:1
+FROM nvidia/cuda:11.6.1-devel-ubuntu20.04
+
+ARG DEBIAN_FRONTEND="noninteractive"
+ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND}
+
+COPY src/requirements.txt /root/
+ENV MAMBA_ROOT_PREFIX=~/micromamba
+WORKDIR /root
+SHELL ["/bin/bash", "-c"]
+
+COPY requirements.txt /root/
+COPY id_rsa.pub /root/.ssh/
+RUN <<EOT
+#!/bin/bash
+apt-get update
+apt-get install -y wget curl git git-lfs jq vim bash libaio-dev build-essential openssh-server lsof
+echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
+echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config
+echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config
+echo "Port 22" >> /etc/ssh/sshd_config
+mkdir /var/run/sshd
+echo 'root:cdcdocker' | chpasswd
+# Install Micromamba
+echo 1 | bash <(curl -s https://cdn.jsdelivr.net/gh/hotwa/MicroMamba_Installer@main/install.sh)
+micromamba shell init -s bash -p ~/micromamba
+mkdir -p ~/.pip
+echo "
+[global]
+index-url = https://mirrors.aliyun.com/pypi/simple/
+
+[install]
+trusted-host=mirrors.aliyun.com
+" >> ~/.pip/pip.conf
+micromamba create -n bgpt -c conda-forge python=3.7.9 -y
+micromamba run -n bgpt pip install -r requirements.txt
+micromamba run -n bgpt pip install ipykernel attrs seaborn
+micromamba run -n bgpt python -m ipykernel install --user --name="bgpt" --display-name="bgpt_env"
+micromamba run -n bgpt pip install seaborn attrs torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116
+echo "micromamba activate bgpt" >> ~/.bashrc
+EOT
+
+# Expose SSH port
+EXPOSE 3222
+
+# Keep the container running
+CMD ["/usr/sbin/sshd", "-D"]
--- a/Dockerfile.codeserver
+++ b/Dockerfile.codeserver
@@ -0,0 +1,73 @@
+# 使用 code-server 的基础镜像
+FROM codercom/code-server:latest
+
+# 安装依赖和工具
+USER root
+RUN apt-get update && apt-get install -y curl
+
+# 安装 VSCode 插件
+USER coder
+RUN code-server --install-extension aaron-bond.better-comments@3.0.2 \
+    && code-server --install-extension aminer.codegeex@2.8.0 \
+    && code-server --install-extension davidanson.vscode-markdownlint@0.55.0 \
+    && code-server --install-extension dotjoshjohnson.xml@2.5.1 \
+    && code-server --install-extension dustypomerleau.rust-syntax@0.6.1 \
+    && code-server --install-extension eamodio.gitlens@14.9.1 \
+    && code-server --install-extension emmanuelbeziat.vscode-great-icons@2.1.105 \
+    && code-server --install-extension esbenp.prettier-vscode@10.4.0 \
+    && code-server --install-extension fittentech.fitten-code@0.10.16 \
+    && code-server --install-extension formulahendry.auto-close-tag@0.5.15 \
+    && code-server --install-extension formulahendry.auto-complete-tag@0.1.0 \
+    && code-server --install-extension formulahendry.auto-rename-tag@0.1.10 \
+    && code-server --install-extension formulahendry.code-runner@0.12.2 \
+    && code-server --install-extension frhtylcn.pythonsnippets@1.0.2 \
+    && code-server --install-extension gera2ld.markmap-vscode@0.1.4 \
+    && code-server --install-extension github.copilot@1.190.0 \
+    && code-server --install-extension github.copilot-chat@0.15.1 \
+    && code-server --install-extension github.vscode-github-actions@0.26.2 \
+    && code-server --install-extension gobystrokreactjs.gobystrok@0.0.1 \
+    && code-server --install-extension hbenl.vscode-test-explorer@2.21.1 \
+    && code-server --install-extension laike9m.cyberbrain@0.2.1 \
+    && code-server --install-extension littlefoxteam.vscode-python-test-adapter@0.8.2 \
+    && code-server --install-extension mechatroner.rainbow-csv@3.11.0 \
+    && code-server --install-extension ms-azuretools.vscode-docker@1.29.1 \
+    && code-server --install-extension ms-ceintl.vscode-language-pack-zh-hans@1.89.2024050109 \
+    && code-server --install-extension ms-python.debugpy@2024.6.0 \
+    && code-server --install-extension ms-python.python@2024.6.0 \
+    && code-server --install-extension ms-python.vscode-pylance@2024.5.1 \
+    && code-server --install-extension ms-toolsai.jupyter@2024.4.0 \
+    && code-server --install-extension ms-toolsai.jupyter-hub@2023.10.1003012305 \
+    && code-server --install-extension ms-toolsai.jupyter-keymap@1.1.2 \
+    && code-server --install-extension ms-vscode-remote.remote-containers@0.362.0 \
+    && code-server --install-extension ms-vscode-remote.remote-ssh@0.110.1 \
+    && code-server --install-extension ms-vscode-remote.remote-ssh-edit@0.86.0 \
+    && code-server --install-extension ms-vscode-remote.remote-wsl@0.88.2 \
+    && code-server --install-extension ms-vscode-remote.vscode-remote-extensionpack@0.25.0 \
+    && code-server --install-extension ms-vscode.notepadplusplus-keybindings@1.0.7 \
+    && code-server --install-extension ms-vscode.powershell@2024.2.1 \
+    && code-server --install-extension ms-vscode.remote-explorer@0.4.3 \
+    && code-server --install-extension ms-vscode.remote-server@1.5.1 \
+    && code-server --install-extension ms-vscode.test-adapter-converter@0.1.9 \
+    && code-server --install-extension njpwerner.autodocstring@0.6.1 \
+    && code-server --install-extension njqdev.vscode-python-typehint@1.5.1 \
+    && code-server --install-extension octref.vetur@0.37.3 \
+    && code-server --install-extension oderwat.indent-rainbow@8.3.1 \
+    && code-server --install-extension prompt-flow.prompt-flow@1.18.0 \
+    && code-server --install-extension ritwickdey.liveserver@5.7.9 \
+    && code-server --install-extension rust-lang.rust-analyzer@0.3.1958 \
+    && code-server --install-extension shiro.pythonpack@0.3.0 \
+    && code-server --install-extension tomoki1207.pdf@1.2.2 \
+    && code-server --install-extension vscode-icons-team.vscode-icons@12.7.0 \
+    && code-server --install-extension wayou.vscode-todo-highlight@1.0.5 \
+    && code-server --install-extension wix.vscode-import-cost@3.3.0 \
+    && code-server --install-extension xirider.livecode@1.3.10 \
+    && code-server --install-extension yzhang.markdown-all-in-one@3.6.2
+
+# 设置工作目录
+WORKDIR /home/coder/project
+
+# 暴露端口
+EXPOSE 8080
+
+# 启动 code-server
+CMD ["code-server", "--bind-addr", "0.0.0.0:8080", "--auth", "none"]
--- a/Dockerfile.ldh
+++ b/Dockerfile.ldh
@@ -0,0 +1,29 @@
+# syntax=docker/dockerfile:1
+FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04
+
+ENV MAMBA_ROOT_PREFIX=~/micromamba
+ENV PATH="/usr/local/bin:$PATH"
+RUN <<EOT
+#!/bin/bash
+apt-get update
+apt-get install -y wget curl git git-lfs jq vim bash libaio-dev build-essential openssh-server lsof
+echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
+echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config
+echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config
+echo "Port 22" >> /etc/ssh/sshd_config
+mkdir /var/run/sshd
+echo 'root:cdcdocker' | chpasswd
+# Install Micromamba
+echo 1 | bash <(curl -s https://cdn.jsdelivr.net/gh/hotwa/MicroMamba_Installer@main/install.sh)
+micromamba shell init -s bash -p ~/micromamba
+mkdir -p ~/.pip
+echo "
+[global]
+index-url = https://mirrors.aliyun.com/pypi/simple/
+
+[install]
+trusted-host=mirrors.aliyun.com
+" >> ~/.pip/pip.conf
+EOT
+
+CMD ["/usr/sbin/sshd", "-D"]
--- a/Dockerfile.zj
+++ b/Dockerfile.zj
@@ -0,0 +1,29 @@
+# syntax=docker/dockerfile:1
+FROM sophgo/tpuc_dev:v3.1
+
+ENV MAMBA_ROOT_PREFIX=~/micromamba
+ENV PATH="/usr/local/bin:$PATH"
+RUN <<EOT
+#!/bin/bash
+apt-get update
+apt-get install -y wget curl git git-lfs jq vim bash libaio-dev build-essential openssh-server lsof
+echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
+echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config
+echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config
+echo "Port 22" >> /etc/ssh/sshd_config
+mkdir /var/run/sshd
+echo 'root:cdcdocker' | chpasswd
+# Install Micromamba
+echo 1 | bash <(curl -s https://cdn.jsdelivr.net/gh/hotwa/MicroMamba_Installer@main/install.sh)
+micromamba shell init -s bash -p ~/micromamba
+mkdir -p ~/.pip
+echo "
+[global]
+index-url = https://mirrors.aliyun.com/pypi/simple/
+
+[install]
+trusted-host=mirrors.aliyun.com
+" >> ~/.pip/pip.conf
+EOT
+
+CMD ["/usr/sbin/sshd", "-D"]
--- a/README_zj.md
+++ b/README_zj.md
@@ -0,0 +1,2 @@
+docker run --privileged --name myname -v $PWD:/workspace -it sophgo/tpuc_dev:v3.1
+/data1/liujiang/workspace/sophone   这个目录映射
--- a/docker-compose-3bfinetune.yml
+++ b/docker-compose-3bfinetune.yml
@@ -0,0 +1,34 @@
+version: '3.8'
+
+services:
+  ubuntu:
+    build: 
+      context: .
+      dockerfile: Dockerfile.3bfinetune
+    volumes:
+      - ./src:/root/data
+      - /mnt/sdb/zly/datas/Datas:/mnt/sdb/zly/datas
+      - /data:/data
+    container_name: ubuntu
+    pull_policy: if_not_present
+    tty: true
+    restart: unless-stopped
+    image: zly/cuda:latest
+    ports:
+      - 8888:8888
+      - 3222:22
+    environment:
+      - OLLAMA_ORIGINS="chrome-extension://*"
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 8
+              capabilities: [gpu]
+    networks:
+      - network2
+
+networks:
+  network2:
+    name: network2
--- a/docker-compose-bgpt.yml
+++ b/docker-compose-bgpt.yml
@@ -0,0 +1,54 @@
+version: '3.8'
+
+services:
+  ubuntu-ssh:
+    build: 
+      context: .
+      dockerfile: Dockerfile.bgpt
+    volumes:
+      - ./src:/root/data
+      - /mnt/sdb/zly/datas/Datas:/mnt/sdb/zly/datas
+      - /data:/data
+    container_name: ubuntu-ssh
+    pull_policy: if_not_present
+    tty: true
+    restart: unless-stopped
+    image: zly/cuda-bgpt:latest
+    ports:
+      - 3222:22
+    environment:
+      - NVIDIA_VISIBLE_DEVICES=all
+      - NVIDIA_DRIVER_CAPABILITIES=compute,utility
+      - OLLAMA_ORIGINS="chrome-extension://*"
+    networks:
+      - network3
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+
+  # openai-whisper-asr-webservice:
+  #   ports:
+  #       - '9000:9000'
+  #   volumes:
+  #         - './whisper-large-v3:/data/whisper'
+  #   environment:
+  #       - ASR_MODEL=large
+  #       - ASR_ENGINE=openai_whisper
+  #       - ASR_MODEL_PATH=/data/whisper
+  #   image: 'onerahmet/openai-whisper-asr-webservice:latest-gpu'
+  #   deploy:
+  #     resources:
+  #       reservations:
+  #         devices:
+  #           - driver: nvidia
+  #             count: 1
+  #             capabilities: [gpu]
+
+
+networks:
+  network3:
+    name: network3
--- a/docker-compose-chipanpan.yml
+++ b/docker-compose-chipanpan.yml
@@ -0,0 +1,34 @@
+version: '3.8'
+
+services:
+  ubuntu-cpp:
+    build: 
+      context: .
+      dockerfile: Dockerfile.bgpt
+    volumes:
+      - /data:/data/
+      - /home/chipan:/home/chipan
+    container_name: ubuntu-cpp
+    pull_policy: if_not_present
+    tty: true
+    restart: unless-stopped
+    image: zly/cuda-bgpt:latest
+    ports:
+      - 3111:22
+    environment:
+      - NVIDIA_VISIBLE_DEVICES=all
+      - NVIDIA_DRIVER_CAPABILITIES=compute,utility
+      - OLLAMA_ORIGINS="chrome-extension://*"
+    networks:
+      - network4
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+
+networks:
+  network4:
+    name: network4
--- a/docker-compose-zj.yml
+++ b/docker-compose-zj.yml
@@ -0,0 +1,32 @@
+version: '3.8'
+
+services:
+  ubuntu-zj:
+    build: 
+      context: .
+      dockerfile: Dockerfile.zj
+    volumes:
+      - /data1/liujiang/workspace/sophone:/data1/liujiang/workspace/sophone
+    container_name: ubuntu-zj
+    pull_policy: if_not_present
+    tty: true
+    restart: unless-stopped
+    image: zj/tpuc_dev:latest
+    ports:
+      - 3393:22
+    environment:
+      - NVIDIA_VISIBLE_DEVICES=all
+      - NVIDIA_DRIVER_CAPABILITIES=compute,utility
+    networks:
+      - network5
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+
+networks:
+  network5:
+    name: network5
--- a/docker_compose_script.sh
+++ b/docker_compose_script.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Docker Compose 安装路径
+DOCKER_COMPOSE_PATH="/usr/local/bin/docker-compose"
+
+install_docker() {
+    echo "正在检查 Docker 是否已安装..."
+    if ! [ -x "$(command -v docker)" ]; then
+        echo "Docker 未安装。正在安装 Docker..."
+        curl -fsSL https://get.docker.com -o get-docker.sh
+        sh get-docker.sh
+        echo "Docker 已安装。"
+    else
+        echo "Docker 已安装。"
+    fi
+}
+
+install_docker_compose() {
+    install_docker
+    echo "正在安装 Docker Compose..."
+
+    # 使用 GitHub API 获取最新版本
+    LATEST_RELEASE=$(curl -s https://api.github.com/repos/docker/compose/releases/latest | jq -r .tag_name)
+    if [ -z "$LATEST_RELEASE" ]; then
+        echo "无法获取 Docker Compose 的最新版本。"
+        exit 1
+    fi
+
+    # 更新 Docker Compose 安装 URL 以使用最新版本
+    DOCKER_COMPOSE_URL="https://ghproxy.dockless.eu.org/https://github.com/docker/compose/releases/download/${LATEST_RELEASE}/docker-compose-$(uname -s)-$(uname -m)"
+
+    sudo curl -L $DOCKER_COMPOSE_URL -o $DOCKER_COMPOSE_PATH
+    sudo chmod +x $DOCKER_COMPOSE_PATH
+    echo "Docker Compose ${LATEST_RELEASE} 已安装。"
+}
+
+uninstall_docker_compose() {
+    echo "正在卸载 Docker Compose..."
+    rm -f $DOCKER_COMPOSE_PATH
+    echo "Docker Compose 已卸载。"
+}
+
+echo "选择操作："
+echo "1) 安装 Docker Compose"
+echo "2) 卸载 Docker Compose"
+echo "3) 退出"
+read -p "输入选择（1-3）：" choice
+
+case $choice in
+    1) install_docker_compose ;;
+    2) uninstall_docker_compose ;;
+    3) exit 0 ;;
+    *) echo "无效输入。" ;;
+esac
+
--- a/id_rsa.pub
+++ b/id_rsa.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC8orKfkGpNyhi85v/lXQm4tjpxU5Br6aT+4jTPHNil1isZNjd1tMgPJmqFNcib3XOK/L/SVthdnBqyu/7YFfw+K9WGLpgmPkRY2SLgs7l8ck9145zn/xYhjVa3naegYof0MWNyG1vSOcYXw6jAm5iNiGp8qYCKRpwcmCG0SD8GFn+Dc7ZneRGKM1ztDPdHhJAiEBZ0h+GAVQAGOdJ28HEnavNZAymfYRn0/8QjX5cOHqxeOJ8gcv17XQI56M2K+MYR50FAXiuizQy8qxD2/uOPv0aY5De3Dgfwjbcxyi4ufG4V9Zkvf1GZzkjUhwpQbBeKRw+sKnRza1GfsOxbQN4KO63/3FZelOja0UHGy/aDOJ9224Drt0U+xELSLCjKrax0jnu/K/ASrveqi7v4by3YBW3ETBuOGS7glIU3fszz2kCVLqkHoaCSAxFu5Ox4Lss7rfjUGpvUUxHSFbFvZu7hC7W3PJEYKTh3NC9Q70UxTgcPEmnB7mP44NdHm4jk2AuldK9StdVQdqSWJtT5Uh6VVOrQyobueIHUxyyUTVKUw6XUsWEy+SFXnwEflffF1SEftaU7vziwyBT6IbMZ1dXJ14MSW76Trkc+HWQQkdnXpG4HXTiptXsyJBrG9cTjPKWwKEGetbs10PPLCj9CTvNO7azz6dWrQLCuUlxgWD4LsQ== pylyzeng@gmail.com
--- a/1
+++ b/1
@@ -0,0 +1 @@
+C2024D513C
--- a/requirements.txt
+++ b/requirements.txt
				`@@ -0,0 +1 @@`
				ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC8orKfkGpNyhi85v/lXQm4tjpxU5Br6aT+4jTPHNil1isZNjd1tMgPJmqFNcib3XOK/L/SVthdnBqyu/7YFfw+K9WGLpgmPkRY2SLgs7l8ck9145zn/xYhjVa3naegYof0MWNyG1vSOcYXw6jAm5iNiGp8qYCKRpwcmCG0SD8GFn+Dc7ZneRGKM1ztDPdHhJAiEBZ0h+GAVQAGOdJ28HEnavNZAymfYRn0/8QjX5cOHqxeOJ8gcv17XQI56M2K+MYR50FAXiuizQy8qxD2/uOPv0aY5De3Dgfwjbcxyi4ufG4V9Zkvf1GZzkjUhwpQbBeKRw+sKnRza1GfsOxbQN4KO63/3FZelOja0UHGy/aDOJ9224Drt0U+xELSLCjKrax0jnu/K/ASrveqi7v4by3YBW3ETBuOGS7glIU3fszz2kCVLqkHoaCSAxFu5Ox4Lss7rfjUGpvUUxHSFbFvZu7hC7W3PJEYKTh3NC9Q70UxTgcPEmnB7mP44NdHm4jk2AuldK9StdVQdqSWJtT5Uh6VVOrQyobueIHUxyyUTVKUw6XUsWEy+SFXnwEflffF1SEftaU7vziwyBT6IbMZ1dXJ14MSW76Trkc+HWQQkdnXpG4HXTiptXsyJBrG9cTjPKWwKEGetbs10PPLCj9CTvNO7azz6dWrQLCuUlxgWD4LsQ== pylyzeng@gmail.com