add step build file

2025-01-18 21:39:38 +08:00
parent 4766780404
commit 77e15d17e5
6 changed files with 473 additions and 0 deletions
--- a/Dockerfile.1.module
+++ b/Dockerfile.1.module
@@ -0,0 +1,58 @@
+# syntax=docker/dockerfile:1.3-labs
+# NOTE: Building this image requires Docker version >= 23.0.
+
+# Stage 1: Build environment
+ARG TAG_VERSION="12.4.1-cudnn-devel-ubuntu22.04"
+FROM nvidia/cuda:${TAG_VERSION}
+
+ENV CUDA_HOME=/usr/local/cuda
+ENV PATH=$CUDA_HOME/bin:$PATH
+ENV LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH
+ENV CPATH=$CUDA_HOME/include:$CPATH
+ENV LIBRARY_PATH=$CUDA_HOME/lib64:$LIBRARY_PATH
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ENV http_proxy=${HTTP_PROXY}
+ENV https_proxy=${HTTPS_PROXY}
+ARG DEBIAN_FRONTEND="noninteractive"
+ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND}
+ENV SSH_PORT=2222
+WORKDIR /root
+SHELL ["/bin/bash", "-c"]
+
+# Replace sources.list with Aliyun mirror
+RUN <<EOT
+#!/bin/bash
+cp /etc/apt/sources.list /etc/apt/sources.list_bak
+cat << EOF > /etc/apt/sources.list
+deb https://mirrors.aliyun.com/ubuntu/ jammy main restricted universe multiverse
+deb-src https://mirrors.aliyun.com/ubuntu/ jammy main restricted universe multiverse
+deb https://mirrors.aliyun.com/ubuntu/ jammy-security main restricted universe multiverse
+deb-src https://mirrors.aliyun.com/ubuntu/ jammy-security main restricted universe multiverse
+deb https://mirrors.aliyun.com/ubuntu/ jammy-updates main restricted universe multiverse
+deb-src https://mirrors.aliyun.com/ubuntu/ jammy-updates main restricted universe multiverse
+deb https://mirrors.aliyun.com/ubuntu/ jammy-backports main restricted universe multiverse
+deb-src https://mirrors.aliyun.com/ubuntu/ jammy-backports main restricted universe multiverse
+EOF
+apt-get update && apt-get install -y tzdata
+ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+echo "Asia/Shanghai" > /etc/timezone
+apt-get install -y tcl-dev \
+    environment-modules \
+    wget \
+    curl \
+    bzip2 \
+    tar \
+    less
+mkdir -p ~/.local/modules
+wget https://sourceforge.net/projects/modules/files/Modules/modules-5.5.0/modules-5.5.0.tar.gz
+# curl -LJO https://github.com/cea-hpc/modules/releases/download/v5.5.0/modules-5.5.0.tar.gz
+tar xfz modules-5.5.0.tar.gz
+cd modules-5.5.0
+./configure --prefix=$HOME/.local/modules --with-tcl-lib=/usr/local/tools/tcl/lib --with-tcl-inc=/usr/local/tools/tcl/include --modulefilesdir=$HOME/.local/modules/modulefiles 
+make -j$(nproc)
+make install
+echo "source $HOME/.local/modules/init/profile.sh" >> ~/.bashrc
+echo "source $HOME/.local/modules/init/profile.sh" >> /etc/profile
+rm -rf /root/modules*
+EOT
--- a/Dockerfile.2.micromamba
+++ b/Dockerfile.2.micromamba
@@ -0,0 +1,39 @@
+# syntax=docker/dockerfile:1.3-labs
+# NOTE: Building this image requires Docker version >= 23.0.
+
+# Stage 1: Build environment
+ARG TAG_VERSION
+ARG BASE
+FROM ${BASE}:${TAG_VERSION}
+
+# Install micromamba silently
+WORKDIR /root
+ENV BIN_FOLDER=/usr/local/bin
+ENV INIT_YES=yes
+ENV CONDA_FORGE_YES=yes
+ENV PREFIX_LOCATION=/opt/micromamba
+RUN <<EOT
+#!/bin/bash
+# Download and install micromamba
+curl -L https://micro.mamba.pm/install.sh | bash -s -- --yes --prefix=${PREFIX_LOCATION} --bin-dir=${BIN_FOLDER}
+# Initialize micromamba for bash
+${BIN_FOLDER}/micromamba shell init --shell=bash --root-prefix=${PREFIX_LOCATION}
+micromamba self-update
+micromamba config append channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/
+micromamba config append channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/
+micromamba config append channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/pytorch/
+micromamba config append channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/pytorch/linux-64/
+micromamba config append channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r
+micromamba config append channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2
+micromamba config append channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free
+micromamba config append channels https://mirrors.bfsu.edu.cn/anaconda/cloud/conda-forge/
+micromamba config append channels https://mirrors.bfsu.edu.cn/anaconda/cloud/bioconda/
+mkdir -p ~/.pip
+cat << EOF > ~/.pip/pip.conf
+[global]
+index-url = https://mirrors.aliyun.com/pypi/simple/
+
+[install]
+trusted-host=mirrors.aliyun.com
+EOF
+EOT
--- a/Dockerfile.3.gromacs
+++ b/Dockerfile.3.gromacs
@@ -0,0 +1,253 @@
+# Stage 1: Build environment
+ARG TAG_VERSION
+ARG BASE
+FROM ${BASE}:${TAG_VERSION} AS build-env
+
+ENV CUDA_HOME=/usr/local/cuda
+ENV PATH=$CUDA_HOME/bin:$PATH
+ENV LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH
+ENV CPATH=$CUDA_HOME/include:$CPATH
+ENV LIBRARY_PATH=$CUDA_HOME/lib64:$LIBRARY_PATH
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ENV http_proxy=${HTTP_PROXY}
+ENV https_proxy=${HTTPS_PROXY}
+ARG DEBIAN_FRONTEND="noninteractive"
+ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND}
+ARG ROOT_PASSWD="root"
+ENV ROOT_PASSWD=${ROOT_PASSWD}
+ENV SSH_PORT=2222
+WORKDIR /root
+SHELL ["/bin/bash", "-c"]
+
+# Install base tools for compilation
+RUN <<EOT
+#!/bin/bash
+apt-get install -y --no-install-recommends \
+    build-essential \
+    autotools-dev \
+    libaio-dev \
+    git \
+    libnuma-dev \
+    python3-pip \
+    python3-venv \
+    python3-setuptools \
+    python3-dev \
+    tcl \
+    libtcl8.6 \
+    swig \
+    libfftw3-dev \
+    libfftw3-bin \
+    libfftw3-doc \
+    libibverbs-dev \
+    rdma-core \
+    gdb \
+    valgrind \
+    autoconf \
+    automake \
+    libtool \
+    flex \
+    gfortran \
+    libfuse3-dev \
+    pkg-config \
+    cmake \
+    bzip2 \
+    curl \
+    bzip2 \
+    tar \
+    curl \
+    wget \
+    jq \
+    openssh-client \
+    openssh-server \
+    htop jq \
+    libgl1-mesa-glx bash-completion \
+    iftop iotop ca-certificates
+EOT
+
+# SSH setup
+RUN <<EOT
+#!/bin/bash
+mkdir ~/.ssh
+printf "Host * \n    ForwardAgent yes\nHost *\n    StrictHostKeyChecking no" > ~/.ssh/config
+cp /etc/ssh/sshd_config /etc/ssh/sshd_config.bak
+sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
+sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/' /etc/ssh/sshd_config
+sed -i 's/^\(\s*\)GSSAPIAuthentication yes/\1GSSAPIAuthentication no/' /etc/ssh/ssh_config
+sed -i "s/^#Port 22/Port ${SSH_PORT}/" /etc/ssh/sshd_config
+ssh-keygen -t rsa -b 4096 -f /root/.ssh/id_rsa -N "" <<< y
+cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+chmod 600 /root/.ssh/authorized_keys
+mkdir /var/run/sshd
+echo "root:${ROOT_PASSWD}" | chpasswd
+EOT
+
+# Install FFTW
+ARG FFTW_VERSION="3.3.10"
+ENV FFTW_VERSION=${FFTW_VERSION}
+ENV PATH=/usr/local/fftw:$PATH
+RUN <<EOT
+#!/bin/bash
+wget http://www.fftw.org/fftw-${FFTW_VERSION}.tar.gz
+tar zxvf fftw-${FFTW_VERSION}.tar.gz
+cd fftw-${FFTW_VERSION}
+./configure --prefix=/usr/local/fftw --enable-sse2 --enable-avx --enable-float --enable-avx2 --enable-shared
+make -j$(nproc)
+make install
+EOT
+
+# Install UCX
+ENV UCX_PREFIX=/usr/local/ucx
+ENV PATH=$UCX_PREFIX/bin:$PATH
+ENV LD_LIBRARY_PATH=$UCX_PREFIX/lib:$LD_LIBRARY_PATH
+ENV CPATH=$UCX_PREFIX/include:$CPATH
+ENV LIBRARY_PATH=$UCX_PREFIX/lib:$LIBRARY_PATH
+RUN <<EOT
+#!/bin/bash
+git clone https://github.com/openucx/ucx.git
+cd ucx
+git checkout master
+./autogen.sh
+mkdir build
+cd build
+../contrib/configure-release --prefix=${UCX_PREFIX} --with-cuda=${CUDA_HOME} --with-mlx5 --with-avx --with-rc --with-ud --with-dc --with-dm --with-verbs
+make -j$(nproc)
+make install
+EOT
+
+# Install OpenMPI with UCX support
+ENV MPI_HOME=/usr/local/openmpi
+ENV PATH=${MPI_HOME}/bin:/usr/bin:$PATH
+ENV LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${MPI_HOME}/lib:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
+ENV LIBRARY_PATH=/usr/local/cuda/lib64:${LIBRARY_PATH}
+ENV CPATH=/usr/local/cuda/include:${MPI_HOME}/include:${CUDA_HOME}/include:$CPATH
+RUN <<EOT
+#!/bin/bash
+wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.6.tar.bz2
+tar -xf openmpi-4.1.6.tar.bz2
+cd openmpi-4.1.6
+# git clone --recursive https://github.com/open-mpi/ompi.git
+# cd ompi
+# git checkout main
+./autogen.pl
+mkdir build
+cd build
+../configure FC=gfortran PYTHON=/usr/bin/python3 --with-cuda=/usr/local/cuda --with-cuda-libdir=/usr/local/cuda/lib64 --enable-python-bindings --enable-mpirun-prefix-by-default --prefix=${MPI_HOME} --with-ucx=${UCX_PREFIX} --enable-mca-dso=btl-smcuda,rcache-rgpusm,rcache-gpusm,accelerator-cuda --enable-mca-no-build=btl-uct --without-hcoll
+make -j$(nproc)
+make install
+EOT
+
+# Install PLUMED
+ARG PLUMED_VERSION="2.9.2"
+ENV PLUMED_VERSION=${PLUMED_VERSION}
+ENV LD_LIBRARY_PATH=/usr/local/plumed/lib:$LD_LIBRARY_PATH
+ENV PATH=/usr/local/plumed:/usr/local/plumed/bin:$PATH
+RUN <<EOT
+#!/bin/bash
+curl -L -o plumed-${PLUMED_VERSION}.tar.gz https://github.com/plumed/plumed2/releases/download/v${PLUMED_VERSION}/plumed-${PLUMED_VERSION}.tgz
+tar zxvf plumed-${PLUMED_VERSION}.tar.gz
+cd plumed-${PLUMED_VERSION}
+./configure --prefix=/usr/local/plumed
+make -j$(nproc)
+make install
+EOT
+
+# Install GROMACS with PLUMED support
+ARG GROMACS_VERSION="2021.7"
+ENV GROMACS_VERSION=${GROMACS_VERSION}
+ENV GROMACS_HOME=/usr/local/gromacs-${GROMACS_VERSION}-plumed-${PLUMED_VERSION}
+ENV PATH=$GROMACS_HOME/bin:$PATH
+ARG CUDA_ARCH="75;86;89"
+ENV CUDA_ARCH=${CUDA_ARCH}
+RUN <<EOT
+#!/bin/bash
+wget -c https://ftp.gromacs.org/gromacs/gromacs-${GROMACS_VERSION}.tar.gz
+tar zxvf gromacs-${GROMACS_VERSION}.tar.gz
+cd gromacs-${GROMACS_VERSION}
+plumed-patch -p -e gromacs-${GROMACS_VERSION}
+mkdir build
+cd build
+cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local/gromacs-${GROMACS_VERSION}-plumed-${PLUMED_VERSION} \
+         -DGMX_BUILD_OWN_FFTW=ON \
+         -DREGRESSIONTEST_DOWNLOAD=ON \
+         -DGMX_GPU=CUDA \
+         -DGMX_CUDA_TARGET_COMPUTE="${CUDA_ARCH}" \
+         -DGMX_CUDA_TARGET_SM="${CUDA_ARCH}" \
+         -DGMX_MPI=ON
+make -j$(nproc)
+make install
+echo "source /usr/local/gromacs-${GROMACS_VERSION}-plumed-${PLUMED_VERSION}/bin/GMXRC.bash" >> /root/.bashrc
+EOT
+
+# Install Boost
+COPY file/boost_1_86_0.tar.gz /root
+ENV MODULEPATH=/opt/modulefiles/boost:$MODULEPATH
+RUN <<EOT
+#!/bin/bash
+tar zxvf boost_1_86_0.tar.gz
+cd boost_1_86_0
+./bootstrap.sh --prefix=/opt/boost --with-libraries=all --with-toolset=gcc
+echo "using mpi : /usr/local/openmpi/bin/mpicxx ;" >> project-config.jam
+./b2 -j$(nproc) --layout=tagged link=static,shared threading=multi install
+mkdir -p /opt/modulefiles/boost
+cat << EOF > /opt/modulefiles/boost/1.86.0-openmpi-5.1.0a1
+#%Module1.0
+set prefix /opt/boost
+prepend-path LD_LIBRARY_PATH \$prefix/lib
+prepend-path CPATH \$prefix/include
+prepend-path LIBRARY_PATH \$prefix/lib
+prepend-path PATH \$prefix/bin
+EOF
+EOT
+
+# Install Intel HPCKit and oneMKL
+COPY file/l_HPCKit_p_2024.2.1.79_offline.sh file/l_onemkl_p_2024.2.2.17_offline.sh /root
+RUN <<EOT
+#!/bin/bash
+chmod +x l_HPCKit_p_2024.2.1.79_offline.sh
+./l_HPCKit_p_2024.2.1.79_offline.sh -a --silent --eula accept --install-dir /opt/intel
+chmod +x l_onemkl_p_2024.2.2.17_offline.sh
+./l_onemkl_p_2024.2.2.17_offline.sh -a --silent --eula accept --install-dir /opt/onemkl
+echo "source /opt/intel/setvars.sh" >> ~/.bashrc
+echo "source /opt/onemkl/setvars.sh" >> ~/.bashrc
+rm -rf l_HPCKit_p_2024.2.1.79_offline.sh
+apt-get clean
+rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+rm -rf /root/modules-5.4.0 /root/fftw-${FFTW_VERSION} /root/ucx /root/ompi /root/plumed-${PLUMED_VERSION} /root/gromacs-${GROMACS_VERSION}
+rm -rf /root/Amber24.tar.bz2 /root/AmberTools24.tar.bz2 /root/boost_1_86_0.tar.gz
+EOT
+
+# # Stage 2: Runtime image
+FROM ${BASE}:${TAG_VERSION}
+
+ARG SSH_PORT
+ENV SSH_PORT=${SSH_PORT}
+ARG GROMACS_VERSION
+ARG PLUMED_VERSION
+
+# 复制必要的软件和配置
+COPY --from=build-env /usr/local/fftw /usr/local/fftw
+COPY --from=build-env /usr/local/ucx /usr/local/ucx
+COPY --from=build-env /usr/local/openmpi /usr/local/openmpi
+COPY --from=build-env /usr/local/plumed /usr/local/plumed
+COPY --from=build-env /usr/local/gromacs-${GROMACS_VERSION}-plumed-${PLUMED_VERSION} /usr/local/gromacs-${GROMACS_VERSION}-plumed-${PLUMED_VERSION}
+COPY --from=build-env /opt/boost /opt/boost
+COPY --from=build-env /opt/intel /opt/intel
+COPY --from=build-env /root/.ssh /root/.ssh
+COPY --from=build-env /opt/modulefiles /opt/modulefiles
+COPY --from=build-env /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu
+
+# Set environment variables
+ENV CUDA_HOME=/usr/local/cuda
+ENV PATH=/usr/local/gromacs-${GROMACS_VERSION}-plumed-${PLUMED_VERSION}/bin:/usr/local/openmpi/bin:/opt/intel/bin:/opt/onemkl/bin:/usr/local/ucx/bin:/usr/local/plumed/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/openmpi/lib:/opt/boost/lib:/opt/intel/lib:/opt/onemkl/lib:/usr/local/ucx/lib:/usr/local/plumed/lib:/usr/lib/x86_64-linux-gnu
+ENV CPATH=/usr/local/cuda/include:/usr/local/openmpi/include:/opt/boost/include:/opt/intel/include:/opt/onemkl/include:/usr/local/ucx/include:/usr/local/plumed/include
+ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/openmpi/lib:/opt/boost/lib:/opt/intel/lib:/opt/onemkl/lib:/usr/local/ucx/lib:/usr/local/plumed/lib
+ENV MODULEPATH=/opt/modulefiles/boost:$MODULEPATH
+
+# Set working directory
+WORKDIR /root
+
+# Start SSH service by default
+CMD ["/usr/sbin/sshd", "-D"]
--- a/docker-compose_1_module.yml
+++ b/docker-compose_1_module.yml
@@ -0,0 +1,34 @@
+version: '3.8'
+
+services:
+  gromacs:
+    build: 
+      context: .
+      dockerfile: Dockerfile.1.module
+    # env_file:
+    #   - .env
+    container_name: cuda_1
+    pull_policy: if_not_present
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    restart: unless-stopped
+    image: hotwa/gromacs:module
+    # privileged: true
+    # cap_add:
+    #   - ALL
+    #   - CAP_SYS_PTRACE
+    # shm_size: '16gb'
+    environment:
+      - NVIDIA_VISIBLE_DEVICES=all
+      - NVIDIA_DRIVER_CAPABILITIES=compute,utility
+      - TMPDIR=/var/tmp
+    command: ["tail", "-f", "/dev/null"]
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
--- a/docker-compose_2_micromamba.yml
+++ b/docker-compose_2_micromamba.yml
@@ -0,0 +1,42 @@
+version: '3.8'
+
+services:
+  gromacs:
+    build: 
+      context: .
+      dockerfile: Dockerfile.2.micromamba
+      args: 
+        CACHEBUST: 1
+        BASE: "hotwa/gromacs"
+        TAG_VERSION: "module"
+    # env_file:
+    #   - .env
+    container_name: cuda_2
+    pull_policy: if_not_present
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    restart: unless-stopped
+    image: hotwa/gromacs:micromamba
+    # privileged: true
+    # cap_add:
+    #   - ALL
+    #   - CAP_SYS_PTRACE
+    # shm_size: '16gb'
+    environment:
+      - NVIDIA_VISIBLE_DEVICES=all
+      - NVIDIA_DRIVER_CAPABILITIES=compute,utility
+      - TMPDIR=/var/tmp
+    # ports:
+    #   - "53322:2222" 
+    # network_mode: host
+    # command: ["/usr/sbin/sshd", "-D"]
+    command: ["tail", "-f", "/dev/null"]
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
--- a/docker-compose_3_gromacs.yml
+++ b/docker-compose_3_gromacs.yml
@@ -0,0 +1,47 @@
+version: '3.8'
+
+services:
+  gromacs:
+    build: 
+      context: .
+      dockerfile: Dockerfile.3.gromacs
+      args: 
+        CACHEBUST: 1
+        BASE: "hotwa/gromacs"
+        TAG_VERSION: "micromamba"
+        PLUMED_VERSION: "2.9.2"
+        FFTW_VERSION: "3.3.10"
+        GROMACS_VERSION: "2021.7"
+        CUDA_ARCH: "75;86;89"
+        BUILDKIT_INLINE_CACHE: 1
+    # env_file:
+    #   - .env
+    container_name: cuda_3
+    pull_policy: if_not_present
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    restart: unless-stopped
+    image: hotwa/gromacs:gromacs
+    # privileged: true
+    # cap_add:
+    #   - ALL
+    #   - CAP_SYS_PTRACE
+    # shm_size: '16gb'
+    environment:
+      - NVIDIA_VISIBLE_DEVICES=all
+      - NVIDIA_DRIVER_CAPABILITIES=compute,utility
+      - TMPDIR=/var/tmp
+    # ports:
+    #   - "53322:2222" 
+    # network_mode: host
+    # command: ["/usr/sbin/sshd", "-D"]
+    command: ["tail", "-f", "/dev/null"]
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]