diff --git a/Dockerfile.1.module b/Dockerfile.1.module new file mode 100644 index 0000000..b4d37c0 --- /dev/null +++ b/Dockerfile.1.module @@ -0,0 +1,58 @@ +# syntax=docker/dockerfile:1.3-labs +# NOTE: Building this image requires Docker version >= 23.0. + +# Stage 1: Build environment +ARG TAG_VERSION="12.4.1-cudnn-devel-ubuntu22.04" +FROM nvidia/cuda:${TAG_VERSION} + +ENV CUDA_HOME=/usr/local/cuda +ENV PATH=$CUDA_HOME/bin:$PATH +ENV LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH +ENV CPATH=$CUDA_HOME/include:$CPATH +ENV LIBRARY_PATH=$CUDA_HOME/lib64:$LIBRARY_PATH +ARG HTTP_PROXY +ARG HTTPS_PROXY +ENV http_proxy=${HTTP_PROXY} +ENV https_proxy=${HTTPS_PROXY} +ARG DEBIAN_FRONTEND="noninteractive" +ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND} +ENV SSH_PORT=2222 +WORKDIR /root +SHELL ["/bin/bash", "-c"] + +# Replace sources.list with Aliyun mirror +RUN < /etc/apt/sources.list +deb https://mirrors.aliyun.com/ubuntu/ jammy main restricted universe multiverse +deb-src https://mirrors.aliyun.com/ubuntu/ jammy main restricted universe multiverse +deb https://mirrors.aliyun.com/ubuntu/ jammy-security main restricted universe multiverse +deb-src https://mirrors.aliyun.com/ubuntu/ jammy-security main restricted universe multiverse +deb https://mirrors.aliyun.com/ubuntu/ jammy-updates main restricted universe multiverse +deb-src https://mirrors.aliyun.com/ubuntu/ jammy-updates main restricted universe multiverse +deb https://mirrors.aliyun.com/ubuntu/ jammy-backports main restricted universe multiverse +deb-src https://mirrors.aliyun.com/ubuntu/ jammy-backports main restricted universe multiverse +EOF +apt-get update && apt-get install -y tzdata +ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime +echo "Asia/Shanghai" > /etc/timezone +apt-get install -y tcl-dev \ + environment-modules \ + wget \ + curl \ + bzip2 \ + tar \ + less +mkdir -p ~/.local/modules +wget https://sourceforge.net/projects/modules/files/Modules/modules-5.5.0/modules-5.5.0.tar.gz +# curl -LJO https://github.com/cea-hpc/modules/releases/download/v5.5.0/modules-5.5.0.tar.gz +tar xfz modules-5.5.0.tar.gz +cd modules-5.5.0 +./configure --prefix=$HOME/.local/modules --with-tcl-lib=/usr/local/tools/tcl/lib --with-tcl-inc=/usr/local/tools/tcl/include --modulefilesdir=$HOME/.local/modules/modulefiles +make -j$(nproc) +make install +echo "source $HOME/.local/modules/init/profile.sh" >> ~/.bashrc +echo "source $HOME/.local/modules/init/profile.sh" >> /etc/profile +rm -rf /root/modules* +EOT \ No newline at end of file diff --git a/Dockerfile.2.micromamba b/Dockerfile.2.micromamba new file mode 100644 index 0000000..a4411e5 --- /dev/null +++ b/Dockerfile.2.micromamba @@ -0,0 +1,39 @@ +# syntax=docker/dockerfile:1.3-labs +# NOTE: Building this image requires Docker version >= 23.0. + +# Stage 1: Build environment +ARG TAG_VERSION +ARG BASE +FROM ${BASE}:${TAG_VERSION} + +# Install micromamba silently +WORKDIR /root +ENV BIN_FOLDER=/usr/local/bin +ENV INIT_YES=yes +ENV CONDA_FORGE_YES=yes +ENV PREFIX_LOCATION=/opt/micromamba +RUN < ~/.pip/pip.conf +[global] +index-url = https://mirrors.aliyun.com/pypi/simple/ + +[install] +trusted-host=mirrors.aliyun.com +EOF +EOT \ No newline at end of file diff --git a/Dockerfile.3.gromacs b/Dockerfile.3.gromacs new file mode 100644 index 0000000..3b703ed --- /dev/null +++ b/Dockerfile.3.gromacs @@ -0,0 +1,253 @@ +# Stage 1: Build environment +ARG TAG_VERSION +ARG BASE +FROM ${BASE}:${TAG_VERSION} AS build-env + +ENV CUDA_HOME=/usr/local/cuda +ENV PATH=$CUDA_HOME/bin:$PATH +ENV LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH +ENV CPATH=$CUDA_HOME/include:$CPATH +ENV LIBRARY_PATH=$CUDA_HOME/lib64:$LIBRARY_PATH +ARG HTTP_PROXY +ARG HTTPS_PROXY +ENV http_proxy=${HTTP_PROXY} +ENV https_proxy=${HTTPS_PROXY} +ARG DEBIAN_FRONTEND="noninteractive" +ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND} +ARG ROOT_PASSWD="root" +ENV ROOT_PASSWD=${ROOT_PASSWD} +ENV SSH_PORT=2222 +WORKDIR /root +SHELL ["/bin/bash", "-c"] + +# Install base tools for compilation +RUN < ~/.ssh/config +cp /etc/ssh/sshd_config /etc/ssh/sshd_config.bak +sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config +sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config +sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/' /etc/ssh/sshd_config +sed -i 's/^\(\s*\)GSSAPIAuthentication yes/\1GSSAPIAuthentication no/' /etc/ssh/ssh_config +sed -i "s/^#Port 22/Port ${SSH_PORT}/" /etc/ssh/sshd_config +ssh-keygen -t rsa -b 4096 -f /root/.ssh/id_rsa -N "" <<< y +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 /root/.ssh/authorized_keys +mkdir /var/run/sshd +echo "root:${ROOT_PASSWD}" | chpasswd +EOT + +# Install FFTW +ARG FFTW_VERSION="3.3.10" +ENV FFTW_VERSION=${FFTW_VERSION} +ENV PATH=/usr/local/fftw:$PATH +RUN <> /root/.bashrc +EOT + +# Install Boost +COPY file/boost_1_86_0.tar.gz /root +ENV MODULEPATH=/opt/modulefiles/boost:$MODULEPATH +RUN <> project-config.jam +./b2 -j$(nproc) --layout=tagged link=static,shared threading=multi install +mkdir -p /opt/modulefiles/boost +cat << EOF > /opt/modulefiles/boost/1.86.0-openmpi-5.1.0a1 +#%Module1.0 +set prefix /opt/boost +prepend-path LD_LIBRARY_PATH \$prefix/lib +prepend-path CPATH \$prefix/include +prepend-path LIBRARY_PATH \$prefix/lib +prepend-path PATH \$prefix/bin +EOF +EOT + +# Install Intel HPCKit and oneMKL +COPY file/l_HPCKit_p_2024.2.1.79_offline.sh file/l_onemkl_p_2024.2.2.17_offline.sh /root +RUN <> ~/.bashrc +echo "source /opt/onemkl/setvars.sh" >> ~/.bashrc +rm -rf l_HPCKit_p_2024.2.1.79_offline.sh +apt-get clean +rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +rm -rf /root/modules-5.4.0 /root/fftw-${FFTW_VERSION} /root/ucx /root/ompi /root/plumed-${PLUMED_VERSION} /root/gromacs-${GROMACS_VERSION} +rm -rf /root/Amber24.tar.bz2 /root/AmberTools24.tar.bz2 /root/boost_1_86_0.tar.gz +EOT + +# # Stage 2: Runtime image +FROM ${BASE}:${TAG_VERSION} + +ARG SSH_PORT +ENV SSH_PORT=${SSH_PORT} +ARG GROMACS_VERSION +ARG PLUMED_VERSION + +# 复制必要的软件和配置 +COPY --from=build-env /usr/local/fftw /usr/local/fftw +COPY --from=build-env /usr/local/ucx /usr/local/ucx +COPY --from=build-env /usr/local/openmpi /usr/local/openmpi +COPY --from=build-env /usr/local/plumed /usr/local/plumed +COPY --from=build-env /usr/local/gromacs-${GROMACS_VERSION}-plumed-${PLUMED_VERSION} /usr/local/gromacs-${GROMACS_VERSION}-plumed-${PLUMED_VERSION} +COPY --from=build-env /opt/boost /opt/boost +COPY --from=build-env /opt/intel /opt/intel +COPY --from=build-env /root/.ssh /root/.ssh +COPY --from=build-env /opt/modulefiles /opt/modulefiles +COPY --from=build-env /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu + +# Set environment variables +ENV CUDA_HOME=/usr/local/cuda +ENV PATH=/usr/local/gromacs-${GROMACS_VERSION}-plumed-${PLUMED_VERSION}/bin:/usr/local/openmpi/bin:/opt/intel/bin:/opt/onemkl/bin:/usr/local/ucx/bin:/usr/local/plumed/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/openmpi/lib:/opt/boost/lib:/opt/intel/lib:/opt/onemkl/lib:/usr/local/ucx/lib:/usr/local/plumed/lib:/usr/lib/x86_64-linux-gnu +ENV CPATH=/usr/local/cuda/include:/usr/local/openmpi/include:/opt/boost/include:/opt/intel/include:/opt/onemkl/include:/usr/local/ucx/include:/usr/local/plumed/include +ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/openmpi/lib:/opt/boost/lib:/opt/intel/lib:/opt/onemkl/lib:/usr/local/ucx/lib:/usr/local/plumed/lib +ENV MODULEPATH=/opt/modulefiles/boost:$MODULEPATH + +# Set working directory +WORKDIR /root + +# Start SSH service by default +CMD ["/usr/sbin/sshd", "-D"] diff --git a/docker-compose_1_module.yml b/docker-compose_1_module.yml new file mode 100644 index 0000000..cfb2751 --- /dev/null +++ b/docker-compose_1_module.yml @@ -0,0 +1,34 @@ +version: '3.8' + +services: + gromacs: + build: + context: . + dockerfile: Dockerfile.1.module + # env_file: + # - .env + container_name: cuda_1 + pull_policy: if_not_present + ulimits: + memlock: + soft: -1 + hard: -1 + restart: unless-stopped + image: hotwa/gromacs:module + # privileged: true + # cap_add: + # - ALL + # - CAP_SYS_PTRACE + # shm_size: '16gb' + environment: + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + - TMPDIR=/var/tmp + command: ["tail", "-f", "/dev/null"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] diff --git a/docker-compose_2_micromamba.yml b/docker-compose_2_micromamba.yml new file mode 100644 index 0000000..de579a8 --- /dev/null +++ b/docker-compose_2_micromamba.yml @@ -0,0 +1,42 @@ +version: '3.8' + +services: + gromacs: + build: + context: . + dockerfile: Dockerfile.2.micromamba + args: + CACHEBUST: 1 + BASE: "hotwa/gromacs" + TAG_VERSION: "module" + # env_file: + # - .env + container_name: cuda_2 + pull_policy: if_not_present + ulimits: + memlock: + soft: -1 + hard: -1 + restart: unless-stopped + image: hotwa/gromacs:micromamba + # privileged: true + # cap_add: + # - ALL + # - CAP_SYS_PTRACE + # shm_size: '16gb' + environment: + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + - TMPDIR=/var/tmp + # ports: + # - "53322:2222" + # network_mode: host + # command: ["/usr/sbin/sshd", "-D"] + command: ["tail", "-f", "/dev/null"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] diff --git a/docker-compose_3_gromacs.yml b/docker-compose_3_gromacs.yml new file mode 100644 index 0000000..4005316 --- /dev/null +++ b/docker-compose_3_gromacs.yml @@ -0,0 +1,47 @@ +version: '3.8' + +services: + gromacs: + build: + context: . + dockerfile: Dockerfile.3.gromacs + args: + CACHEBUST: 1 + BASE: "hotwa/gromacs" + TAG_VERSION: "micromamba" + PLUMED_VERSION: "2.9.2" + FFTW_VERSION: "3.3.10" + GROMACS_VERSION: "2021.7" + CUDA_ARCH: "75;86;89" + BUILDKIT_INLINE_CACHE: 1 + # env_file: + # - .env + container_name: cuda_3 + pull_policy: if_not_present + ulimits: + memlock: + soft: -1 + hard: -1 + restart: unless-stopped + image: hotwa/gromacs:gromacs + # privileged: true + # cap_add: + # - ALL + # - CAP_SYS_PTRACE + # shm_size: '16gb' + environment: + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + - TMPDIR=/var/tmp + # ports: + # - "53322:2222" + # network_mode: host + # command: ["/usr/sbin/sshd", "-D"] + command: ["tail", "-f", "/dev/null"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu]