update

finetune/Dockerfile

@@ -311,6 +311,8 @@ cd ..
 EOT
 
 # CUDA_ARCH_LIST="80;86;89;90"
+ARG DEEPSPEED_VERSION="0.8.3"
+ENV DEEPSPEED_VERSION=${DEEPSPEED_VERSION}
 ARG DEEPSPEED_INSTALL_FLAGS="--allow_sudo --pip_sudo --no_clean"
 ENV DEEPSPEED_INSTALL_FLAGS=${DEEPSPEED_INSTALL_FLAGS}
 ARG CUDA_ARCH_LIST="80;86;89;90"
@@ -319,7 +321,7 @@ ARG DS_BUILD_SPARSE_ATTN=0
 ENV DS_BUILD_SPARSE_ATTN=${DS_BUILD_SPARSE_ATTN}
 ARG DS_BUILD_FUSED_ADAM=1
 ENV DS_BUILD_FUSED_ADAM=${DS_BUILD_FUSED_ADAM}
-ARG DS_BUILD_CPU_ADAM=0
+ARG DS_BUILD_CPU_ADAM=1
 ENV DS_BUILD_CPU_ADAM=${DS_BUILD_CPU_ADAM}
 ARG DS_BUILD_OPS=1
 ENV DS_BUILD_OPS=${DS_BUILD_OPS}
@@ -338,50 +340,47 @@ git checkout master
 mkdir build
 cd build 
 cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local
-make -j install
+make -j"$(nproc)" install
 git clone https://github.com/microsoft/DeepSpeed-Kernels.git ${STAGE_DIR}/DeepSpeed-Kernels
 cd ${STAGE_DIR}/DeepSpeed-Kernels
 CUDA_ARCH_LIST=${CUDA_ARCH_LIST} python setup.py bdist_wheel
-# pip install dist/deepspeed_kernels-*.whl
-CUDA_ARCH_LIST=${CUDA_ARCH_LIST} pip install -v .
+pip install dist/deepspeed_kernels-*.whl
+# CUDA_ARCH_LIST=${CUDA_ARCH_LIST} pip install -v .
 git clone https://github.com/microsoft/DeepSpeed.git ${STAGE_DIR}/DeepSpeed
 cd ${STAGE_DIR}/DeepSpeed 
 git checkout . 
-git checkout master 
+git checkout v0.8.3
 python setup.py bdist_wheel
 DS_BUILD_OPS=${DS_BUILD_OPS} pip install dist/deepspeed*.whl --force-reinstall
 # DS_BUILD_OPS=${DS_BUILD_OPS} pip install -r requirements/requirements.txt
-# DS_BUILD_OPS=0 DS_BUILD_SPARSE_ATTN=0 DS_BUILD_CPU_ADAM=0 DS_BUILD_FUSED_ADAM=1 pip install -U --no-cache-dir .
 # ./install.sh ${DEEPSPEED_INSTALL_FLAGS} --hostfile /job/hostfile # ./install.sh --allow_sudo --pip_sudo --no_clean --hostfile /path/to/your/hostfile
-cd ..
-# rm -rf ${STAGE_DIR}/DeepSpeed
 EOT
 
-RUN <<EOT
-#!/bin/bash
-source /opt/conda/etc/profile.d/conda.sh
-conda activate ${CONDA_ENV_NAME}
-# install transformers
-git clone https://github.com/huggingface/transformers ${STAGE_DIR}/transformers
-cd ${STAGE_DIR}/transformers
-python3 ./setup.py develop
-python3 -m pip install -U --no-cache-dir "pydantic<2"
-# install flash-attn
-# pip install packaging -i https://pypi.org/simple/ --trusted-host pypi.org
-pip install flash-attn --no-build-isolation -i https://pypi.org/simple/ --trusted-host pypi.org
-EOT
+# RUN <<EOT
+# #!/bin/bash
+# source /opt/conda/etc/profile.d/conda.sh
+# conda activate ${CONDA_ENV_NAME}
+# # install transformers
+# git clone https://github.com/huggingface/transformers ${STAGE_DIR}/transformers
+# cd ${STAGE_DIR}/transformers
+# python3 ./setup.py develop
+# python3 -m pip install -U --no-cache-dir "pydantic<2"
+# # install flash-attn
+# # pip install packaging -i https://pypi.org/simple/ --trusted-host pypi.org
+# pip install flash-attn --no-build-isolation -i https://pypi.org/simple/ --trusted-host pypi.org
+# EOT
 
-RUN <<EOT
-#!/bin/bash
-source /opt/conda/etc/profile.d/conda.sh
-conda activate ${CONDA_ENV_NAME}
-pip install optimum
-pip install peft tiktoken \
-    tqdm matplotlib seaborn numpy pandas scikit-learn diffusers \
-    huggingface_hub spacy blobfile pycocotools \
-    xformers open_clip_torch \
-    zstandard -i https://pypi.org/simple/ --trusted-host pypi.org
-EOT
+# RUN <<EOT
+# #!/bin/bash
+# source /opt/conda/etc/profile.d/conda.sh
+# conda activate ${CONDA_ENV_NAME}
+# pip install optimum
+# pip install peft tiktoken \
+#     tqdm matplotlib seaborn numpy pandas scikit-learn diffusers \
+#     huggingface_hub spacy blobfile pycocotools \
+#     xformers open_clip_torch \
+#     zstandard -i https://pypi.org/simple/ --trusted-host pypi.org
+# EOT
 
 # add vscode server
 # RUN <<EOT