update

finetune/docker-compose_stack.yml

@@ -42,7 +42,7 @@ services:
         reservations:
           generic_resources:
             - discrete_resource_spec:
-                kind: NVIDIAGPU
+                kind: "NVIDIA-GPU"
                 value: 1
       placement:
         constraints: [node.platform.os == linux]

finetune/docker-compose_stack1.yml

@@ -15,7 +15,7 @@ services:
         reservations:
           generic_resources:
             - discrete_resource_spec:
-                kind: NVIDIAGPU
+                kind: "NVIDIA-GPU"
                 value: 1
       placement:
         constraints:

finetune/docker-compose_stack2.yml

@@ -22,10 +22,14 @@ services:
         DEEPSPEED_VERSION: "master"
         DEEPSPEED_INSTALL_FLAGS: "--allow_sudo"
     volumes:
-      - ./src:/bbtft
+      - type: tmpfs
-      - ./id_rsa_finetune:/root/.ssh/id_rsa
+        target: /dev/shm
-      - ./id_rsa.pub:/root/.ssh/id_rsa.pub
+        tmpfs:
-    container_name: ubuntu-finetune
+          size: 32000000000 # 32GB
+      # - ./src:/bbtft
+      # - ./id_rsa_finetune:/root/.ssh/id_rsa
+      # - ./id_rsa.pub:/root/.ssh/id_rsa.pub
+    # container_name: ubuntu-finetune
     image: hotwa/deepspeed:pt23
     shm_size: '32gb'
     ports:
@@ -37,19 +41,19 @@ services:
     networks:
       - my-custom-bridge
     deploy:
-      replicas: 1
+      replicas: 4
       resources:
         reservations:
           generic_resources:
             - discrete_resource_spec:
-                kind: NVIDIAGPU
+                kind: "NVIDIA-GPU"
-                value: 1
+                value: 8
       placement:
         constraints: [node.labels.gpu == true]
     cap_add:
       - IPC_LOCK
     devices:
-      - /dev/infiniband:/dev/infiniband
+      - "/dev/infiniband:/dev/infiniband:rwm"
 
 networks:
   my-custom-bridge: