From e86aaf240dd03335def0659d48d41f96b7998233 Mon Sep 17 00:00:00 2001 From: hotwa Date: Fri, 5 Jan 2024 15:39:36 +0800 Subject: [PATCH 01/13] add gpu --- README.md | 43 ++++++++++++++++++++++++++++++++++++++++++- jupyterhub_config.py | 11 ++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 22ed3d4..5ae1c32 100644 --- a/README.md +++ b/README.md @@ -63,4 +63,45 @@ docker buildx build -t hotwa/notebook:latest . -f Dockerfile - [openshift/source-to-image](https://github.com/openshift/source-to-image) - A tool for building artifacts from source code and injecting them into docker images - [jupyter-on-openshift/jupyter-notebooks](https://github.com/jupyter-on-openshift/jupyter-notebooks) - - OpenShift compatible S2I builder for basic notebook images \ No newline at end of file + OpenShift compatible S2I builder for basic notebook images + +## nvidia-container-toolkit add + +参考了 llama [容器部署](https://hub.docker.com/r/ollama/ollama) + +```shell +curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \ + | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg +curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \ + | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \ + | sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list +sudo apt-get update +sudo apt-get install -y nvidia-container-toolkit +``` + +Configure Docker to use Nvidia driver + +```shell +sudo nvidia-ctk runtime configure --runtime=docker +sudo systemctl restart docker +``` + +配置完成之后： + +```shell +docker-compose up -d +``` + +在jupyterhub_config.py中添加：这样jupyterhub的容器启动镜像就可以使用宿主机的显卡了 + +```python +# GPU 和网络配置 +c.DockerSpawner.extra_host_config = { + #'network_mode': 'host', + 'runtime': 'nvidia' +} +c.DockerSpawner.environment = { + 'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility', + 'NVIDIA_VISIBLE_DEVICES': 'all' +} +``` diff --git a/jupyterhub_config.py b/jupyterhub_config.py index 74885fd..eac5bcb 100644 --- a/jupyterhub_config.py +++ b/jupyterhub_config.py @@ -55,4 +55,13 @@ c.NativeAuthenticator.open_signup = True # Allowed admins admin = os.environ.get("JUPYTERHUB_ADMIN") if admin: - c.Authenticator.admin_users = [admin] \ No newline at end of file + c.Authenticator.admin_users = [admin] + +# GPU 和网络配置 +c.DockerSpawner.extra_host_config = { + 'runtime': 'nvidia' +} +c.DockerSpawner.environment = { + 'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility', + 'NVIDIA_VISIBLE_DEVICES': 'all' +} From eb7aa5b5e43ab634af443ad309c3a825fdf34f23 Mon Sep 17 00:00:00 2001 From: hotwa Date: Sat, 13 Jan 2024 21:44:03 +0800 Subject: [PATCH 02/13] add remove --- README.md | 12 ++++++++ spawnerdockerfile/Dockerfile.base-notebook | 34 +++++++++++++--------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 5ae1c32..d31ce1a 100644 --- a/README.md +++ b/README.md @@ -105,3 +105,15 @@ c.DockerSpawner.environment = { 'NVIDIA_VISIBLE_DEVICES': 'all' } ``` + +## 清除卷 + + +```shell +docker volume ls +docker-compose down +docker rm +docker volume rm basic-example_jupyterhub-data +docker volume rm docker-jupyterhub_jupyterhub-data +docker volume rm jupyterhub-user-admin +``` \ No newline at end of file diff --git a/spawnerdockerfile/Dockerfile.base-notebook b/spawnerdockerfile/Dockerfile.base-notebook index 70e5fa8..d4a5a26 100644 --- a/spawnerdockerfile/Dockerfile.base-notebook +++ b/spawnerdockerfile/Dockerfile.base-notebook @@ -88,20 +88,17 @@ ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime echo 'Asia/Shanghai' > /etc/timezone dpkg-reconfigure -f noninteractive tzdata # 安装所需的软件包 -apt-get install -y python3 python3-pip gcc g++ build-essential nodejs npm gdebi-core curl wget openssh-server vim lrzsz net-tools sudo git texlive-full +apt-get install -y python3 python3-pip gcc g++ build-essential nodejs npm gdebi-core curl wget openssh-server vim lrzsz net-tools sudo git # 创建新用户 useradd -m -s /bin/bash ${CREATE_USER} echo "${CREATE_USER}:${CREATE_USER_PASSWD}" | chpasswd echo "${CREATE_USER} ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers -# 安装micromamba -echo "1" | bash <(curl -s https://raw.githubusercontent.com/hotwa/MicroMamba_Installer/main/install.sh) +# # 安装micromamba +# echo "1" | bash <(curl -s https://raw.githubusercontent.com/hotwa/MicroMamba_Installer/main/install.sh) # 安装 Jupyter 和相关软件 npm install -g configurable-http-proxy -python3 -m pip install jupyterhub jupyterlab notebook radian pycurl jupyter-rsession-proxy ipykernel jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -EOT - -RUN < Date: Sat, 13 Jan 2024 23:28:48 +0800 Subject: [PATCH 03/13] update --- spawnerdockerfile/docker_healthcheck.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/spawnerdockerfile/docker_healthcheck.py b/spawnerdockerfile/docker_healthcheck.py index 8f3338e..05229f4 100755 --- a/spawnerdockerfile/docker_healthcheck.py +++ b/spawnerdockerfile/docker_healthcheck.py @@ -3,6 +3,7 @@ # Distributed under the terms of the Modified BSD License. import json import os +import subprocess from pathlib import Path import requests @@ -10,7 +11,19 @@ import requests # Several operations below deliberately don't check for possible errors # As this is a healthcheck, it should succeed or raise an exception on error -runtime_dir = Path("/home/") / os.environ["NB_USER"] / ".local/share/jupyter/runtime/" +# Docker runs healtchecks using an exec +# It uses the default user configured when running the image: root for the case of a custom NB_USER or jovyan for the case of the default image user. +# We manually change HOME to make `jupyter --runtime-dir` report a correct path +# More information: +result = subprocess.run( + ["jupyter", "--runtime-dir"], + check=True, + capture_output=True, + text=True, + env=dict(os.environ) | {"HOME": "/home/" + os.environ["NB_USER"]}, +) +runtime_dir = Path(result.stdout.rstrip()) + json_file = next(runtime_dir.glob("*server-*.json")) url = json.loads(json_file.read_bytes())["url"] @@ -23,4 +36,4 @@ proxies = { r = requests.get(url, proxies=proxies, verify=False) # request without SSL verification r.raise_for_status() -print(r.content) +print(r.content) \ No newline at end of file From e4faf406221cab34a42106e3e7419181e882bbdd Mon Sep 17 00:00:00 2001 From: hotwa Date: Sat, 13 Jan 2024 23:29:14 +0800 Subject: [PATCH 04/13] update --- spawnerdockerfile/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spawnerdockerfile/Dockerfile b/spawnerdockerfile/Dockerfile index 545e2db..aabd396 100644 --- a/spawnerdockerfile/Dockerfile +++ b/spawnerdockerfile/Dockerfile @@ -72,4 +72,4 @@ HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \ # Switch back to jovyan to avoid accidental container runs as root USER ${NB_UID} -WORKDIR "${HOME}" +WORKDIR "${HOME}" \ No newline at end of file From 3ce8af9a0e6580b2d0b314f5eb21cf14c2424197 Mon Sep 17 00:00:00 2001 From: hotwa Date: Sat, 13 Jan 2024 23:29:53 +0800 Subject: [PATCH 05/13] change --- README.md | 6 ++++ spawnerdockerfile/Dockerfile.base-notebook | 36 +++++++++++++--------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index d31ce1a..3e16874 100644 --- a/README.md +++ b/README.md @@ -116,4 +116,10 @@ docker rm docker volume rm basic-example_jupyterhub-data docker volume rm docker-jupyterhub_jupyterhub-data docker volume rm jupyterhub-user-admin +``` + +## 构建 + +```shell +docker buildx build -t hotwa/notebook:latest -f Dockerfile.base-notebook . --load ``` \ No newline at end of file diff --git a/spawnerdockerfile/Dockerfile.base-notebook b/spawnerdockerfile/Dockerfile.base-notebook index d4a5a26..d3a9600 100644 --- a/spawnerdockerfile/Dockerfile.base-notebook +++ b/spawnerdockerfile/Dockerfile.base-notebook @@ -124,22 +124,30 @@ ENV MAMBA_ROOT_PREFIX=/home/${CREATE_USER}/mamba # add pdb env RUN < Date: Sat, 13 Jan 2024 23:47:32 +0800 Subject: [PATCH 06/13] origin version --- Dockerfile.jupyterhub.bak | 86 ++++----------------------------------- 1 file changed, 9 insertions(+), 77 deletions(-) diff --git a/Dockerfile.jupyterhub.bak b/Dockerfile.jupyterhub.bak index bf6ab79..3741af8 100644 --- a/Dockerfile.jupyterhub.bak +++ b/Dockerfile.jupyterhub.bak @@ -1,80 +1,12 @@ -FROM jupyter/scipy-notebook -LABEL base_image="Ubuntu 20.04.3 LTS" +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +ARG JUPYTERHUB_VERSION +FROM quay.io/jupyterhub/jupyterhub:$JUPYTERHUB_VERSION -ARG CREATE_USER="jovyan" -ARG CREATE_USER_PASSWD="password" -ARG ROOT_PASSWD="password" -ARG HOME="/home/${CREATE_USER}" -ARG DEBIAN_FRONTEND="noninteractive" -ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND} - -USER root -RUN < /etc/timezone -dpkg-reconfigure -f noninteractive tzdata -# 安装所需的软件包 -apt-get install -y python3 python3-pip gcc g++ build-essential nodejs npm gdebi-core curl wget openssh-server vim lrzsz net-tools sudo git -# 创建新用户 -useradd -m -s /bin/bash ${CREATE_USER} -echo "${CREATE_USER}:${CREATE_USER_PASSWD}" | chpasswd -echo "${CREATE_USER} ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers -EOT - -RUN < Date: Sat, 13 Jan 2024 23:47:53 +0800 Subject: [PATCH 07/13] no change --- Dockerfile.jupyterhub | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Dockerfile.jupyterhub b/Dockerfile.jupyterhub index 09b5841..0cfd894 100644 --- a/Dockerfile.jupyterhub +++ b/Dockerfile.jupyterhub @@ -61,18 +61,6 @@ echo "1" | bash <(curl -s https://raw.githubusercontent.com/hotwa/MicroMamba_Ins mkdir -p /root/workspace/personal /root/workspace/project EOT -# RUN < Date: Sun, 14 Jan 2024 21:11:59 +0800 Subject: [PATCH 08/13] update --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index 3e16874..b42c2d5 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,30 @@ docker volume rm jupyterhub-user-admin ## 构建 +构建基础notebook镜像 + ```shell docker buildx build -t hotwa/notebook:latest -f Dockerfile.base-notebook . --load +``` + +构建jupyterhub镜像 + +```shell +docker compose build +``` + +## 推送镜像至阿里云私有仓库 + +```shell +docker login --username=ze.ga@qq.com registry.cn-hangzhou.aliyuncs.com +# 重命名 +docker tag 2ad3860183ce registry.cn-hangzhou.aliyuncs.com/hotwa/jupyterhub:latest +docker tag ddf815cbaa9b registry.cn-hangzhou.aliyuncs.com/hotwa/notebook:latest +# 推送 +docker login --username=ze.ga@qq.com registry.cn-hangzhou.aliyuncs.com +docker push registry.cn-hangzhou.aliyuncs.com/hotwa/jupyterhub:latest +docker push registry.cn-hangzhou.aliyuncs.com/hotwa/notebook:latest +# 拉取 +docker pull registry.cn-hangzhou.aliyuncs.com/hotwa/jupyterhub:latest +docker pull registry.cn-hangzhou.aliyuncs.com/hotwa/notebook:latest ``` \ No newline at end of file From 2ca76bf252f10cd2efa9b3c55a78073467fd2f4c Mon Sep 17 00:00:00 2001 From: hotwa Date: Sun, 14 Jan 2024 23:42:48 +0800 Subject: [PATCH 09/13] add JUPYTERHUB_IP env --- docker-compose.yml | 2 ++ jupyterhub_config.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index fc9c028..9013453 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,8 @@ services: DOCKER_NOTEBOOK_IMAGE: hotwa/notebook:latest # Notebook directory inside user image DOCKER_NOTEBOOK_DIR: /home/jovyan/work + # add tailscale ip + JUPYTERHUB_IP: 0.0.0.0 volumes: jupyterhub-data: diff --git a/jupyterhub_config.py b/jupyterhub_config.py index eac5bcb..44c870b 100644 --- a/jupyterhub_config.py +++ b/jupyterhub_config.py @@ -39,7 +39,7 @@ c.DockerSpawner.remove = True c.DockerSpawner.debug = True # User containers will access hub by container name on the Docker network -c.JupyterHub.hub_ip = "jupyterhub" +c.JupyterHub.hub_ip = os.environ.get('JUPYTERHUB_IP', '0.0.0.0') c.JupyterHub.hub_port = 8080 # Persist hub data on volume mounted inside container From f692b646b9d0cfbd1da3963dc09e0ac957def2b4 Mon Sep 17 00:00:00 2001 From: hotwa Date: Sun, 14 Jan 2024 23:57:26 +0800 Subject: [PATCH 10/13] restore --- docker-compose.yml | 2 -- jupyterhub_config.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9013453..fc9c028 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,8 +35,6 @@ services: DOCKER_NOTEBOOK_IMAGE: hotwa/notebook:latest # Notebook directory inside user image DOCKER_NOTEBOOK_DIR: /home/jovyan/work - # add tailscale ip - JUPYTERHUB_IP: 0.0.0.0 volumes: jupyterhub-data: diff --git a/jupyterhub_config.py b/jupyterhub_config.py index 44c870b..10fba88 100644 --- a/jupyterhub_config.py +++ b/jupyterhub_config.py @@ -39,7 +39,7 @@ c.DockerSpawner.remove = True c.DockerSpawner.debug = True # User containers will access hub by container name on the Docker network -c.JupyterHub.hub_ip = os.environ.get('JUPYTERHUB_IP', '0.0.0.0') +c.JupyterHub.hub_ip = 'jupyterhub' c.JupyterHub.hub_port = 8080 # Persist hub data on volume mounted inside container From 207781bed9dc32b332386dd575a9c82ac81f5bbc Mon Sep 17 00:00:00 2001 From: hotwa Date: Sun, 14 Jan 2024 23:58:02 +0800 Subject: [PATCH 11/13] explanin --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index fc9c028..0e0ddc6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,3 +42,5 @@ volumes: networks: jupyterhub-network: name: jupyterhub-network + +# use 127.0.0.1:8000 access From d04a3d2d16fa769a8e962fb779af486a470a01f8 Mon Sep 17 00:00:00 2001 From: hotwa Date: Mon, 15 Jan 2024 09:31:31 +0800 Subject: [PATCH 12/13] =?UTF-8?q?add=20nginx=20but=20notebook=20show=20con?= =?UTF-8?q?necting=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yml | 11 +++++++++++ nginx.conf | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 nginx.conf diff --git a/docker-compose.yml b/docker-compose.yml index 0e0ddc6..5d5afca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,17 @@ services: DOCKER_NOTEBOOK_IMAGE: hotwa/notebook:latest # Notebook directory inside user image DOCKER_NOTEBOOK_DIR: /home/jovyan/work + # nginx: + # image: nginx:latest + # volumes: + # - ./nginx.conf:/etc/nginx/nginx.conf:ro + # ports: + # - "18188:80" + # depends_on: + # - hub + # networks: + # - jupyterhub-network + volumes: jupyterhub-data: diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000..9bc4ad3 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,26 @@ +worker_processes 1; + +events { + worker_connections 1024; +} + +http { + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 80; + + location / { + proxy_pass http://hub:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } + } +} From c51cc537a51df8aaa507e82204e2c7d9f0a44f5c Mon Sep 17 00:00:00 2001 From: hotwa Date: Mon, 15 Jan 2024 10:02:45 +0800 Subject: [PATCH 13/13] change registry.cn-hangzhou.aliyuncs.com source --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5d5afca..c00daa2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: args: JUPYTERHUB_VERSION: latest restart: always - image: hotwa/jupyterhub + image: registry.cn-hangzhou.aliyuncs.com/hotwa/jupyterhub container_name: jupyterhub networks: - jupyterhub-network @@ -32,7 +32,7 @@ services: # All containers will join this network DOCKER_NETWORK_NAME: jupyterhub-network # JupyterHub will spawn this Notebook image for users - DOCKER_NOTEBOOK_IMAGE: hotwa/notebook:latest + DOCKER_NOTEBOOK_IMAGE: registry.cn-hangzhou.aliyuncs.com/hotwa/notebook:latest # Notebook directory inside user image DOCKER_NOTEBOOK_DIR: /home/jovyan/work # nginx: