From a92e4bc77f4cd0c7023a2e59c30afc23e40946e7 Mon Sep 17 00:00:00 2001 From: hotwa Date: Thu, 22 Feb 2024 16:55:08 +0800 Subject: [PATCH 01/21] add mdgnn --- spawnerdockerfile/Dockerfile.mdgnn | 183 +++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 spawnerdockerfile/Dockerfile.mdgnn diff --git a/spawnerdockerfile/Dockerfile.mdgnn b/spawnerdockerfile/Dockerfile.mdgnn new file mode 100644 index 0000000..d9f65c9 --- /dev/null +++ b/spawnerdockerfile/Dockerfile.mdgnn @@ -0,0 +1,183 @@ +# syntax=docker/dockerfile:1 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +# https://github.com/jupyter/docker-stacks/blob/main/images/base-notebook/Dockerfile +ARG REGISTRY=quay.io +ARG OWNER=jupyter +ARG BASE_CONTAINER=$REGISTRY/$OWNER/docker-stacks-foundation +FROM $BASE_CONTAINER + +LABEL maintainer="Jupyter Project " + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# Install all OS dependencies for the Server that starts but lacks all +# features (e.g., download as all possible file formats) +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends \ + fonts-liberation \ + # - pandoc is used to convert notebooks to html files + # it's not present in the aarch64 Ubuntu image, so we install it here + pandoc \ + # - run-one - a wrapper script that runs no more + # than one unique instance of some command with a unique set of arguments, + # we use `run-one-constantly` to support the `RESTARTABLE` option + run-one && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +USER ${NB_UID} + +# Install JupyterLab, Jupyter Notebook, JupyterHub and NBClassic +# Generate a Jupyter Server config +# Cleanup temporary files +# Correct permissions +# Do all this in a single RUN command to avoid duplicating all of the +# files across image layers when the permissions change +WORKDIR /tmp +RUN mamba install --yes \ + 'jupyterlab' \ + 'notebook' \ + 'jupyterhub' \ + 'nbclassic' && \ + jupyter server --generate-config && \ + mamba clean --all -f -y && \ + npm cache clean --force && \ + jupyter lab clean && \ + rm -rf "/home/${NB_USER}/.cache/yarn" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +ENV JUPYTER_PORT=8888 +EXPOSE $JUPYTER_PORT + +# Configure container startup +CMD ["start-notebook.py"] + +# Copy local files as late as possible to avoid cache busting +COPY start-notebook.py start-notebook.sh start-singleuser.py start-singleuser.sh /usr/local/bin/ +COPY jupyter_server_config.py docker_healthcheck.py /etc/jupyter/ + +# Fix permissions on /etc/jupyter as root +USER root +RUN fix-permissions /etc/jupyter/ + +# HEALTHCHECK documentation: https://docs.docker.com/engine/reference/builder/#healthcheck +# This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server`, and `retro` jupyter commands +# https://github.com/jupyter/docker-stacks/issues/915#issuecomment-1068528799 +HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \ + CMD /etc/jupyter/docker_healthcheck.py || exit 1 + +# here add +ARG CREATE_USER="jovyan" +ARG CREATE_USER_PASSWD="password" +ARG ROOT_PASSWD="password" +ARG HOME="/home/${CREATE_USER}" +ARG DEBIAN_FRONTEND="noninteractive" +ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND} + +RUN < /etc/timezone +dpkg-reconfigure -f noninteractive tzdata +# 安装所需的软件包 +apt-get install -y python3 python3-pip gcc g++ build-essential nodejs npm gdebi-core curl wget openssh-server vim lrzsz net-tools sudo git +# 创建新用户 +useradd -m -s /bin/bash ${CREATE_USER} +echo "${CREATE_USER}:${CREATE_USER_PASSWD}" | chpasswd +echo "${CREATE_USER} ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers +# 安装micromamba +echo "1" | bash <(curl -s https://raw.githubusercontent.com/hotwa/MicroMamba_Installer/main/install.sh) +# 安装 Jupyter 和相关软件 +npm install -g configurable-http-proxy +python3 -m pip install jupyterhub jupyterlab notebook radian pycurl jupyter-rsession-proxy ipykernel jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com +EOT + +RUN < Date: Thu, 22 Feb 2024 16:56:08 +0800 Subject: [PATCH 02/21] add id_rsa github --- jupyterhub_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jupyterhub_config.py b/jupyterhub_config.py index 0c347e9..0b4b44b 100644 --- a/jupyterhub_config.py +++ b/jupyterhub_config.py @@ -35,7 +35,8 @@ c.DockerSpawner.notebook_dir = notebook_dir # notebook directory in the container c.DockerSpawner.volumes = { "jupyterhub-user-{username}": notebook_dir, - "/mnt/mydrive": "/home/jovyan/work/mydrive" + "/mnt/mydrive": "/home/jovyan/work/mydrive", + "/mnt/mydrive/project/docker-jupyterhub/id_rsa": "/home/jovyan/.ssh/id_rsa", } From 07c412fa8ba4cbf5cdffd7cab8891b816a9cd0f2 Mon Sep 17 00:00:00 2001 From: hotwa Date: Thu, 22 Feb 2024 16:58:16 +0800 Subject: [PATCH 03/21] update --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 0caca6f..344f72a 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,26 @@ cd docker-stacks/images/base-notebook docker buildx build -t hotwa/notebook:latest . -f Dockerfile ``` +## 修改jupyterhub启动notebook的路径映射 + +修改jupyterhub_config.py文件，修改卷的路径： + +```python +c.DockerSpawner.volumes = { + "jupyterhub-user-{username}": notebook_dir, + "/mnt/mydrive": "/home/jovyan/work/mydrive", + "/mnt/mydrive/project/docker-jupyterhub/id_rsa": "/home/jovyan/.ssh/id_rsa", +} +``` + +## 启动notebook修改 + +启动notebook修改docker-compose.yml文件，修改notebook的基础镜像： + +```yaml + DOCKER_NOTEBOOK_IMAGE: hotwa/notebook:latest +``` + ## Alternatives - [rocker/binder](https://rocker-project.org/images/versioned/binder.html) - From 7743e1da8e1ac34f94bc064b3b481afb6f3db36e Mon Sep 17 00:00:00 2001 From: hotwa Date: Thu, 22 Feb 2024 17:09:33 +0800 Subject: [PATCH 04/21] update --- spawnerdockerfile/Dockerfile.base-notebook | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/spawnerdockerfile/Dockerfile.base-notebook b/spawnerdockerfile/Dockerfile.base-notebook index 9b7eb37..6007e20 100644 --- a/spawnerdockerfile/Dockerfile.base-notebook +++ b/spawnerdockerfile/Dockerfile.base-notebook @@ -164,6 +164,7 @@ EOT # Install Rust RUN < Date: Thu, 22 Feb 2024 17:38:41 +0800 Subject: [PATCH 05/21] update --- spawnerdockerfile/Dockerfile.base-notebook | 1 - 1 file changed, 1 deletion(-) diff --git a/spawnerdockerfile/Dockerfile.base-notebook b/spawnerdockerfile/Dockerfile.base-notebook index 6007e20..19a55c5 100644 --- a/spawnerdockerfile/Dockerfile.base-notebook +++ b/spawnerdockerfile/Dockerfile.base-notebook @@ -141,7 +141,6 @@ ENV AUTH_KEY=$AUTH_KEY RUN curl https://get.modular.com | sh - && \ modular auth $AUTH_KEY -RUN ARG MODULAR_HOME="/home/${CREATE_USER}/.modular" ENV MODULAR_HOME=$MODULAR_HOME From 970007d84791189e6c0f8d662dcbd5cc387010a0 Mon Sep 17 00:00:00 2001 From: hotwa Date: Fri, 23 Feb 2024 09:34:53 +0800 Subject: [PATCH 06/21] updat --- spawnerdockerfile/Dockerfile.base-notebook | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spawnerdockerfile/Dockerfile.base-notebook b/spawnerdockerfile/Dockerfile.base-notebook index 19a55c5..03c50de 100644 --- a/spawnerdockerfile/Dockerfile.base-notebook +++ b/spawnerdockerfile/Dockerfile.base-notebook @@ -136,7 +136,7 @@ EOT # install mojo ENV PATH="/home/${CREATE_USER}/.local/bin:$PATH" # A random default token -ARG AUTH_KEY=5ca1ab1e +ARG AUTH_KEY="mut_efe460b898e3482bb6208bfcd4a51d7e" ENV AUTH_KEY=$AUTH_KEY RUN curl https://get.modular.com | sh - && \ @@ -162,6 +162,7 @@ micromamba run -n mixtral_env python -m ipykernel install --user --name="mixtral EOT # Install Rust +ENV PATH="/home/${CREATE_USER}/.cargo/bin:$PATH" RUN < Date: Sat, 24 Feb 2024 15:15:26 +0800 Subject: [PATCH 07/21] add --- spawnerdockerfile/Dockerfile.base | 137 ++++++++++++++++++++++++++++++ spawnerdockerfile/Dockerfile.plot | 117 +++++++++++++++++++++++++ 2 files changed, 254 insertions(+) create mode 100644 spawnerdockerfile/Dockerfile.base create mode 100644 spawnerdockerfile/Dockerfile.plot diff --git a/spawnerdockerfile/Dockerfile.base b/spawnerdockerfile/Dockerfile.base new file mode 100644 index 0000000..3af89ab --- /dev/null +++ b/spawnerdockerfile/Dockerfile.base @@ -0,0 +1,137 @@ +# syntax=docker/dockerfile:1 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +# https://github.com/jupyter/docker-stacks/blob/main/images/base-notebook/Dockerfile +ARG REGISTRY=quay.io +ARG OWNER=jupyter +ARG BASE_CONTAINER=$REGISTRY/$OWNER/docker-stacks-foundation +FROM $BASE_CONTAINER + +LABEL maintainer="Jupyter Project " + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# Install all OS dependencies for the Server that starts but lacks all +# features (e.g., download as all possible file formats) +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends \ + fonts-liberation \ + # - pandoc is used to convert notebooks to html files + # it's not present in the aarch64 Ubuntu image, so we install it here + pandoc \ + # - run-one - a wrapper script that runs no more + # than one unique instance of some command with a unique set of arguments, + # we use `run-one-constantly` to support the `RESTARTABLE` option + run-one && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +USER ${NB_UID} + +# Install JupyterLab, Jupyter Notebook, JupyterHub and NBClassic +# Generate a Jupyter Server config +# Cleanup temporary files +# Correct permissions +# Do all this in a single RUN command to avoid duplicating all of the +# files across image layers when the permissions change +WORKDIR /tmp +RUN mamba install --yes \ + 'jupyterlab' \ + 'notebook' \ + 'jupyterhub' \ + 'nbclassic' && \ + jupyter server --generate-config && \ + mamba clean --all -f -y && \ + npm cache clean --force && \ + jupyter lab clean && \ + rm -rf "/home/${NB_USER}/.cache/yarn" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +ENV JUPYTER_PORT=8888 +EXPOSE $JUPYTER_PORT + +# Configure container startup +CMD ["start-notebook.py"] + +# Copy local files as late as possible to avoid cache busting +COPY start-notebook.py start-notebook.sh start-singleuser.py start-singleuser.sh /usr/local/bin/ +COPY jupyter_server_config.py docker_healthcheck.py /etc/jupyter/ + +# Fix permissions on /etc/jupyter as root +USER root +RUN fix-permissions /etc/jupyter/ + +# HEALTHCHECK documentation: https://docs.docker.com/engine/reference/builder/#healthcheck +# This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server`, and `retro` jupyter commands +# https://github.com/jupyter/docker-stacks/issues/915#issuecomment-1068528799 +HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \ + CMD /etc/jupyter/docker_healthcheck.py || exit 1 + +# here add +ARG CREATE_USER="jovyan" +ARG CREATE_USER_PASSWD="password" +ARG ROOT_PASSWD="password" +ARG HOME="/home/${CREATE_USER}" +ARG DEBIAN_FRONTEND="noninteractive" +ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND} + +RUN < /etc/timezone +dpkg-reconfigure -f noninteractive tzdata +# 安装所需的软件包 +apt-get install -y python3 python3-pip gcc g++ build-essential nodejs npm gdebi-core curl wget openssh-server vim lrzsz net-tools sudo git +# 创建新用户 +useradd -m -s /bin/bash ${CREATE_USER} +echo "${CREATE_USER}:${CREATE_USER_PASSWD}" | chpasswd +echo "${CREATE_USER} ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers +# 安装micromamba +echo "1" | bash <(curl -s https://cdn.jsdelivr.net/gh/hotwa/MicroMamba_Installer@main/install.sh) +# 安装 Jupyter 和相关软件 +npm install -g configurable-http-proxy +python3 -m pip install jupyterhub jupyterlab notebook radian pycurl jupyter-rsession-proxy ipykernel>=6.25.0 jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com +EOT + +# Switch back to jovyan to avoid accidental container runs as root +USER ${NB_UID} +ENV MAMBA_ROOT_PREFIX=/home/${CREATE_USER}/micromamba + +# install mojo +ENV PATH="/home/${CREATE_USER}/.local/bin:$PATH" +# A random default token +ARG AUTH_KEY="mut_efe460b898e3482bb6208bfcd4a51d7e" +ENV AUTH_KEY=$AUTH_KEY + +RUN curl https://get.modular.com | sh - && \ + modular auth $AUTH_KEY + +ARG MODULAR_HOME="/home/${CREATE_USER}/.modular" +ENV MODULAR_HOME=$MODULAR_HOME +ENV PATH="$PATH:$MODULAR_HOME/pkg/packages.modular.com_mojo/bin" +RUN < /etc/timezone +dpkg-reconfigure -f noninteractive tzdata +# 安装所需的软件包 +apt-get install -y python3 python3-pip gcc g++ build-essential nodejs npm gdebi-core curl wget openssh-server vim lrzsz net-tools sudo git +# 创建新用户 +useradd -m -s /bin/bash ${CREATE_USER} +echo "${CREATE_USER}:${CREATE_USER_PASSWD}" | chpasswd +echo "${CREATE_USER} ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers +# 安装micromamba +echo "1" | bash <(curl -s https://cdn.jsdelivr.net/gh/hotwa/MicroMamba_Installer@main/install.sh) +# 安装 Jupyter 和相关软件 +npm install -g configurable-http-proxy +python3 -m pip install jupyterhub jupyterlab notebook radian pycurl jupyter-rsession-proxy ipykernel>=6.25.0 jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com +EOT + +# Switch back to jovyan to avoid accidental container runs as root +USER ${NB_UID} +ENV MAMBA_ROOT_PREFIX=/home/${CREATE_USER}/micromamba +RUN < Date: Sat, 24 Feb 2024 17:37:33 +0800 Subject: [PATCH 08/21] update --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7b0a54d..8e2defa 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -/docker-stacks \ No newline at end of file +/docker-stacks +*.tar \ No newline at end of file From 57ee612b16634b06b8534b1cbd2b0a78012c2779 Mon Sep 17 00:00:00 2001 From: hotwa Date: Sat, 24 Feb 2024 17:38:00 +0800 Subject: [PATCH 09/21] update install micromamba shell script --- spawnerdockerfile/Dockerfile.base | 5 +- spawnerdockerfile/Dockerfile.base-notebook | 32 +-- spawnerdockerfile/Dockerfile.mdgnn | 74 +------ spawnerdockerfile/Dockerfile.plot | 5 +- spawnerdockerfile/install.sh | 229 +++++++++++++++++++++ 5 files changed, 248 insertions(+), 97 deletions(-) create mode 100755 spawnerdockerfile/install.sh diff --git a/spawnerdockerfile/Dockerfile.base b/spawnerdockerfile/Dockerfile.base index 3af89ab..c913fae 100644 --- a/spawnerdockerfile/Dockerfile.base +++ b/spawnerdockerfile/Dockerfile.base @@ -79,6 +79,7 @@ ARG HOME="/home/${CREATE_USER}" ARG DEBIAN_FRONTEND="noninteractive" ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND} +COPY install.sh /tmp RUN <> /etc/sudoers # 安装micromamba -echo "1" | bash <(curl -s https://cdn.jsdelivr.net/gh/hotwa/MicroMamba_Installer@main/install.sh) +echo "1" | bash /tmp/install.sh +mkdir -p /home/${CREATE_USER}/micromamba +chown -R ${NB_UID}:${NB_GID} /home/${CREATE_USER}/micromamba # 安装 Jupyter 和相关软件 npm install -g configurable-http-proxy python3 -m pip install jupyterhub jupyterlab notebook radian pycurl jupyter-rsession-proxy ipykernel>=6.25.0 jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com diff --git a/spawnerdockerfile/Dockerfile.base-notebook b/spawnerdockerfile/Dockerfile.base-notebook index 03c50de..365585b 100644 --- a/spawnerdockerfile/Dockerfile.base-notebook +++ b/spawnerdockerfile/Dockerfile.base-notebook @@ -79,6 +79,7 @@ ARG HOME="/home/${CREATE_USER}" ARG DEBIAN_FRONTEND="noninteractive" ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND} +COPY install.sh /tmp RUN <> /etc/sudoers # 安装micromamba -echo 1 | bash <(curl -s https://cdn.jsdelivr.net/gh/hotwa/MicroMamba_Installer@main/install.sh) +echo "1" | bash /tmp/install.sh +mkdir -p /home/${CREATE_USER}/micromamba +chown -R ${NB_UID}:${NB_GID} /home/${CREATE_USER}/micromamba # 安装 Jupyter 和相关软件 npm install -g configurable-http-proxy -python3 -m pip install jupyterhub jupyterlab notebook radian pycurl jupyter-rsession-proxy ipykernel jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com +python3 -m pip install jupyterhub jupyterlab notebook radian pycurl jupyter-rsession-proxy ipykernel>=6.25.0 jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com EOT RUN <> /etc/sudoers # 安装micromamba -echo "1" | bash <(curl -s https://raw.githubusercontent.com/hotwa/MicroMamba_Installer/main/install.sh) +echo "1" | bash /tmp/install.sh +mkdir -p /home/${CREATE_USER}/micromamba +chown -R ${NB_UID}:${NB_GID} /home/${CREATE_USER}/micromamba # 安装 Jupyter 和相关软件 npm install -g configurable-http-proxy -python3 -m pip install jupyterhub jupyterlab notebook radian pycurl jupyter-rsession-proxy ipykernel jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -EOT - -RUN <=6.25.0 jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com EOT # Switch back to jovyan to avoid accidental container runs as root USER ${NB_UID} ENV MAMBA_ROOT_PREFIX=/home/${CREATE_USER}/micromamba -RUN <> /etc/sudoers # 安装micromamba -echo "1" | bash <(curl -s https://cdn.jsdelivr.net/gh/hotwa/MicroMamba_Installer@main/install.sh) +echo "1" | bash /tmp/install.sh +mkdir -p /home/${CREATE_USER}/micromamba +chown -R ${NB_UID}:${NB_GID} /home/${CREATE_USER}/micromamba # 安装 Jupyter 和相关软件 npm install -g configurable-http-proxy python3 -m pip install jupyterhub jupyterlab notebook radian pycurl jupyter-rsession-proxy ipykernel>=6.25.0 jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com diff --git a/spawnerdockerfile/install.sh b/spawnerdockerfile/install.sh new file mode 100755 index 0000000..0c63e86 --- /dev/null +++ b/spawnerdockerfile/install.sh @@ -0,0 +1,229 @@ +#!/bin/bash + +# 安装所需依赖 +install_dependencies() { + # 检查并安装 jq, curl, tar, bzip2 + for dep in jq curl tar bzip2; do + if ! command -v $dep >/dev/null 2>&1; then + sudo apt-get update && sudo apt-get install -y $dep + fi + done +} + +# 在脚本开始时调用依赖安装函数 +install_dependencies + +# 默认的MAMBA_ROOT_PREFIX路径 +DEFAULT_MAMBA_ROOT_PREFIX="/usr/local/bin" + +# 显示用法信息 +usage() { + echo "用法: sudo $0 [--silent-install [custom_root_prefix]] [--silent-uninstall] [--silent-update]" + echo " --silent-install 静默模式安装，不进行任何交互" + echo " custom_root_prefix (可选) 在静默模式下设置 MAMBA_ROOT_PREFIX 的路径" + echo " --silent-uninstall 静默模式卸载，不进行任何交互" + echo " --silent-update 静默模式更新，不进行任何交互" + exit 1 +} + +# 安装micromamba +install_micromamba() { + local mamba_prefix="$1" + + # 自动检测操作系统和处理器架构 + OS="$(uname)" + ARCH="$(uname -m)" + + # 根据操作系统和架构设置下载链接 + case "$OS" in + "Linux") + case "$ARCH" in + "x86_64") URL="https://micro.mamba.pm/api/micromamba/linux-64/latest" ;; + "aarch64") URL="https://micro.mamba.pm/api/micromamba/linux-aarch64/latest" ;; + "ppc64le") URL="https://micro.mamba.pm/api/micromamba/linux-ppc64le/latest" ;; + *) echo "不支持的架构: $ARCH"; exit 1 ;; + esac + ;; + "Darwin") + case "$ARCH" in + "x86_64") URL="https://micro.mamba.pm/api/micromamba/osx-64/latest" ;; + "arm64") URL="https://micro.mamba.pm/api/micromamba/osx-arm64/latest" ;; + *) echo "不支持的架构: $ARCH"; exit 1 ;; + esac + ;; + *) + echo "不支持的操作系统: $OS"; exit 1 ;; + esac + + # 下载并解压 micromamba 到指定的路径 + echo "正在下载 micromamba..." + if curl -Ls "$URL" | tar -xvj -C "$mamba_prefix"; then + echo "micromamba 下载并解压完成。" + ls -l /usr/local/bin/bin/micromamba + echo "micromamba 赋予执行权限" + chmod +x /usr/local/bin/bin/micromamba + echo "micromamba 安装完成" + echo "MAMBA_ROOT_PREFIX 路径：$mamba_prefix" + # 初始化 shell 环境 + echo "正在为root初始化 micromamba shell 环境..." + "$mamba_prefix/bin/micromamba" shell init -s bash -p "~/micromamba" + ln -s /usr/local/bin/bin/micromamba /usr/local/bin/micromamba + else + echo "micromamba 下载失败" + fi +} + +# 清理 .bashrc 文件 +cleanup_bashrc() { + local mamba_bin_path="$1" + + # 删除 .bashrc 中相关的行 + sed -i '/MICROMAMBA_BIN_PATH/d' ~/.bashrc + sed -i '/MAMBA_ROOT_PREFIX/d' ~/.bashrc + sed -i '/alias mba=/d' ~/.bashrc +} + +# 删除micromamba +uninstall_micromamba() { + + echo "正在卸载 micromamba..." + rm -rf /usr/local/bin/micromamba /usr/local/bin/bin /usr/local/bin/info /usr/local/bin/etc + + # 清理 .bashrc 文件 + cleanup_bashrc "$mamba_prefix" + + echo "micromamba 已卸载。" +} + + +# 获取当前 micromamba 版本 +get_current_version() { + local mamba_path="$1/micromamba" + if [ -f "$mamba_path" ]; then + local current_version + current_version=$("$mamba_path" --version | grep -o 'version [^ ]*' | cut -d ' ' -f2 | cut -d '-' -f1) + echo "$current_version" + else + echo "未安装" + fi +} + +# 获取最新的 micromamba 版本 +get_latest_version() { + local latest_version + latest_version=$(curl -s https://api.github.com/repos/mamba-org/micromamba-releases/releases/latest | jq -r '.tag_name' | cut -d '-' -f1) + echo "$latest_version" +} + +# 检查是否有更新并提示用户 +check_for_updates() { + local current_version latest_version + current_version=$(get_current_version "$1") + latest_version=$(get_latest_version) + + if [ "$latest_version" != "$current_version" ]; then + echo "有可用的更新：当前版本 $current_version, 最新版本 $latest_version" + read -p "是否更新到最新版本? (y/N): " confirm + if [[ "$confirm" =~ ^[Yy]$ ]]; then + update_micromamba "$1" + else + echo "更新已取消" + fi + else + echo "当前已是最新版本 ($current_version)" + fi +} + +# 更新micromamba +update_micromamba() { + local mamba_prefix="$1" + local current_version latest_version + current_version=$(get_current_version "$mamba_prefix") + latest_version=$(get_latest_version) + + if [ "$latest_version" != "$current_version" ]; then + echo "有可用的更新：当前版本 $current_version, 最新版本 $latest_version" + read -p "是否更新到最新版本? (y/N): " confirm + if [[ "$confirm" =~ ^[Yy]$ ]]; then + echo "正在卸载旧版本 micromamba..." + uninstall_micromamba "$mamba_prefix" + echo "正在安装最新版本 micromamba..." + install_micromamba "$mamba_prefix" + else + echo "更新已取消" + fi + else + echo "当前已是最新版本 ($current_version)" + fi +} + + +# 交互式安装、卸载和更新菜单 +interactive_menu() { + local current_version latest_version + current_version=$(get_current_version "$DEFAULT_MAMBA_ROOT_PREFIX") + latest_version=$(get_latest_version) + + # 检测并显示当前和最新版本 + echo "当前 micromamba 版本: $current_version" + echo "最新 micromamba 版本: $latest_version" + if [ "$latest_version" != "$current_version" ]; then + echo "有可用的更新。" + fi + + echo "选择操作：" + echo "1) 安装 micromamba" + echo "2) 删除 micromamba" + echo "3) 更新 micromamba" + echo "q) 退出" + read -p "请输入选项（1、2、3或q）: " main_choice + + case "$main_choice" in + 1) + echo "MAMBA_ROOT_PREFIX 的路径: $DEFAULT_MAMBA_ROOT_PREFIX: " + install_micromamba "$DEFAULT_MAMBA_ROOT_PREFIX" + ;; + 2) + uninstall_micromamba "$DEFAULT_MAMBA_ROOT_PREFIX" + ;; + 3) + update_micromamba "$DEFAULT_MAMBA_ROOT_PREFIX" + ;; + q) + echo "退出。" + exit 0 + ;; + *) + echo "无效的选项。退出。" + exit 1 + ;; + esac +} + +# 静默卸载 +silent_uninstall() { + # 在卸载之前保留虚拟环境目录 + local envs_dir="$DEFAULT_MAMBA_ROOT_PREFIX/envs" + mkdir -p "$envs_dir" + mv "$DEFAULT_MAMBA_ROOT_PREFIX/envs" "$(mktemp -d)" + uninstall_micromamba "$DEFAULT_MAMBA_ROOT_PREFIX" + mv "$(mktemp -d)/envs" "$envs_dir" +} + +# 静默更新 +silent_update() { + update_micromamba "$DEFAULT_MAMBA_ROOT_PREFIX" +} + +# 解析命令行参数 +if [ "$1" = "--silent-install" ]; then + silent_install "$2" +elif [ "$1" = "--silent-uninstall" ]; then + silent_uninstall +elif [ "$1" = "--silent-update" ]; then + silent_update +elif [ "$1" = "--help" ]; then + usage +else + interactive_menu +fi From 938cf1b69b0cab5fa6d19135c20352161deaccbe Mon Sep 17 00:00:00 2001 From: hotwa Date: Sat, 24 Feb 2024 17:58:13 +0800 Subject: [PATCH 10/21] rserver --- spawnerdockerfile/Dockerfile.rserver | 133 +++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 spawnerdockerfile/Dockerfile.rserver diff --git a/spawnerdockerfile/Dockerfile.rserver b/spawnerdockerfile/Dockerfile.rserver new file mode 100644 index 0000000..037ce3f --- /dev/null +++ b/spawnerdockerfile/Dockerfile.rserver @@ -0,0 +1,133 @@ +# syntax=docker/dockerfile:1 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +# https://github.com/jupyter/docker-stacks/blob/main/images/base-notebook/Dockerfile +ARG REGISTRY=quay.io +ARG OWNER=jupyter +ARG BASE_CONTAINER=$REGISTRY/$OWNER/docker-stacks-foundation +FROM $BASE_CONTAINER + +LABEL maintainer="Jupyter Project " + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# Install all OS dependencies for the Server that starts but lacks all +# features (e.g., download as all possible file formats) +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends \ + fonts-liberation \ + # - pandoc is used to convert notebooks to html files + # it's not present in the aarch64 Ubuntu image, so we install it here + pandoc \ + # - run-one - a wrapper script that runs no more + # than one unique instance of some command with a unique set of arguments, + # we use `run-one-constantly` to support the `RESTARTABLE` option + run-one && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +USER ${NB_UID} + +# Install JupyterLab, Jupyter Notebook, JupyterHub and NBClassic +# Generate a Jupyter Server config +# Cleanup temporary files +# Correct permissions +# Do all this in a single RUN command to avoid duplicating all of the +# files across image layers when the permissions change +WORKDIR /tmp +RUN mamba install --yes \ + 'jupyterlab' \ + 'notebook' \ + 'jupyterhub' \ + 'nbclassic' && \ + jupyter server --generate-config && \ + mamba clean --all -f -y && \ + npm cache clean --force && \ + jupyter lab clean && \ + rm -rf "/home/${NB_USER}/.cache/yarn" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +ENV JUPYTER_PORT=8888 +EXPOSE $JUPYTER_PORT + +# Configure container startup +CMD ["start-notebook.py"] + +# Copy local files as late as possible to avoid cache busting +COPY start-notebook.py start-notebook.sh start-singleuser.py start-singleuser.sh /usr/local/bin/ +COPY jupyter_server_config.py docker_healthcheck.py /etc/jupyter/ + +# Fix permissions on /etc/jupyter as root +USER root +RUN fix-permissions /etc/jupyter/ + +# HEALTHCHECK documentation: https://docs.docker.com/engine/reference/builder/#healthcheck +# This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server`, and `retro` jupyter commands +# https://github.com/jupyter/docker-stacks/issues/915#issuecomment-1068528799 +HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \ + CMD /etc/jupyter/docker_healthcheck.py || exit 1 + +# here add +ARG CREATE_USER="jovyan" +ARG CREATE_USER_PASSWD="password" +ARG ROOT_PASSWD="password" +ARG HOME="/home/${CREATE_USER}" +ARG DEBIAN_FRONTEND="noninteractive" +ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND} + +COPY install.sh /tmp +RUN < /etc/timezone +dpkg-reconfigure -f noninteractive tzdata +# 安装所需的软件包 +apt-get install -y python3 python3-pip gcc g++ build-essential nodejs npm gdebi-core curl wget openssh-server vim lrzsz net-tools sudo git +# 创建新用户 +useradd -m -s /bin/bash ${CREATE_USER} +echo "${CREATE_USER}:${CREATE_USER_PASSWD}" | chpasswd +echo "${CREATE_USER} ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers +# 安装micromamba +echo "1" | bash /tmp/install.sh +mkdir -p /home/${CREATE_USER}/micromamba +chown -R ${NB_UID}:${NB_GID} /home/${CREATE_USER}/micromamba +# 安装 Jupyter 和相关软件 +npm install -g configurable-http-proxy +python3 -m pip install jupyterhub jupyterlab notebook radian pycurl jupyter-rsession-proxy ipykernel>=6.25.0 jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com +EOT + +ARG RSERVER_VERSION="rstudio-server-2023.12.1-402-amd64.deb" +ENV RSERVER_VERSION=${RSERVER_VERSION} +RUN < Date: Fri, 24 May 2024 17:02:27 +0800 Subject: [PATCH 11/21] add pytorch-notebook construct method --- spawnerdockerfile/README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) mode change 100644 => 100755 spawnerdockerfile/README.md diff --git a/spawnerdockerfile/README.md b/spawnerdockerfile/README.md old mode 100644 new mode 100755 index 5563db8..67eb2cc --- a/spawnerdockerfile/README.md +++ b/spawnerdockerfile/README.md @@ -52,3 +52,34 @@ source ~/.bashrc 这个操作会激活micromamba(mamba, conda)的安装目录`etc/profile.d/micromamba.sh`激活添加到bash初始化文件`.bashrc`里面 +# docker镜像 + +关于这个jupyterlab的spawner的notebook启动的镜像来源于：quay.io/jupyterdocker-stacks-foundation + +这个镜像的构建仓库是：https://github.com/jupyter/docker-stacks/blob/main/images/docker-stacks-foundation/Dockerfile + +可以把这个Dockerfile的ARG参数ROOT_CONTAINER修改为docker pull nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04 + +pytorch-notebook构建顺序是： + +docker-stacks-foundation +base-notebook +minimal-notebook +scipy-notebook +pytorch-notebook + +第一步 + +```shell +git clone https://github.com/jupyter/docker-stacks.git +cd docker-stacks/images/docker-stacks-foundation +docker buildx build --build-arg ROOT_CONTAINER=nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04 -t quay.io/hotwa/docker-stacks-foundation:latest . --load # docker pull nvidia/cuda:12.4.1-devel-ubuntu22.04 +cd ../base-notebook +docker buildx build --build-arg OWNER=hotwa -t quay.io/hotwa/base-notebook:latest . --load +cd ../minimal-notebook/ +docker buildx build --build-arg OWNER=hotwa -t quay.io/hotwa/minimal-notebook:latest . --load +cd ../scipy-notebook +docker buildx build --build-arg OWNER=hotwa -t quay.io/hotwa/scipy-notebook:latest . --load +cd ../pytorch-notebook +docker buildx build --build-arg OWNER=hotwa -t quay.io/hotwa/pytorch-notebook:latest . --load +``` From 6adc5144c783d0be39224f769097fbccdf43b09a Mon Sep 17 00:00:00 2001 From: hotwa Date: Sat, 25 May 2024 09:47:59 +0800 Subject: [PATCH 12/21] update rserver version --- spawnerdockerfile/Dockerfile.rserver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 spawnerdockerfile/Dockerfile.rserver diff --git a/spawnerdockerfile/Dockerfile.rserver b/spawnerdockerfile/Dockerfile.rserver old mode 100644 new mode 100755 index 037ce3f..fadeba9 --- a/spawnerdockerfile/Dockerfile.rserver +++ b/spawnerdockerfile/Dockerfile.rserver @@ -104,7 +104,7 @@ npm install -g configurable-http-proxy python3 -m pip install jupyterhub jupyterlab notebook radian pycurl jupyter-rsession-proxy ipykernel>=6.25.0 jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com EOT -ARG RSERVER_VERSION="rstudio-server-2023.12.1-402-amd64.deb" +ARG RSERVER_VERSION="rstudio-server-2024.04.1-748-amd64.deb" ENV RSERVER_VERSION=${RSERVER_VERSION} RUN < Date: Sat, 25 May 2024 11:03:27 +0800 Subject: [PATCH 13/21] remove micromamba and add new way to install mojo --- spawnerdockerfile/Dockerfile.base | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) mode change 100644 => 100755 spawnerdockerfile/Dockerfile.base diff --git a/spawnerdockerfile/Dockerfile.base b/spawnerdockerfile/Dockerfile.base old mode 100644 new mode 100755 index c913fae..6581bb2 --- a/spawnerdockerfile/Dockerfile.base +++ b/spawnerdockerfile/Dockerfile.base @@ -79,7 +79,6 @@ ARG HOME="/home/${CREATE_USER}" ARG DEBIAN_FRONTEND="noninteractive" ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND} -COPY install.sh /tmp RUN <> /etc/sudoers -# 安装micromamba -echo "1" | bash /tmp/install.sh -mkdir -p /home/${CREATE_USER}/micromamba -chown -R ${NB_UID}:${NB_GID} /home/${CREATE_USER}/micromamba # 安装 Jupyter 和相关软件 npm install -g configurable-http-proxy python3 -m pip install jupyterhub jupyterlab notebook radian pycurl jupyter-rsession-proxy ipykernel>=6.25.0 jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com @@ -106,25 +101,22 @@ EOT # Switch back to jovyan to avoid accidental container runs as root USER ${NB_UID} -ENV MAMBA_ROOT_PREFIX=/home/${CREATE_USER}/micromamba # install mojo -ENV PATH="/home/${CREATE_USER}/.local/bin:$PATH" -# A random default token -ARG AUTH_KEY="mut_efe460b898e3482bb6208bfcd4a51d7e" -ENV AUTH_KEY=$AUTH_KEY - -RUN curl https://get.modular.com | sh - && \ - modular auth $AUTH_KEY - ARG MODULAR_HOME="/home/${CREATE_USER}/.modular" ENV MODULAR_HOME=$MODULAR_HOME -ENV PATH="$PATH:$MODULAR_HOME/pkg/packages.modular.com_mojo/bin" +ENV PATH="$MODULAR_HOME/pkg/packages.modular.com_mojo/bin:/home/${CREATE_USER}/.local/bin:$PATH" RUN <> "$BASHRC" +echo 'export PATH="'$MOJO_PATH'/bin:$PATH"' >> "$BASHRC" +source "$BASHRC" EOT # Install Rust From 3b3ee98f2d4eabf380325b4af66b59275d2a122c Mon Sep 17 00:00:00 2001 From: hotwa Date: Sat, 25 May 2024 11:10:45 +0800 Subject: [PATCH 14/21] change to mojo --- spawnerdockerfile/Dockerfile.base-notebook | 41 ++++--- spawnerdockerfile/Dockerfile.mdgnn | 119 --------------------- 2 files changed, 20 insertions(+), 140 deletions(-) mode change 100644 => 100755 spawnerdockerfile/Dockerfile.base-notebook delete mode 100644 spawnerdockerfile/Dockerfile.mdgnn diff --git a/spawnerdockerfile/Dockerfile.base-notebook b/spawnerdockerfile/Dockerfile.base-notebook old mode 100644 new mode 100755 index 365585b..9ac8703 --- a/spawnerdockerfile/Dockerfile.base-notebook +++ b/spawnerdockerfile/Dockerfile.base-notebook @@ -90,20 +90,27 @@ ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime echo 'Asia/Shanghai' > /etc/timezone dpkg-reconfigure -f noninteractive tzdata # 安装所需的软件包 -apt-get install -y python3 python3-pip gcc g++ build-essential nodejs npm gdebi-core curl wget openssh-server vim lrzsz net-tools sudo git +apt-get install -y python3 python3-pip gcc g++ build-essential nodejs npm gdebi-core curl wget openssh-server vim lrzsz net-tools sudo git nodejs # 创建新用户 useradd -m -s /bin/bash ${CREATE_USER} echo "${CREATE_USER}:${CREATE_USER_PASSWD}" | chpasswd echo "${CREATE_USER} ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers -# 安装micromamba -echo "1" | bash /tmp/install.sh -mkdir -p /home/${CREATE_USER}/micromamba -chown -R ${NB_UID}:${NB_GID} /home/${CREATE_USER}/micromamba -# 安装 Jupyter 和相关软件 -npm install -g configurable-http-proxy -python3 -m pip install jupyterhub jupyterlab notebook radian pycurl jupyter-rsession-proxy ipykernel>=6.25.0 jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com EOT +# 安装 Jupyter 和相关软件 +RUN <=6.25.0 jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com +EOT + +ARG RSERVER_VERSION="rstudio-server-2024.04.1-748-amd64.deb" +ENV RSERVER_VERSION=${RSERVER_VERSION} RUN < /etc/timezone -dpkg-reconfigure -f noninteractive tzdata -# 安装所需的软件包 -apt-get install -y python3 python3-pip gcc g++ build-essential nodejs npm gdebi-core curl wget openssh-server vim lrzsz net-tools sudo git -# 创建新用户 -useradd -m -s /bin/bash ${CREATE_USER} -echo "${CREATE_USER}:${CREATE_USER_PASSWD}" | chpasswd -echo "${CREATE_USER} ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers -# 安装micromamba -echo "1" | bash /tmp/install.sh -mkdir -p /home/${CREATE_USER}/micromamba -chown -R ${NB_UID}:${NB_GID} /home/${CREATE_USER}/micromamba -# 安装 Jupyter 和相关软件 -npm install -g configurable-http-proxy -python3 -m pip install jupyterhub jupyterlab notebook radian pycurl jupyter-rsession-proxy ipykernel>=6.25.0 jupyterlab-language-pack-zh-CN jupyterlab-git jupyterlab-system-monitor jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyterlab_widgets jupyterlab-drawio jupyterlab-spreadsheet-editor jupyterlab-cell-flash jedi-language-server jupyterlab_code_formatter jupyterlab-spellchecker jupyterlab_vim nbresuse ipydrawio jedi ipympl black isort theme-darcula ipywidgets tensorboard jupyterlab_latex jupyter_bokeh autopep8 xeus-python jupyterlab-lsp python-lsp-server nglview dockerspawner jupyterhub-nativeauthenticator lckr_jupyterlab_variableinspector -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -EOT - -# Switch back to jovyan to avoid accidental container runs as root -USER ${NB_UID} -ENV MAMBA_ROOT_PREFIX=/home/${CREATE_USER}/micromamba -RUN < Date: Sat, 25 May 2024 11:10:58 +0800 Subject: [PATCH 15/21] change to mojo --- spawnerdockerfile/Dockerfile.base | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/spawnerdockerfile/Dockerfile.base b/spawnerdockerfile/Dockerfile.base index 6581bb2..5929256 100755 --- a/spawnerdockerfile/Dockerfile.base +++ b/spawnerdockerfile/Dockerfile.base @@ -109,14 +109,9 @@ ENV PATH="$MODULAR_HOME/pkg/packages.modular.com_mojo/bin:/home/${CREATE_USER}/. RUN <> "$BASHRC" -echo 'export PATH="'$MOJO_PATH'/bin:$PATH"' >> "$BASHRC" -source "$BASHRC" EOT # Install Rust From 0af46b7c0c2bb293a7835528a64a1e4537b1a44a Mon Sep 17 00:00:00 2001 From: hotwa Date: Sat, 25 May 2024 11:11:42 +0800 Subject: [PATCH 16/21] update --- spawnerdockerfile/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/spawnerdockerfile/README.md b/spawnerdockerfile/README.md index 67eb2cc..d8576ff 100755 --- a/spawnerdockerfile/README.md +++ b/spawnerdockerfile/README.md @@ -83,3 +83,11 @@ docker buildx build --build-arg OWNER=hotwa -t quay.io/hotwa/scipy-notebook:late cd ../pytorch-notebook docker buildx build --build-arg OWNER=hotwa -t quay.io/hotwa/pytorch-notebook:latest . --load ``` + +# 然后构建自己的基础镜像 + +```shell +docker buildx build --build-arg OWNER=hotwa -t quay.io/hotwa/notebook:latest -f Dockerfile.base-notebook . --load +# 导出保存 + +``` From a162b38603a78f5ecd31879df6e9a8d49e8ddfb2 Mon Sep 17 00:00:00 2001 From: hotwa Date: Sat, 25 May 2024 11:18:16 +0800 Subject: [PATCH 17/21] all update --- .gitignore | 0 Dockerfile | 0 Dockerfile.jupyterhub | 0 Dockerfile.jupyterhub.bak | 0 README.md | 0 dhparam.pem | 0 docker-compose.yml | 0 id_rsa | 0 jupyterhub_config.py | 5 +++-- jupyterhub_config.py.bak | 0 micromamba_install.sh | 0 nginx-selfsigned.crt | 0 nginx-selfsigned.key | 0 nginx.conf | 0 spawnerdockerfile/Dockerfile | 0 spawnerdockerfile/Dockerfile.plot | 0 spawnerdockerfile/docker-compose.yml | 17 +++++++++++++++++ spawnerdockerfile/jupyter_server_config.py | 0 18 files changed, 20 insertions(+), 2 deletions(-) mode change 100644 => 100755 .gitignore mode change 100644 => 100755 Dockerfile mode change 100644 => 100755 Dockerfile.jupyterhub mode change 100644 => 100755 Dockerfile.jupyterhub.bak mode change 100644 => 100755 README.md mode change 100644 => 100755 dhparam.pem mode change 100644 => 100755 docker-compose.yml mode change 100644 => 100755 id_rsa mode change 100644 => 100755 jupyterhub_config.py mode change 100644 => 100755 jupyterhub_config.py.bak mode change 100644 => 100755 micromamba_install.sh mode change 100644 => 100755 nginx-selfsigned.crt mode change 100644 => 100755 nginx-selfsigned.key mode change 100644 => 100755 nginx.conf mode change 100644 => 100755 spawnerdockerfile/Dockerfile mode change 100644 => 100755 spawnerdockerfile/Dockerfile.plot create mode 100755 spawnerdockerfile/docker-compose.yml mode change 100644 => 100755 spawnerdockerfile/jupyter_server_config.py diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/Dockerfile b/Dockerfile old mode 100644 new mode 100755 diff --git a/Dockerfile.jupyterhub b/Dockerfile.jupyterhub old mode 100644 new mode 100755 diff --git a/Dockerfile.jupyterhub.bak b/Dockerfile.jupyterhub.bak old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/dhparam.pem b/dhparam.pem old mode 100644 new mode 100755 diff --git a/docker-compose.yml b/docker-compose.yml old mode 100644 new mode 100755 diff --git a/id_rsa b/id_rsa old mode 100644 new mode 100755 diff --git a/jupyterhub_config.py b/jupyterhub_config.py old mode 100644 new mode 100755 index 0b4b44b..83f17d8 --- a/jupyterhub_config.py +++ b/jupyterhub_config.py @@ -70,9 +70,10 @@ if admin: # "environment": {"JUPYTER_ENABLE_LAB": "yes"} # }) -# 启动jupyter时候增加跨域支持 +# 启动jupyter时候增加跨域支持, 否则反向代理的时候出现问题 +# --NotebookApp.iopub_data_rate_limit=10000000 给nglview使用 c.DockerSpawner.extra_create_kwargs.update({ - "environment": {"NOTEBOOK_ARGS": "--NotebookApp.allow_origin='*'"} + "environment": {"NOTEBOOK_ARGS": "--NotebookApp.allow_origin='*' --NotebookApp.iopub_data_rate_limit=10000000"} }) diff --git a/jupyterhub_config.py.bak b/jupyterhub_config.py.bak old mode 100644 new mode 100755 diff --git a/micromamba_install.sh b/micromamba_install.sh old mode 100644 new mode 100755 diff --git a/nginx-selfsigned.crt b/nginx-selfsigned.crt old mode 100644 new mode 100755 diff --git a/nginx-selfsigned.key b/nginx-selfsigned.key old mode 100644 new mode 100755 diff --git a/nginx.conf b/nginx.conf old mode 100644 new mode 100755 diff --git a/spawnerdockerfile/Dockerfile b/spawnerdockerfile/Dockerfile old mode 100644 new mode 100755 diff --git a/spawnerdockerfile/Dockerfile.plot b/spawnerdockerfile/Dockerfile.plot old mode 100644 new mode 100755 diff --git a/spawnerdockerfile/docker-compose.yml b/spawnerdockerfile/docker-compose.yml new file mode 100755 index 0000000..14b1c6b --- /dev/null +++ b/spawnerdockerfile/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3.8' +services: + molscribe: + build: + context: . + dockerfile: Dockerfile.base-notebook + image: hotwa/notebook:latest + container_name: notebook + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + # command: micromamba run -n molscribe jupyter notebook --ip=0.0.0.0 --NotebookApp.allow_origin='*' --debug --no-browser --NotebookApp.disable_check_xsrf=True --port=8888 --allow-root --NotebookApp.token='' --NotebookApp.password='' --log-level=DEBUG + diff --git a/spawnerdockerfile/jupyter_server_config.py b/spawnerdockerfile/jupyter_server_config.py old mode 100644 new mode 100755 From e8f862a2bc32b75d9edcaed5dcfccc3e7eb32bbe Mon Sep 17 00:00:00 2001 From: hotwa Date: Sat, 25 May 2024 18:30:00 +0800 Subject: [PATCH 18/21] update mojo --- spawnerdockerfile/Dockerfile.base-notebook | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/spawnerdockerfile/Dockerfile.base-notebook b/spawnerdockerfile/Dockerfile.base-notebook index 9ac8703..f9cb670 100755 --- a/spawnerdockerfile/Dockerfile.base-notebook +++ b/spawnerdockerfile/Dockerfile.base-notebook @@ -138,12 +138,17 @@ USER ${NB_UID} ARG MODULAR_HOME="/home/${CREATE_USER}/.modular" ENV MODULAR_HOME=$MODULAR_HOME ENV PATH="$MODULAR_HOME/pkg/packages.modular.com_mojo/bin:/home/${CREATE_USER}/.local/bin:$PATH" +ARG HTTP_PROXY="" +ARG HTTPS_PROXY="" +ARG NO_PROXY="localhost,127.0.0.1" +ENV HTTP_PROXY=$HTTP_PROXY +ENV HTTPS_PROXY=$HTTPS_PROXY RUN < Date: Sat, 25 May 2024 19:55:39 +0800 Subject: [PATCH 19/21] update rust install --- spawnerdockerfile/Dockerfile.base-notebook | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/spawnerdockerfile/Dockerfile.base-notebook b/spawnerdockerfile/Dockerfile.base-notebook index f9cb670..cf74c55 100755 --- a/spawnerdockerfile/Dockerfile.base-notebook +++ b/spawnerdockerfile/Dockerfile.base-notebook @@ -151,11 +151,24 @@ mamba activate mojo && modular install mojo mamba run -n mojo python -m ipykernel install --user --name="mojo" --display-name="mojo_env" EOT -# Install Rust +# Install Rust https://rsproxy.cn/#getStarted +ENV RUSTUP_DIST_SERVER="https://rsproxy.cn" +ENV RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup" ENV PATH="/home/${CREATE_USER}/.cargo/bin:$PATH" +ENV CARGO_UNSTABLE_SPARSE_REGISTRY="true" RUN <> ~/.cargo/config.toml source $HOME/.cargo/env cargo install evcxr_jupyter evcxr_jupyter --install From 2c58a44cdea34e18bc98b5fff93a4d03bcc37903 Mon Sep 17 00:00:00 2001 From: hotwa Date: Sat, 25 May 2024 19:57:23 +0800 Subject: [PATCH 20/21] change to hotwa quay images --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ffd8ff4..25d86ca 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,7 @@ services: # All containers will join this network DOCKER_NETWORK_NAME: jupyterhub-network # JupyterHub will spawn this Notebook image for users - DOCKER_NOTEBOOK_IMAGE: hotwa/notebook:latest + DOCKER_NOTEBOOK_IMAGE: quay.io/hotwa/notebook:latest # Notebook directory inside user image DOCKER_NOTEBOOK_DIR: /home/jovyan/work From 98265022aa14f2e7b94aa541698ed61169116522 Mon Sep 17 00:00:00 2001 From: hotwa Date: Sun, 26 May 2024 14:23:25 +0800 Subject: [PATCH 21/21] add torch sciplot env --- spawnerdockerfile/Dockerfile.base-notebook | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/spawnerdockerfile/Dockerfile.base-notebook b/spawnerdockerfile/Dockerfile.base-notebook index cf74c55..bd39340 100755 --- a/spawnerdockerfile/Dockerfile.base-notebook +++ b/spawnerdockerfile/Dockerfile.base-notebook @@ -174,5 +174,14 @@ cargo install evcxr_jupyter evcxr_jupyter --install EOT +RUN <