tools►Poetry

Poetry

说明

Poetry 用于管理 Python 项目信息及依赖。

安装

1
curl -sSL https://install.python-poetry.org | python3 -

Workflow

初始化

1
poetry init
tools►Jupyter

Jupyter

部署

Jupyter 的镜像定义在这里，Docker Hub 中的 Jupyter 组织下镜像不再更新。

关于镜像选择，all-spark-notebook 基于 pyspark-notebook 构建。

Docker Compose 部署

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
version: "3.9"

services:
  jupyter:
    image: quay.io/jupyter/all-spark-notebook:latest
    container_name: jupyterlab
    restart: unless-stopped
    ports:
      - "8888:8888"
    environment:
      - JUPYTER_ENABLE_LAB=yes
      - JUPYTER_TOKEN=jupyter
    volumes:
      - jupyter-data:/home/jovyan/work
volumes:
  jupyter-data:

ROOT 权限

1
$ docker exec -it -u root jupyterlab bash # root 用户登录容器，可以在 root 下赋予 jovyan 管理员权限
tools►MinIO

MinIO

Server

Docker

1
2
3
4
5
6
7
8
9
10
mkdir -p ~/minio/data

docker run \
   -p 9000:9000 \
   -p 9001:9001 \
   --name minio \
   -v ~/minio/data:/data \
   -e "MINIO_ROOT_USER=ROOTNAME" \
   -e "MINIO_ROOT_PASSWORD=CHANGEME123" \
   quay.io/minio/minio server /data --console-address ":9001"

Docker Compose

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
version: '3.8'

services:
  minio:
    image: quay.io/minio/minio
    container_name: minio
    ports:
      - "9000:9000"  # API 端口
      - "9001:9001"  # 控制台端口
    environment:
      MINIO_ROOT_USER: ROOTNAME
      MINIO_ROOT_PASSWORD: CHANGEME123
    volumes:
      - ~/minio/data:/data
    command: server /data --console-address ":9001"

Client

官方文档

Linux

1
2
3
4
5
6
7
# Linux
wget https://dl.min.io/client/mc/release/linux-amd64/mc
chmod +x mc
sudo mv mc /usr/local/bin/mc
# Alias
mc alias set local http://127.0.0.1:9000 {MINIO_ROOT_USER} {MINIO_ROOT_PASSWORD}
mc admin info local

Mac

1
brew install minio/stable/mc

客户端常用命令

官方文档

研习录►机器学习

MetaSpore - 构建

步骤

  • 构建基础镜像（dev / compile 镜像）
  • 训练
    • 构建 Training Build 镜像（打包 MetaSpore Wheel 安装包）
    • 构建 Training Build Release 镜像（训练镜像，包含 Spark 等依赖）
  • 开发
    • 构建 Jupyter 镜像
    • 构建 Code Server 镜像
  • 在线 Serving
    • 构建 Serving Build 镜像（基于 dev 镜像）
    • 构建 Serving Release 镜像（Release 版本的可发布镜像，基于 ubuntu 镜像，安装必要依赖、拷贝 Release 版本二进制文件）

构建

在 Fork 的代码库中新增了 Makefile 用于方便地构建目标、镜像。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
.PHONY: dev training serving jupyter code-server all

REPOSITORY := sunzhenkai
VERSION := 0.0.1
RUNTIME := cpu
FIX_ARG := --network host --build-arg RUNTIME=$(RUNTIME) --build-arg http_proxy=${http_proxy} --build-arg https_proxy=${https_proxy}
DOCKER_CMD := DOCKER_BUILDKIT=1 docker build $(FIX_ARG)

DEV_IMAGE := $(REPOSITORY)/metaspore-dev-$(RUNTIME):$(VERSION)
TRAINING_BUILD_IMAGE := $(REPOSITORY)/metaspore-training-build:$(VERSION)
TRAINING_RELEASE_IMAGE := $(REPOSITORY)/metaspore-training-release:$(VERSION)
SERVING_BUILD_IMAGE := $(REPOSITORY)/metaspore-serving-build:$(VERSION)
SERVING_RELEASE_IMAGE := $(REPOSITORY)/metaspore-serving-release:$(VERSION)
JUPYTER_IMAGE := $(REPOSITORY)/metaspore-training-jupyter:$(VERSION)
CODESERVER_IMAGE := $(REPOSITORY)/metaspore-codeserver:$(VERSION)

dev:
	@$(DOCKER_CMD) $(FIX_ARG) -f docker/ubuntu20.04/Dockerfile_dev -t $(DEV_IMAGE) .

training: dev
	@DOCKER_BUILDKIT=1 docker build $(FIX_ARG) -f docker/ubuntu20.04/Dockerfile_training_build --build-arg DEV_IMAGE=$(DEV_IMAGE) -t $(TRAINING_BUILD_IMAGE) .
	@DOCKER_BUILDKIT=1 docker build $(FIX_ARG) -f docker/ubuntu20.04/Dockerfile_training_release --build-arg METASPORE_RELEASE=build --build-arg METASPORE_BUILD_IMAGE=$(TRAINING_BUILD_IMAGE) -t $(TRAINING_RELEASE_IMAGE) --target release .

serving: dev
	@DOCKER_BUILDKIT=1 docker build $(FIX_ARG) -f docker/ubuntu20.04/Dockerfile_serving_build --build-arg DEV_IMAGE=$(DEV_IMAGE) -t $(SERVING_BUILD_IMAGE) .
	@DOCKER_BUILDKIT=1 docker build $(FIX_ARG) -f docker/ubuntu20.04/Dockerfile_serving_release --build-arg BUILD_IMAGE=$(SERVING_BUILD_IMAGE) -t $(SERVING_RELEASE_IMAGE) --target serving_release .

jupyter:
	@DOCKER_BUILDKIT=1 docker build $(FIX_ARG) -f docker/ubuntu20.04/Dockerfile_jupyter --build-arg RELEASE_IMAGE=$(TRAINING_RELEASE_IMAGE) -t $(JUPYTER_IMAGE) docker/ubuntu20.04

code-server:
	@DOCKER_BUILDKIT=1 docker build $(FIX_ARG) -f docker/ubuntu20.04/Dockerfile_codeserver --build-arg RELEASE_IMAGE=$(TRAINING_RELEASE_IMAGE) -t $(CODESERVER_IMAGE) docker/ubuntu20.04

all: dev training serving jupyter code-server

运行 make all 就可以构建基础镜像、Serving、Training、Jupyter、Code Server 镜像。

工具►ssh

ssh

配置

ProxyCommand

1
2
3
4
5
6
7
8
9
Host relay
  HostName 192.168.6.5
  User wii
  IdentityFile ~/.ssh/id_rsa
Host dev
  HostName 192.168.6.2
  User wii
  IdentityFile ~/.ssh/id_rsa
  ProxyCommand ssh relay -W %h:%p
工具►fdisk

fdisk

创建并挂载分区

  1. 查看硬盘
1
$ fdisk -l
  1. 使用 fdisk 创建分区表
1
2
3
$ fdisk /dev/sda
(fdisk) p # 打印分区信息
(fdisk) g # 创建 GPT 分区表

2.1 创建分区

创建 /boot 分区

1
2
3
4
(fdisk) n  # 创建分区
(fdisk) Partition number...: # 回车
(fdisk) First section...:    # 回车
(fdisk) Last section...: +1G

创建 / 分区

1
2
3
4
(fdisk) n  # 创建分区
(fdisk) Partition number...: # 回车, 默认值
(fdisk) First section...:    # 回车
(fdisk) Last section...: 		 # 回车, 使用最大 section number

确认

1
(fdisk) p   # 重新打印分区，有新创建的两个分区

保存

1
(fdisk) w
  1. 格式化分区
1
2
$ mkfs.fat -F32 /dev/sda1   # 格式化 /boot 分区（假设为 /dev/sda1）
$ mkfs.ext4 /dev/sda2       # 格式化 / 分区（假设为 /dev/sda2）
  1. 挂载
1
2
3
$ mount /dev/sda2 /mnt       # 挂载根分区
$ mkdir /mnt/boot            # 创建 boot 挂载点
$ mount /dev/sda1 /mnt/boot  # 挂载 /boot 分区