# 不要忘了安装 openstack client
$ yum install python-openstackclient

# 使用 pip 安装 openstack client
$ yum install python-devel python-pip
$ pip install python-openstackclient

$ yum install openstack-keystone -y
Loaded plugins: fastestmirror, langpacks
Loading mirror speeds from cached hostfile
 * base: mirrors.bupt.edu.cn
 * epel: mirrors.bfsu.edu.cn
 * extras: mirrors.bupt.edu.cn
 * updates: mirrors.bupt.edu.cn
No package openstack-keystone available.
Error: Nothing to do

$ yum search openstack
Loaded plugins: fastestmirror, langpacks
Loading mirror speeds from cached hostfile
 * base: mirrors.bupt.edu.cn
 * epel: mirrors.tuna.tsinghua.edu.cn
 * extras: mirrors.bupt.edu.cn
 * updates: mirrors.bupt.edu.cn
============================================ N/S matched: openstack ============================================
ansible-openstack-modules.noarch : Unofficial Ansible modules for managing Openstack
centos-release-openstack-queens.noarch : OpenStack from the CentOS Cloud SIG repo configs
centos-release-openstack-rocky.noarch : OpenStack from the CentOS Cloud SIG repo configs
centos-release-openstack-stein.noarch : OpenStack from the CentOS Cloud SIG repo configs
centos-release-openstack-train.noarch : OpenStack from the CentOS Cloud SIG repo configs
diskimage-builder.noarch : Image building tools for OpenStack
golang-github-rackspace-gophercloud-devel.noarch : The Go SDK for Openstack http://gophercloud.io
php-opencloud.noarch : PHP SDK for OpenStack/Rackspace APIs
php-opencloud-doc.noarch : Documentation for PHP SDK for OpenStack/Rackspace APIs
python2-oslo-sphinx.noarch : OpenStack Sphinx Extensions and Theme for Python 2

  Name and summary matches only, use "search all" for everything.

$ yum install centos-release-openstack-train.noarch

# 先创建用户 keystone
$ useradd -g keystone keystone

# 安装 glance 之前，创建 domain、project
# 创建 domain
$ openstack domain create --description "Default Domain" default
# 创建 project
$ openstack project create --domain default --description "Service Project" service

export OS_USERNAME=admin
export OS_PASSWORD=ADMIN_PASS
export OS_PROJECT_NAME=admin
export OS_USER_DOMAIN_NAME=Default
export OS_PROJECT_DOMAIN_NAME=Default
export OS_AUTH_URL=http://controller:5000/v3
export OS_IDENTITY_API_VERSION=3

# 第二步有个 source admin-openrc，即为安装 keystone 时最后给出的内容

# 创建
$ pvcreate /dev/sda
$ vgcreate cinder-volumes /dev/sda

# 删除
## 移除卷
$ lvremove cinder--volumes-cinder--volumes--pool_tmeta
## 删除组
$ vgremove cinder-volumes
## 删除物理卷
$ pvremove /dev/sda

filter = [ "a/sda/", "a/nvme/", "r/.*/" ]

"a|.*|"

"r|/dev/cdrom|"

# apt install apache2 libapache2-mod-uwsgi tgt 

# vim /etc/hosts
## 添加如下内容, 注意替换 ip
192.168.6.55 controller

# add-apt-repository cloud-archive:antelope
# apt install nova-compute
# apt install python3-openstackclient

# apt install mariadb-server python3-pymysql

# vim /etc/mysql/mariadb.conf.d/99-openstack.cnf
## 添加如下内容
[mysqld]
bind-address = 192.168.6.55

default-storage-engine = innodb
innodb_file_per_table = on
max_connections = 4096
collation-server = utf8_general_ci
character-set-server = utf8

# service mysql restart

# mysql_secure_installation

# apt install rabbitmq-server
# rabbitmqctl add_user openstack RABBIT_PASS
# rabbitmqctl set_permissions openstack ".*" ".*" ".*"

# apt install memcached python3-memcache
# vim /etc/memcached.conf
## 将 -l 127.0.0.1 替换为 Controller 的 ip
-l 192.168.6.55

# apt install etcd
# vim /etc/default/etcd
## 参考如下内容修改
ETCD_NAME="controller"
ETCD_DATA_DIR="/var/lib/etcd"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster-01"
ETCD_INITIAL_CLUSTER="controller=http://192.168.6.55:2380"
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.6.55:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.6.55:2379"
ETCD_LISTEN_PEER_URLS="http://0.0.0.0:2380"
ETCD_LISTEN_CLIENT_URLS="http://192.168.6.55:2379"
# systemctl enable etcd
# systemctl restart etcd

# mysql
MariaDB [(none)]> CREATE DATABASE keystone;
## 注意替换掉下面的 KEYSTONE_DBPASS
MariaDB [(none)]> GRANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'localhost' \
IDENTIFIED BY 'KEYSTONE_DBPASS';
MariaDB [(none)]> GRANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'%' \
IDENTIFIED BY 'KEYSTONE_DBPASS';
MariaDB [(none)]> exit

# apt install keystone
# vim /etc/keystone/keystone.conf
## 修改如下内容, 注意替换 KEYSTONE_DBPASS
[database]
# ...
connection = mysql+pymysql://keystone:KEYSTONE_DBPASS@controller/keystone
[token]
# ...
provider = fernet

# su -s /bin/sh -c "keystone-manage db_sync" keystone
# keystone-manage fernet_setup --keystone-user keystone --keystone-group keystone
# keystone-manage credential_setup --keystone-user keystone --keystone-group keystone
# keystone-manage bootstrap --bootstrap-password ADMIN_PASS \
  --bootstrap-admin-url http://controller:5000/v3/ \
  --bootstrap-internal-url http://controller:5000/v3/ \
  --bootstrap-public-url http://controller:5000/v3/ \
  --bootstrap-region-id RegionOne
# vim /etc/apache2/apache2.conf
## 添加如下内容
ServerName controller

## 环境变量中添加如下内容
$ export OS_USERNAME=admin, 注意替换 ADMIN_PASS
$ export OS_PASSWORD=ADMIN_PASS
$ export OS_PROJECT_NAME=admin
$ export OS_USER_DOMAIN_NAME=Default
$ export OS_PROJECT_DOMAIN_NAME=Default
$ export OS_AUTH_URL=http://controller:5000/v3
$ export OS_IDENTITY_API_VERSION=3

# mysql
## 注意修改 GLANCE_DBPASS
MariaDB [(none)]> CREATE DATABASE glance;
MariaDB [(none)]> GRANT ALL PRIVILEGES ON glance.* TO 'glance'@'localhost' \
  IDENTIFIED BY 'GLANCE_DBPASS';
MariaDB [(none)]> GRANT ALL PRIVILEGES ON glance.* TO 'glance'@'%' \
  IDENTIFIED BY 'GLANCE_DBPASS';
MariaDB [(none)]> exit
## 注意保持上面设置的环境变量
$ openstack user create --domain default --password-prompt glance
$ openstack role add --project service --user glance admin
$ openstack service create --name glance \
  --description "OpenStack Image" image
$ openstack endpoint create --region RegionOne \
  image public http://controller:9292
# apt install glance

$ vim /etc/tgt/targets.conf
# 添加如下内容
include /var/lib/cinder/volumes/*

# Failed to set locale, defaulting to C

# vim /etc/profile
# 输入
export LANG=en_US.UTF-8
export LANGUAGE=en_US.UTF-8
export LC_COLLATE=C
export LC_CTYPE=en_US.UTF-8

# 问题出在 Amazon Linux 2，所有 yum 命令卡住
ps aux | grep yum

kill -9 <pid>  # kill 所有 yum 进程

# 保存
CentOS Linux 8 - AppStream                                                                                          9.5  B/s |  38  B     00:03
Error: Failed to download metadata for repo 'appstream': Cannot prepare internal mirrorlist: No URLs in mirrorlist

# 解决
dnf --disablerepo '*' --enablerepo=extras swap centos-linux-repos centos-stream-repos -y
dnf distro-sync -y

sudo vi /etc/sysconfig/network-scripts/ifcfg-eth0

# 添加 DNS1
DNS1=192.168.6.1

# 重启网络服务
sudo systemctl restart NetworkManager

sudo yum install gcc72-c++

# lcov
yum install lcov.noarch

# vim /etc/netplan/99_config.yaml
network:
  version: 2
  renderer: networkd
  ethernets:
    eth0:
      addresses:
        - 10.10.10.2/24  # 静态地址
      gateway4: 10.10.10.1
      nameservers:       # 可选
          search: [mydomain, otherdomain]
          addresses: [10.10.10.1, 1.1.1.1]

$ sudo netplan apply

# method 1
$ sudo netplan apply

# method 2
$ sudo nmcli networking off
$ sudo nmcli networking on

# method 3
$ sudo systemctl start NetworkManager 
$ sudo systemctl stop NetworkManager 

# method 4
$ sudo service network-manager restart

# method 5
$ sudo ifdown -a
$ sudo ifup -a

# method 6
$ sudo systemctl restart sytemd-networking

# up / down
ip link set dev ens7 up    # up
ip link set dev ens7 down  # down

# 展示路由配置
ip route show

# 添加路由
ip route add <network_ip>/<cidr> via <gateway_ip> dev <network_card_name>
## 示例
ip route add 192.168.6.0/24 dev eth0
ip route add default 

# 删除路由
ip route del default # 删除默认路由

## ubuntu 20.04+
# cat /etc/netplan/***.yaml
network:
    version: 2
    ethernets:
        ens4:
            dhcp4: true
            match:
                macaddress: fa:16:3e:65:2c:6b
            mtu: 1450
            set-name: ens4
            nameservers:
                addresses: [192.168.6.1,8.8.8.8]  # 设置 dns

# 生效
sudo netplan apply

# 查看防火墙状态
$ sudo ufw status
# active: 激活, inactive: 非激活

# disable
$ sudo ufw disable

# 弃用版本
gateway4: 192.168.6.1
# 新版设置
routes:
  - to: default
		via: 192.168.6.1

apt install tshark

tshark -i <entwork-interface>

# 筛选端口
tshark -i enp0 -f "src port 30800"

# 配置
$ cat /etc/netplan/*_config.yaml
# Let NetworkManager manage all devices on this system
network:
  version: 2
  renderer: NetworkManager
  ethernets:
    enp4s0:
      match:
        macaddress: 2a:04:a0:46:08:38
      dhcp4: true
      wakeonlan: true
    enp5s0:
      match:
        macaddress: 2a:04:a0:46:08:39
      dhcp4: true
      wakeonlan: true
        
# 生效
$ sudo netplan apply

$ yarn logs -applicationId <application-id>

$ yarn app -kill <application-id>

$ yarn top

# 切换 tab
Command + 方向键
Command + 数字 

# 切换 panel
Command + Options + 方向键

# 定义数组
>>> l = [1, 2, 3, 4, 5, 6, 7]
# 随机选一个
>>> random.choice(l)  			
6
# 随机选多个，可重复
>>> random.choices(l, k=3)
[7, 4, 7]
# 随机选多个，不重复
>>> random.sample(l, 3)
[3, 5, 7]

# 安装 vncserver
$ sudo yum install tigervnc-server

# 设置密码
$ vncpasswd

# 拷贝配置
$ sudo cp /lib/systemd/system/vncserver@.service /etc/systemd/system/vncserver@:1.service

# 修改配置
$ sudo vim /etc/systemd/system/vncserver@\:1.service

[Unit]
Description=Remote desktop service (VNC)
After=syslog.target network.target

[Service]
Type=forking
ExecStartPre=/bin/sh -c '/usr/bin/vncserver -kill %i > /dev/null 2>&1 || :'
ExecStart=/sbin/runuser -l wii -c "/usr/bin/vncserver %i -geometry 1920x1080"
# PIDFile=/home/wii/.vnc/%H%i.pid
ExecStop=/bin/sh -c '/usr/bin/vncserver -kill %i > /dev/null 2>&1 || :'

[Install]
WantedBy=multi-user.target

$ sudo systemctl daemon-reload         # 重新加载配置
$ sudo systemctl start vncserver@:1    # 启动 vnc server
$ sudo systemctl status vncserver@:1   # 查看状态
$ sudo systemctl enable vncserver@:1   # 开启自启动

$ ./jmeter -n -t config.jmx -l results.jtl

# 命令格式
jmeter -n -t test-file [-p property-file] [-l results-file] [-j log-file]

# 参数
-n	non gui
-t	test file
-l	result file

...
: "${HEAP:="-Xms4g -Xmx4g -XX:MaxMetaspaceSize=1024m"}"  # 调大堆内存和 Meta 内存
...

# ubuntu
## 安装依赖
sudo apt-get install -y ca-certificates curl gnupg lsb-release
## 添加 GPG key
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
## 添加 sources repository
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
## install docker engine
sudo apt-get update
sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin

# cri-docker.service
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket

[Service]
Type=notify
ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --network-plugin=
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
# Both the old, and new location are accepted by systemd 229 and up, so using the old location
# to make them work for either version of systemd.
StartLimitBurst=3

# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
# this option work for either version of systemd.
StartLimitInterval=60s

# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

# Comment TasksMax if your systemd version does not support it.
# Only systemd 226 and above support this option.
TasksMax=infinity
Delegate=yes
KillMode=process

[Install]
WantedBy=multi-user.target

# cri-docker.socket
[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service

[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target

# 移动二进制
sudo mv cri-docker /usr/bin
# 移动 systemd 配置
sudo mv cri-docker.service /etc/systemd/system/
sudo mv cri-docker.socket /etc/systemd/system/

sudo systemctl daemon-reload
sudo systemctl enable cri-docker.service
sudo systemctl enable --now cri-docker.socket

# 启动服务
sudo service cri-docker start

sudo apt-get update && apt-get install -y apt-transport-https
curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add - 
# root 用户运行
cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF
sudo apt-get update
sudo apt-get install -y kubelet kubeadm kubectl

$ sudo kubeadm config images pull \
	--image-repository registry.aliyuncs.com/google_containers \
	--cri-socket unix:///var/run/cri-dockerd.sock

# 10.244.0.0/16 是 chennel 扩展的配置
# --apiserver-advertise-address 是 master 节点的 ip，如果是单机，即为该机器 ip 地址
$ kubeadm init --image-repository registry.aliyuncs.com/google_containers \
  --pod-network-cidr 10.244.0.0/16 \
  --control-plane-endpoint 10.1.0.145 \
	--cri-socket unix:///var/run/cri-dockerd.sock

root@k8s-master-1:/home/ubuntu# export KUBECONFIG=/etc/kubernetes/admin.conf
root@k8s-master-1:/home/ubuntu# kubectl get nodes
NAME           STATUS   ROLES           AGE     VERSION
k8s-master-1   Ready    control-plane   5m55s   v1.24.1

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

ubuntu@k8s-master-1:~$ kubectl get nodes
NAME           STATUS   ROLES           AGE     VERSION
k8s-master-1   Ready    control-plane   4m52s   v1.24.1

kubeadm reset [glags] 

preflight              Run reset pre-flight checks
update-cluster-status  Remove this node from the ClusterStatus object.
remove-etcd-member     Remove a local etcd member.
cleanup-node           Run cleanup node.

$ kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

$ mk-docker-opts.sh -d /run/docker_opts.env -c

# root 用户执行
$ vim /lib/systemd/system/docker.service
# 添加这一行
EnvironmentFile=/run/docker_opts.env
# 修改这一行
ExecStart=/usr/bin/dockerd $DOCKER_OPTS -H fd:// ...

# 重启 docker
$ systemctl daemon-reload
$ systemctl restart docker

# 在 master 节点
$ kubeadm token create --print-join-command
... 

# 在待加入的节点, 在上面生成的命令后面指定 cri socket
$ kubeadm join 10.1.0.145:6443 --token nxxcv7.gge00x97wiphualw --discovery-token-ca-cert-hash sha256:cfb324b2ee7ee548b08e38d2e6d60905e392553bf6715504e87888183a1238fd
u --cri-socket unix:///var/run/cri-dockerd.sock

# 为新节点指定 label
$ kubectl label node k8s-worker-1 node-role.kubernetes.io/worker=worker

# 验证
ubuntu@k8s-master-1:~$ kubectl get nodes
NAME           STATUS   ROLES           AGE     VERSION
k8s-master-1   Ready    control-plane   9h      v1.24.1
k8s-worker-1   Ready    worker          3m17s   v1.24.1

# 安装 dashboard
$ kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.6.0/aio/deploy/recommended.yaml

# 启动代理
$ kubectl proxy --address=0.0.0.0

apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard

kubectl apply -f account.yaml

kubectl apply -f permission.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard

kubectl -n kubernetes-dashboard create token admin-user

# 自认证证书
# 生成 dashboard.pass.key
$ openssl genrsa -des3 -passout pass:over4chars -out dashboard.pass.key 2048
# 生成 dashboard.key
$ openssl rsa -passin pass:over4chars -in dashboard.pass.key -out dashboard.key
$ rm dashboard.pass.key # 可以删除了
# 生成 dashboard.csr
$ openssl req -new -key dashboard.key -out dashboard.csr # 一直回车
# 生成 dashboard.crt
$ openssl x509 -req -sha256 -days 365 -in dashboard.csr -signkey dashboard.key -out dashboard.crt

$ kubectl delete -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.6.0/aio/deploy/recommended.yaml

$ kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

$ kubectl get nodes
NAME                    STATUS     ROLES                  AGE     VERSION
localhost.localdomain   NotReady   control-plane,master   2m13s   v1.22.1

$ kubectl get pods --all-namespaces
NAMESPACE     NAME                                            READY   STATUS    RESTARTS   AGE
kube-system   coredns-78fcd69978-9dk4n                        0/1     Pending   0          2m52s
kube-system   coredns-78fcd69978-w52zc                        0/1     Pending   0          2m52s
kube-system   etcd-localhost.localdomain                      1/1     Running   0          3m6s
kube-system   kube-apiserver-localhost.localdomain            1/1     Running   0          3m6s
kube-system   kube-controller-manager-localhost.localdomain   1/1     Running   0          3m8s
kube-system   kube-proxy-4w84n                                1/1     Running   0          2m52s
kube-system   kube-scheduler-localhost.localdomain            1/1     Running   0          3m6s

# 查看 pod  信息
$ kubectl get pods -A  # A = all-namespaces

# 删除 pod
$ kubectl delete pod kubernetes-dashboard --namespace=kubernetes-dashboard

ubuntu@k8s-master-1:~$ systemctl status kubelet
● kubelet.service - kubelet: The Kubernetes Node Agent
     Loaded: loaded (/lib/systemd/system/kubelet.service; enabled; vendor preset: enabled)
    Drop-In: /etc/systemd/system/kubelet.service.d
             └─10-kubeadm.conf
     Active: active (running) since Wed 2022-06-08 16:18:53 UTC; 2s ago
       Docs: https://kubernetes.io/docs/home/
   Main PID: 69055 (kubelet)
      Tasks: 29 (limit: 38495)
     Memory: 39.5M
     CGroup: /system.slice/kubelet.service
             └─69055 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/co>

Jun 08 16:18:55 k8s-master-1 kubelet[69055]: E0608 16:18:55.158403   69055 kubelet_node_status.go:92] "Unable to register node with API server" err="Post \"https://10.1.0>
Jun 08 16:18:55 k8s-master-1 kubelet[69055]: E0608 16:18:55.193156   69055 kubelet.go:2419] "Error getting node" err="node \"k8s-master-1\" not found"
...

root@k8s-master-1:~# service docker status
● docker.service - Docker Application Container Engine
     Loaded: loaded (/lib/systemd/system/docker.service; enabled; vendor preset: enabled)
     Active: active (running) since Wed 2022-06-08 16:30:32 UTC; 34min ago
TriggeredBy: ● docker.socket
       Docs: https://docs.docker.com
   Main PID: 1039 (dockerd)
      Tasks: 23
     Memory: 134.1M
     CGroup: /system.slice/docker.service
             └─1039 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock

Jun 08 17:04:52 k8s-master-1 dockerd[1039]: time="2022-06-08T17:04:52.874831490Z" level=error msg="Handler for POST /v1.40/images/create returned error: Get \"https://k8s.gcr.io/v2/\": context deadline exceeded"
Jun 08 17:04:58 k8s-master-1 dockerd[1039]: time="2022-06-08T17:04:58.867350258Z" level=warning msg="Error getting v2 registry: Get \"https://k8s.gcr.io/v2/\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"
...

# 查看服务实时日志
$ journalctl -u docker -f

[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[kubelet-check] Initial timeout of 40s passed.

sudo mkdir -p /etc/systemd/system/docker.service.d 
sudo touch /etc/systemd/system/docker.service.d/proxy.conf
sudo chmod 777 /etc/systemd/system/docker.service.d/proxy.conf
sudo echo '
[Service]
Environment="HTTP_PROXY=socks5://192.168.6.19:3213" 
Environment="HTTPS_PROXY=socks5://192.168.6.19:3213"
' >> /etc/systemd/system/docker.service.d/proxy.conf
sudo systemctl daemon-reload
sudo systemctl restart docker
sudo systemctl restart kubelet

{
  "kind": "Status",
  "apiVersion": "v1",
  "metadata": {
    
  },
  "status": "Failure",
  "message": "error trying to reach service: dial tcp 172.17.0.3:8443: connect: connection refused",
  "reason": "ServiceUnavailable",
  "code": 503
}

# 查看 flannel 网络
$ cat /run/flannel/subnet.env
FLANNEL_NETWORK=10.244.0.0/16
FLANNEL_SUBNET=10.244.0.1/24
FLANNEL_MTU=1400
FLANNEL_IPMASQ=true

# 找到 mk-docker-opts.sh，在 flannel 镜像里面；也可以下载 flannel 的二进制包找到这个脚本
$ sudo find / -name 'mk-docker-opts.sh'
/var/lib/docker/overlay2/8779d2bd83ddf0e237da15f5c0e62fd79bbf6d3868cea87ec926c471f1184774/merged/opt/bin/mk-docker-opts.sh
/var/lib/docker/overlay2/99462f1d9e955f5c40a11844119dc1e0f295208c20a696e7bea76b39324a9943/diff/opt/bin/mk-docker-opts.sh

# root 用户; 生成 docker opts
$ alias mk-docker-opts="/var/lib/docker/overlay2/99462f1d9e955f5c40a11844119dc1e0f295208c20a696e7bea76b39324a9943/diff/opt/bin/mk-docker-opts.sh"
$ mk-docker-opts -d /run/docker_opts.env -c

# root 用户执行
$ vim /lib/systemd/system/docker.service
# 添加这一行
EnvironmentFile=/run/docker_opts.env
# 修改这一行
ExecStart=/usr/bin/dockerd $DOCKER_OPTS -H fd:// ...

# 重启 docker
$ systemctl daemon-reload
$ systemctl restart docker

# alias k8s="kubectl --namespace=default"
$ k8s get pods
NAME                                        READY   STATUS    RESTARTS   AGE
dashboard-metrics-scraper-8c47d4b5d-2bbbx   1/1     Running   0          43s
kubernetes-dashboard-59fccbc7d7-9wmn9       1/1     Running   0          43s
$ k8s describe pod kubernetes-dashboard-59fccbc7d7-9wmn9
Name:         kubernetes-dashboard-59fccbc7d7-9wmn9
Namespace:    default
Priority:     0
Node:         k8s-worker-1/10.1.0.123
Start Time:   Sat, 11 Jun 2022 07:03:10 +0000
Labels:       k8s-app=kubernetes-dashboard
              pod-template-hash=59fccbc7d7
Annotations:  seccomp.security.alpha.kubernetes.io/pod: runtime/default
Status:       Running
IP:           10.244.1.3
...

Insecure access detected. Sign in will not be available. Access Dashboard securely over HTTPS or using localhost. Read more here .

--auto-generate-certificates
--namespace=default
--insecure-bind-address=0.0.0.0
--insecure-port=8080
--enable-insecure-login	

$ kubectl port-forward -n default service/kubernetes-dashboard 18082:443 --address='0.0.0.0' 

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: default
spec:
  ports:
    - port: 443
      targetPort: 8443
  type: NodePort
  selector:
    k8s-app: kubernetes-dashboard

upstream k8s-dashboard {
    server 10.244.1.2:8443;
}

server {
    # auth_basic "Auth Message";
    # auth_basic_user_file /etc/apache2/.htpasswd;
    client_max_body_size 100m;
	  listen 8443 ssl default_server;
	  listen [::]:8443 ssl default_server;
	  server_name  192.168.6.103;
	  include snippets/self-signed.conf;

    location / {
        proxy_pass         https://k8s-dashboard;
        proxy_set_header   Host             $host;
        proxy_set_header   X-Real-IP        $remote_addr;
        proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
        
        # enable websocket
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
    }
}

# 浏览器输入地址
https://192.168.6.103:8443/#/login