spark = pyspark.sql.SparkSession.builder.master("spark://datascience-spark:7077").getOrCreate()

# 只需要指定 hadoop name node 的地址即可
# namenode 是 hadoop namenode 的机器名，这里可以反解出 ip
df = spark.read.json("hdfs://namenode/sample.json") 

# arch
$ sudo pacman -S curl zip unzip tar cmake ninja

$ git clone https://github.com/Microsoft/vcpkg.git
$ ./vcpkg/bootstrap-vcpkg.sh

# 命令行
cmake ... -DCMAKE_TOOLCHAIN_FILE=/path/to/vcpkg/scripts/buildsystems/vcpkg.cmake

# CMakeLists.txt 内
## NOTE: 要在 project 之前
set(CMAKE_TOOLCHAIN_FILE /path/to/vcpkg/scripts/buildsystems/vcpkg.cmake)

{
  "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg-configuration.schema.json",
  "default-registry": {
    "kind": "git",
    "repository": "https://internal/mirror/of/github.com/Microsoft/vcpkg",
    "baseline": "eefee7408133f3a0fef711ef9c6a3677b7e06fd7"
  },
  "registries": [
    {
      "kind": "git",
      "repository": "https://github.com/northwindtraders/vcpkg-registry",
      "baseline": "dacf4de488094a384ca2c202b923ccc097956e0c",
      "packages": [ "beicode", "beison" ]
    }
  ],
  "overlay-ports": [
    "./team-ports",
    "./custom-ports"
   ],
  "overlay-triplets": [ "./my-triplets" ]
}

# Configure
# 1. 如果没有 configure.ac 等自动化配置，需要 SKIP_CONFIGURE，且 COPY_SOURCE
# vcpkg_configure_make(SOURCE_PATH "${SOURCE_PATH}" OPTIONS ${MAKE_OPTS} COPY_SOURCE SKIP_CONFIGURE)
vcpkg_configure_make(SOURCE_PATH "${SOURCE_PATH}" OPTIONS ${MAKE_OPTS})
vcpkg_build_make()
vcpkg_fixup_pkgconfig()

vcpkg hash /path/to/source.tar.gz

git rev-parse HEAD:ports/<library>

# 生成
git diff > patch
# 应用
git apply patch

rm -r {VCPKG_HOME}/buildtrees/{library}
rm -r {VCPKG_HOME}/packages/{library}

-- Checking for module 'cassandra_static'
--   No package 'cassandra_static' found
CMake Error at /snap/cmake/1328/share/cmake-3.27/Modules/FindPkgConfig.cmake:607 (message):
  A required package was not found

$ rm -r {VCPKG_HOME}/buildtrees/protobuf

include_directories(${JSON11_INCLUDE_DIRS})
target_link_libraries({target} ${JSON11_LIBRARIES}) 

CMake Error: CMake was unable to find a build program corresponding to "Ninja".  CMAKE_MAKE_PROGRAM is not set.  You probably need to select a different build tool.

mkdir /tmp/<tree-ish>
git archive <tree-ish> | tar -x -C /tmp/<tree-ish>

git archive --remote=https://github.com/user/repo.git <tree-ish> | tar -x -C /tmp/<tree-ish>

# 从本地检出
git -C ~/.local/vcpkg archive f9ee8bb31f04f4e6a8c0d3e96fbb98deeb448d45 | tar -x -C .

# 从 remote 检出 (不一定能行)
git archive --remote=git@github.com:grpc/grpc.git f9ee8bb31f04f4e6a8c0d3e96fbb98deeb448d45 | tar -x -C .

export CC=/usr/bin/gcc10-cc
export CXX=/usr/bin/gcc10-c++

config-x64-linux-dbg-err.log:invalid configuration `gcc10': machine `gcc10-unknown' not recognized
config-x64-linux-dbg-config.log:host=gcc10

ln -s /usr/bin/gcc10-cc /usr/bin/cc
ln -s /usr/bin/gcc10-c++ /usr/bin/c++
export CC=/usr/bin/cc
export CXX=/usr/bin/c++

CMake Error at scripts/cmake/vcpkg_execute_build_process.cmake:134 (message):
    Command failed: /usr/bin/make STATIC=1 SHARED=0 V=1 -j 1 -f Makefile all

vcpkg_build_make(OPTIONS STATIC=1 SHARED=0 DISABLE_PARALLEL BUILD_TARGET build INSTALL_TARGET install)

CKPT2 : ar

CKPT2 : ar rc

$(info CKPT3 AR=$(AR))
AR= ar rc
$(info CKPT4 AR=$(AR))

# vcpkg 输出:
CKPT3 AR=ar
CKPT4 AR=ar

# 手动运行输出:
CKPT3 AR=ar
CKPT4 AR=ar rc

# 打印代码
$(info CKPT6 AR 的来源: $(origin AR))

# vcpkg 运行
CKPT6 AR 的来源: environment override
# 手动运行
CKPT6 AR 的来源: default

# 修改 makefile 中的赋值语句，添加 override 强制赋值
override AR= ar rc

# 在父 makefile 中添加 unexport
unexport AR

# 定义 AR 参数
vcpkg_build_make(DISABLE_PARALLEL OPTIONS "AR=ar rc")

# install prefix fixup: 由于 aerospike client 没有 configure 脚本，所以需要手动设置安装路径 INSTALL_PREFIX
# INSTALL_PREFIX 是代码文件 pkg/install 内定义的
# 这里的 INSTALL_PREFIX 是 {prefix_dir}/{install_dir}
# vcpkg_configure_make 默认会指定 --prefix, 但是这里没有 configure 脚本，添加了 SKIP_CONFIGURE 选项
# 因此这里修复 INSTALL_PREFIX 为 {prefix_dir}/{install_dir}
# 不然这里会报错: https://github.com/microsoft/vcpkg/blob/master/scripts/cmake/vcpkg_build_make.cmake#L177C8-L177C102
# 报错代码： file(RENAME "${CURRENT_PACKAGES_DIR}_tmp${Z_VCPKG_INSTALL_PREFIX}" "${CURRENT_PACKAGES_DIR}") 
string(REGEX REPLACE "([a-zA-Z]):/" "/\\1/" Z_VCPKG_INSTALL_PREFIX "${CURRENT_INSTALLED_DIR}")
set(ENV_INSTALL_PREFIX_ "$ENV{INSTALL_PREFIX}")
set(ENV{INSTALL_PREFIX} "${CURRENT_PACKAGES_DIR}${Z_VCPKG_INSTALL_PREFIX}")

vcpkg install {package} --binarysource="files,/root/.cache/vcpkg/archives --dry-run --debug

[DEBUG] <abientries for zlib:x86-windows>
[DEBUG]   0001-Prevent-invalid-inclusions-when-HAVE_-is-set-to-0.patch|750b9542cb55e6328cca01d3ca997f1373b9530afa95e04213168676936e7bfa
[DEBUG]   0002-skip-building-examples.patch|835ddecfed752e0f49be9b0f8ff7ba76541cb0a150044327316e22ca84f8d0c2
[DEBUG]   0003-build-static-or-shared-not-both.patch|d6026271dcb3d8fc74b41e235620ae31576a798e77aa411c3af8cd9e948c02b1
[DEBUG]   0004-android-and-mingw-fixes.patch|37a43eddbcb1b7dde49e7659ae895dfd0ff1df66666c1371ba7d5bfc49d8b438
[DEBUG]   cmake|3.26.2
[DEBUG]   features|core
[DEBUG]   portfile.cmake|ac63047b644fa758860dd7ba48ff9a13b058c6f240b8e8d675b8fbba035976be
[DEBUG]   ports.cmake|5a8e00cedff0c898b1f90f7d129329d0288801bc9056562b039698caf31ff3f3
[DEBUG]   post_build_checks|2
[DEBUG]   powershell|7.3.6
[DEBUG]   triplet|x86-windows
[DEBUG]   triplet_abi|3e71dd1d4afa622894ae367adbbb1ecbd42c57c51428a86b675fa1c8cad3a581-36b818778ba6f2c16962495caedb9a7b221d5be4c60de1cd3060f549319a9931-f5d02a6542664cfbd4a38db478133cbb1a18f315
[DEBUG]   usage|be22662327df993eebc437495add75acb365ab18d37c7e5de735d4ea4f5d3083
[DEBUG]   vcpkg-cmake|1b3dac4b9b0bcbef227c954b495174863feebe3900b2a6bdef0cd1cf04ca1213
[DEBUG]   vcpkg-cmake-wrapper.cmake|5d49ef2ee6448479c2aad0e5f732e2676eaba0411860f9bebabe6002d66f57d1
[DEBUG]   vcpkg.json|bc94e2540efabe36130a806381a001c57194e7de67454ab7ff1e30aa15e6ce23
[DEBUG]   vcpkg_copy_pdbs|d57e4f196c82dc562a9968c6155073094513c31e2de475694143d3aa47954b1c
[DEBUG]   vcpkg_fixup_pkgconfig|588d833ff057d3ca99c14616c7ecfb5948b5e2a9e4fc02517dceb8b803473457
[DEBUG]   vcpkg_from_git|8f27bff0d01c6d15a3e691758df52bfbb0b1b929da45c4ebba02ef76b54b1881
[DEBUG]   vcpkg_from_github|b743742296a114ea1b18ae99672e02f142c4eb2bef7f57d36c038bedbfb0502f
[DEBUG]   vcpkg_replace_string|d43c8699ce27e25d47367c970d1c546f6bc36b6df8fb0be0c3986eb5830bd4f1
[DEBUG] </abientries>

# 百分比
100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[1m])) * 100)

avg by (instance) (irate(node_cpu_seconds_total{mode!='idle'}[1m]))
# 区分 mode
avg by (instance,mode) (irate(node_cpu_seconds_total{mode!='idle'}[1m]))

$ sudo pamac install virtualbox
$ sudo modprobe vboxdrv
$ sudo modprobe vboxnetadp
$ sudo modprobe vboxnetflt

# 启动 minikube
$ minikube start --driver=virtualbox

$ minikube addons configure registry-creds

Do you want to enable AWS Elastic Container Registry? [y/n]: n

Do you want to enable Google Container Registry? [y/n]: n

Do you want to enable Docker Registry? [y/n]: y
-- Enter docker registry server url: registry.cn-hangzhou.aliyuncs.com
-- Enter docker registry username: <username>
-- Enter docker registry password:

Do you want to enable Azure Container Registry? [y/n]: n
✅  registry-creds was successfully configured
$ minikube addons enable registry-creds

$ kubectl create secret generic regcred \
    --from-file=.dockerconfigjson=$HOME/.docker/config.json \
    --type=kubernetes.io/dockerconfigjson \
    --namespace=default

# 在 deployment.yml 中添加如下内容
spec:
		...
    spec:
      imagePullSecrets:
        - name: regcred

# 安装并启用 sshd.service
$ sudo pacman install openssl openssh
$ sudo systemctl enable sshd.service
$ sudo systemctl start sshd.service

$ pacman -h
$ pacman -S -h

# manjaro
# 1. 刷新
$ sudo pacman-mirrors -i -c China -m rank
# 选择合适的源
# 2. 更新
$ sudo pacman -Syy

$ sudo pacman-mirrors --geoip && sudo pacman -Syyu

# -S
-S 安装
-S --needed --noconfirm
--needed: 跳过已经安装到最新版本的包
--noconfirm: 跳过确认

-R 卸载

# 所有安装的包
$ pacman -Q
# 查询指定包
$ pacman -Q vim

$ pacman -Ss vim

sudo pacman -S --needed git base-devel
cd /tmp
git clone https://aur.archlinux.org/yay.git
cd yay
makepkg -si

$ sudo pacman -S --needed base-devel
$ git clone https://aur.archlinux.org/paru.git
$ cd paru
$ makepkg -si

# 搜索
$ paru -Ss gcc7

paru <target> -- Interactively search and install <target>.
paru -- Alias for paru -Syu.
paru -S <target> -- Install a specific package.
paru -Sua -- Upgrade AUR packages.
paru -Qua -- Print available AUR updates.
paru -G <target> -- Download the PKGBUILD and related files of <target>.
paru -Gp <target> -- Print the PKGBUILD of <target>.
paru -Gc <target> -- Print the AUR comments of <target>.
paru --gendb -- Generate the devel database for tracking *-git packages. This is only needed when you initially start using paru.
paru -Bi . -- Build and install a PKGBUILD in the current directory.

# 安装 pamac
$ pacman -S pamac-cli
# 搜索 aur 包
$ pamac search ttf-ms-fonts --aur
# 安装
$ pamac build ttf-ms-fonts 

sudo pacman -S hyprland 
# yay -S hyprland-git

# tools
## mako 通知
## xdg-desktop-portal-hyprland 通讯
## dolphin 文件管理器
## sddm 桌面显示管理器
## pipewire、wireplumber 屏幕共享 
## waybar 状态栏
## hyprpaper 设置壁纸
## brightnessctl 调整亮度
## playerctl 播放控制
## network-manager-applet 网络图形界面
## polkit-kde-agent 权限认证界面
## wl-clipboard	粘贴板
sudo pacman -S sddm dolphin mako pipewire wireplumber xdg-desktop-portal-hyprland waybar hyprpaper brightnessctl playerctl network-manager-applet polkit-kde-agent firefox wl-clipboard	uwsm
yay -S cliphist wl-clip-persist eww

# start tools
sudo systemctl enable --now sddm.service

# fonts
sudo pacman -S ttf-dejavu ttf-liberation noto-fonts noto-fonts-cjk noto-fonts-emoji ttf-droid ttf-opensans ttf-fira-code ttf-font-awesome

$ pip3 install torch transformers fschat
# 安装 lfs
## ubuntu
sudo apt install git-lfs
# 验证
$ git lfs install

# llama 原始模型
$ git clone https://huggingface.co/decapoda-research/llama-13b-hf
# vicuna 模型
$ git clone https://huggingface.co/lmsys/vicuna-13b-delta-v1.1

$ python3 -m fastchat.model.apply_delta \
    --base-model-path /data/llama-13b-hf \
    --target-model-path /data/vicuna-13b \
    --delta-path /data/vicuna-13b-delta-v1.1

nohup python3 -m fastchat.serve.controller &
nohup python3 -m fastchat.serve.model_worker --model-path /data/vicuna-13b --device cpu &
nohup python3 -m fastchat.serve.gradio_web_server &

sudo apt-get purge docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin docker-ce-rootless-extras
sudo rm -rf /var/lib/docker
sudo rm -rf /var/lib/containerd
for pkg in docker.io docker-doc docker-compose docker-compose-v2 podman-docker containerd runc; do sudo apt-get remove $pkg; done
sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
sudo systemctl restart docker

services:
  ollama:
    image: ollama/ollama:latest
    restart: always
    hostname: ollama
    runtime: nvidia
    user: root
    ports:
      - '11434:11434'
    volumes:
      - /data/docker/llm/ollama:/root/.ollama
    networks:
      - default
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]

no-cgroups = false # 修改, true -> false

sudo systemctl restart docker

[sources.test]
type = "file"
include = [ "/tmp/test.log" ]
line_delimiter = "$$\n"

# unicode
line_delimiter = "\u0003\u0004\n"

[api]
enabled = true
address = "0.0.0.0:8686"

[sources.test]
type = "file"
include = [ "/tmp/test.log" ]
line_delimiter = "$$\n"

[sinks.console]
inputs = ["test"]
target = "stdout"
type = "console"
encoding.codec = "text"

# pull image
$ docker pull timberio/vector:0.30.0-debian
# create container
$ docker run \
  -d \
  -v $PWD/vector.toml:/etc/vector/vector.toml:ro \
  -v /tmp/test.log:/tmp/test.log \
  --name vector \
  -p 8686:8686 \
  timberio/vector:0.30.0-debian