coding►c++

c++ threads

Thread

thread

1
2
3
#include <thread>

std::thread trd(function, args...);

线程私有数据

1
2
3
4
5
6
7
8
# 创建线程私有数据
int pthread_key_create(pthread_key_t *key, void (*destructor)(void*)); # 第二个函数为 destructor 函数，线程结束时调用
# 设置私有数据
int pthread_setspecific(pthread_key_t key,const void *pointer));  # 第一个参数由 pthread_key_create 产生
# 获取私有数据
void *pthread_getspecific(pthread_key_t key);
# 删除
int pthread_key_delete(pthread_key_t key);

Mutex

1
2
#include <mutex>
#include <shared_mutex>
python►tools►gunicorn

gunicorn logging

多进程日志问题

使用 gunicorn 启动 python 服务，会使用多进程。如果每个进程使用单独的 logger，操作相同的日志文件，那么对于按时间滚动的日志（TimedRotatingFileHandler），会出现冲突（多个进程同一时间 reopen 日志文件）。冲突造成的现象比较复杂，比如丢失日志，再比如开始运行正常但是过一段时间无法写入日志。这里 可以了解，gunicorn 使用 os.fork 创建的多进程，可以公用日志文件的 offset 和句柄。但是对于多进程写到同一个文件仍可能有问题，解决方案可以参考这里，方式是使用 QueueHandler 或 SocketHandler。

另外一种方案，是把日志写到一个文件里面，使用 logrotate 工具分隔日志。建议使用这种方式，不使用内置的 TimedRotatingFileHandler，代价是新增一个 logrotate 的配置 + crontab 配置。

当然，还有其他方式，比如继承 TimedRotatingFileHandler，使用锁来规避问题，这里 和 这里是个示例。

logrotate

1
2
3
4
5
6
7
8
9
10
11
12
13
# /home/ubuntu/server/config/server-logrotate.conf
/home/ubuntu/server/logs/server.log {
    daily
    rotate 30
    missingok
    nocompress
    copytruncate
    notifempty
    dateext
    dateyesterday
}
# crontab
0 0 * * * cd /home/ubuntu/server; logrotate -v -s logs/logrotate-status configs/logrotate.conf >> logs/logrotate.log 2>&1 &

Log Server

一个 Log Server 可以对应多个 Python 服务，只需修改 config 文件即可。多个进程 / 线程通过网络把日志写到日志服务，日志服务负责落日志，这样就避免了多个进程同时操作同一个日志文件的问题。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
import logging
import logging.config
import logging.handlers
import pickle
import socketserver
import struct

logging.config.fileConfig('config/log_config.ini')


class LogRecordStreamHandler(socketserver.StreamRequestHandler):
    """Handler for a streaming logging request.

    This basically logs the record using whatever logging policy is
    configured locally.
    """

    def handle(self):
        """
        Handle multiple requests - each expected to be a 4-byte length,
        followed by the LogRecord in pickle format. Logs the record
        according to whatever policy is configured locally.
        """
        while True:
            chunk = self.connection.recv(4)
            if len(chunk) < 4:
                break
            slen = struct.unpack('>L', chunk)[0]
            chunk = self.connection.recv(slen)
            while len(chunk) < slen:
                chunk = chunk + self.connection.recv(slen - len(chunk))
            obj = self.unPickle(chunk)
            record = logging.makeLogRecord(obj)
            self.handleLogRecord(record)

    def unPickle(self, data):
        return pickle.loads(data)

    def handleLogRecord(self, record):
        # if a name is specified, we use the named logger rather than the one
        # implied by the record.
        if self.server.logname is not None:
            name = self.server.logname
        else:
            name = record.name
        logger = logging.getLogger(name)
        # N.B. EVERY record gets logged. This is because Logger.handle
        # is normally called AFTER logger-level filtering. If you want
        # to do filtering, do it at the client end to save wasting
        # cycles and network bandwidth!
        logger.handle(record)


class LogRecordSocketReceiver(socketserver.ThreadingTCPServer):
    """
    Simple TCP socket-based logging receiver suitable for testing.
    """

    allow_reuse_address = True

    def __init__(self, host='127.0.0.1',
                 port=logging.handlers.DEFAULT_TCP_LOGGING_PORT,
                 handler=LogRecordStreamHandler):
        socketserver.ThreadingTCPServer.__init__(self, (host, port), handler)
        self.abort = 0
        self.timeout = 1
        self.logname = None

    def serve_until_stopped(self):
        import select
        abort = 0
        while not abort:
            rd, wr, ex = select.select([self.socket.fileno()],
                                       [], [],
                                       self.timeout)
            if rd:
                self.handle_request()
            abort = self.abort


def run(port: int = logging.handlers.DEFAULT_TCP_LOGGING_PORT):
    tcpserver = LogRecordSocketReceiver(port=port)
    print('About to start TCP server...')
    tcpserver.serve_until_stopped()


if __name__ == '__main__':
    run()

日志配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
; declarations
[loggers]
keys = root,serving,webhook

[handlers]
keys = generic,serving,webhook

[formatters]
keys = generic

; loggers
[logger_root]
level = INFO
handlers = generic

[logger_serving]
level = INFO
handlers = serving
propagate = 0
qualname = serving

[logger_webhook]
level = INFO
handlers = webhook
propagate = 0
qualname = webhook

; handlers
[handler_generic]
class = handlers.TimedRotatingFileHandler
args = ('./logs/generic.log', 'midnight', 1, 30,)
level = INFO
formatter = generic

[handler_serving]
class = handlers.TimedRotatingFileHandler
args = ('./logs/serving.log', 'midnight', 1, 30,)
level = INFO
formatter = generic

[handler_webhook]
class = handlers.TimedRotatingFileHandler
args = ('./logs/webhook.log', 'midnight', 1, 30,)
level = INFO
formatter = generic

; formatters
[formatter_generic]
format = %(asctime)s-%(levelname)s: %(message)s
datefmt = %Y-%m-%d %H:%M:%S%z
class = logging.Formatter

使用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
def get_logger(name, port: int = logging.handlers.DEFAULT_TCP_LOGGING_PORT, level=logging.INFO):
    log = logging.getLogger(name)
    log.setLevel(level)
    # socket handler
    sh = logging.handlers.SocketHandler('127.0.0.1', port)
    formatter = logging.Formatter('%(asctime)s-%(levelname)s: %(message)s')
    # consul handler
    ch = logging.StreamHandler()
    ch.setLevel(level)
    ch.setFormatter(formatter)

    log.addHandler(sh)
    log.addHandler(ch)
    return log


logger = get_logger('serving')
webhook_logger = get_logger('webhook')
架构►存储►mongodb

mongo - cxx driver

安装

1
2
# brew
brew insall mongo-cxx-deriver

其他参考这里。

示例

官方示例参考这里。

连接数据库

1
2
3
mongocxx::instance instance{};
mongocxx::uri uri("mongodb://192.168.4.13:27017");
mongocxx::client client(uri);

读取数组

1
2
3
4
5
6
7
8
auto arr_element = doc["contribs"];
if (arr_element && arr_element.type() == bsoncxx::type::k_array) {
    bsoncxx::array::view arr(arr_element.get_array().value);
    std::cout << arr.length() << std::endl; // 67; not count of elements
    for (auto e: arr) {
        std::cout << "E " << e.get_utf8().value.to_string() << std::endl;
    }
}
架构►监控►grafana

grafana

安装

可以从 清华源 下载。

1
2
3
# centos
wget https://mirrors.tuna.tsinghua.edu.cn/grafana/yum/rpm/grafana-7.5.3-1.x86_64.rpm
sudo npm install grafana-7.5.3-1.x86_64.rpm

插件

Grafana Image Renderer

image

1
2
# 需要安装的依赖
yum install -y libXcomposite libXdamage libXtst cups libXScrnSaver pango atk adwaita-cursor-theme adwaita-icon-theme at at-spi2-atk at-spi2-core cairo-gobject colord-libs dconf desktop-file-utils ed emacs-filesystem gdk-pixbuf2 glib-networking gnutls gsettings-desktop-schemas gtk-update-icon-cache gtk3 hicolor-icon-theme jasper-libs json-glib libappindicator-gtk3 libdbusmenu libdbusmenu-gtk3 libepoxy liberation-fonts liberation-narrow-fonts liberation-sans-fonts liberation-serif-fonts libgusb libindicator-gtk3 libmodman libproxy libsoup libwayland-cursor libwayland-egl libxkbcommon m4 mailx nettle patch psmisc redhat-lsb-core redhat-lsb-submod-security rest spax time trousers xdg-utils xkeyboard-config alsa-lib

安装

1
grafana-cli plugins install grafana-image-renderer
架构►框架►seastar

seastar with docker

创建容器

1
docker run ... --sysctl net.core.somaxconn=10240 ...

设置

1
2
3
4
5
6
7
8
9
10
11
12
13
# 临时修改 aio-max-nr
# 注意 image-name 是镜像的名称，不是 container 的名称
docker run --privileged <image-name> sh -c 'echo 1048576 > /proc/sys/fs/aio-max-nr'

# 或者
docker run --privileged -it zhenkai.sun/external-with-openssh bash
sysctl -w fs.aio-max-nr=1048576

# 验证
cat /proc/sys/fs/aio-max-nr
cat /proc/sys/net/core/somaxconn

docker run --privileged --sysctl net.core.somaxconn=10240 zhenkai.sun/external-with-openssh bash
coding►c++►gcc

gcc

Install

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# 下载 
## 从这里 https://mirrors.aliyun.com/gnu/gcc/ 选一个版本，或使用其他源。
VERSION=7.3.0
wget https://mirrors.aliyun.com/gnu/gcc/gcc-${VERSION}/gcc-${VERSION}.tar.gz
# 解压
tar xzf gcc-${VERSION}.tar.gz
cd gcc-${VERSION}

# 下载依赖
./contrib/download_prerequisites 

# 配置
./configure --prefix=/opt/gcc/${VERSION} --enable-languages=c,c++ --disable-multilib
# 编译
make -j$(nproc)
make install

# 如果不想污染源文件
mkdir gcc-build
cd gcc-build
../configure --prefix=/opt/gcc/${VERSION} --enable-languages=c,c++ --disable-multilib
## 另一个配置
../configure --enable-bootstrap --enable-languages=c,c++,objc,obj-c++,fortran,go,lto --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --enable-libmpx --enable-libsanitizer --enable-gnu-indirect-function --enable-libcilkrts --enable-libatomic --enable-libquadmath --enable-libitm --with-tune=generic --with-arch_32=x86-64

错误处理

7.3.0

1
2
3
4
5
6
7
8
9
10
# 错误
../../.././libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc:157:10: fatal error: sys/ustat.h: No such file or directory
 #include <sys/ustat.h>
 
# 解决方案
## 1. 不编译 sanitizer, configure 时添加参数 --disable-libsanitizer
vim libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
## 2. 注释第 157、250 行
// #include <sys/ustat.h>
// unsigned struct_ustat_sz = sizeof(struct ustat);

编译

1
2
3
4
5
6
7
gcc sample.cpp # 默认生成 a.out

# 指定输出文件
gcc -o sample.exe sample.cpp

# 指定 c++ std 版本
gcc -std=c++17 -o sample.exe sample.cpp

参数

1
2
3
-c	仅编译
-Dname=value	定义 name 值
-o file-name	输出

Flags

1
2
3
4
5
6
7
8
9
10
11
# warning
-Wall			开启警告（all: 应该开启的最小警告集合）
-Wextra		开启扩展警告
-Werror 	所有警告视为错误
-Wno-error=...	    关闭某项警告视为错误
-Wno-<warning-name>	关闭某项警告
-Wextra
-w                  忽略所有警告

# 常用警告
-Werror=return-type    函数没有 return 视为错误

示例

1
2
3
4
5
6
export CFLAGS='-g -O3'
export CXXFLAGS='-ggdb3 -O0 -Wno-narrowing'
export CPPFLAGS='-DX=1 -DY=2 -Wno-narrowing'
export CCFLAGS='--asdf -Wno-narrowing'

make CXXFLAGS='-ggdb3 -O0 -Wno-narrowing' CPPFLAGS='-DX=1 -DY=2 -Wno-narrowing' CCFLAGS='--asdf -Wno-narrowing' all -j

注意

  • -w 和 -Werror... 同时用会有冲突，-w 会短路 -Werror。-w -Werror=... 和 -Werror=... -w 都不行。

编译库

1
// common.h

查看查找链接库路径

1
2
gcc -Xlinker -v
g++ -Xlinker -v

链接

  • libraries 允许未定义的符号（undefined symbols）
  • executable 不允许有未定义的符号
  • 在代码中定义的符号（如函数名）还未使用到之前，链接器并不会把它加入到连接表中

修改系统默认库查找路径

1
export LIBRARY_PATH={path}:{path}

AUR 低版本 GCC

1
2
3
4
configure:4314: $? = 0
configure:4303: gcc -V >&5
gcc: error: unrecognized command-line option '-V'
gcc: fatal error: no input files

上面的报错可以忽略，只是尝试探测 gcc 版本，详见 这里。

架构►存储►redis

redis usage

登录

1
$ redis-cli -h <host> -p <port>

查询

集群信息

1
2
3
4
5
6
# 集群信息
> cluster info
# 节点信息
> cluster nodes 
# 内存信息
> info memory

导出数据

redis-cli

redis-shake

1
2
./redis-shake.linux -type=dump -conf=redis-shake.conf

redis-shake.conf 配置参考这里。

搭建集群

1
2
3
4
5
6
nohup redis-server --cluster-enabled yes --port 6379 --cluster-config-file nodes1.conf &
nohup redis-server --cluster-enabled yes --port 6378 --cluster-config-file nodes2.conf &
nohup redis-server --cluster-enabled yes --port 6377 --cluster-config-file nodes3.conf &

sleep 3 # 等服务节点起来
redis-cli --cluster create 127.0.0.1:6379 127.0.0.1:6378 127.0.0.1:6377

nodes1.conf、nodes1.conf、nodes1.conf 可以不存在，运行后 redis 自动创建。不同的 redis 实例，需要绑定不同的配置文件，默认为 node.conf。

coding►c++

c++ 最佳实践

工程

工具

网络

服务治理

配置中心

定义全局变量

1
2
3
4
5
6
7
8
9
// global.h
namespace {
  extern int i;
}

// global.cpp
namespace {
  int i = 1024;
}

随机数

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
// random_util.h
#include "random"
#include "climits"
namespace utils {
    extern std::random_device gRandomDev;
    extern std::mt19937 gRandomGenerator;
    extern std::uniform_int_distribution<std::mt19937::result_type> gRandomDist; // (0, INT_MAX)
  
  	uint32_t RandomInt(const int &min = 0, const int &max = 0);
}

// random_util.cpp
#include "random_util.h"
namespace ranker::utils {
    std::random_device gRandomDev;
    std::mt19937 gRandomGenerator(gRandomDev());
    std::uniform_int_distribution<std::mt19937::result_type> gRandomDist(0, INT_MAX);

    uint32_t RandomInt(const int &min, const int &max) {
        if (min == max && min == 0) {
            return (uint32_t) gRandomDist(gRandomGenerator);
        } else {
            std::uniform_int_distribution<std::mt19937::result_type> dist(min, max);
            return (uint32_t) dist(gRandomGenerator);
        }
    }
}

/*
满足泊松分布的随机数
std::poisson_distribution<> d{/*mean=*/4};
d(gRandomGenerator);
*/