spring kafka

连接 Kafka

1
2
3
4
5
6
7
// 连接 kafka
@Bean
public KafkaAdmin admin() {
Map<String, Object> configs = new HashMap<>();
configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:9092");
return new KafkaAdmin(configs);
}

管理 Topic

1
2
3
4
5
6
7
8
@Bean
public NewTopic divinerInternalStrategyOperation() {
return TopicBuilder.name("example-topic")
.partitions(1)
.replicas(1)
.compact()
.build();
}

订阅 topic

1
2
3
4
@KafkaListener(id = "consumer-group", topics = "diviner_internal_strategy_operation")
public void listen(String in) {
// ...
}

发送消息

1
2
3
4
@Autowired
private KafkaTemplate<Object, Object> template;

template.send(key, value);

参考

MapR

Install

发行版支持矩阵参考这里

ubuntu cloud 18.04

MapR

1
2
# wget https://package.mapr.com/releases/installer/mapr-setup.sh -P /tmp
# sudo bash /tmp/mapr-setup.sh

登录

访问 https://<ip>:9443,注意是 https,chrome 不能访问的话换个浏览器。

image-20210826232821748

spring cacheable

说明

spring 提供了缓存功能,接下来完成一个示例,然后看下怎么不缓存空结果以及怎么写单测。

引入依赖

1
2
3
4
5
6
7
8
9
10
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-cache</artifactId>
<version>2.3.3.RELEASE</version>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
<version>2.6.2</version>
</dependency>

编写代码

1
2
3
4
5
6
7
├── cache
│   ├── CacheableService.java
│   ├── CustomCacheManager.java
│   └── CustomKeyGenerator.java
├── controller
│   ├── CacheController.java
├── ...

CustomCacheManager.java

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
package pub.wii.cook.springboot.cache;

import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.collect.Lists;
import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.cache.caffeine.CaffeineCacheManager;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.util.concurrent.TimeUnit;

@EnableCaching
@Configuration
public class CustomCacheManager {
private static final int CACHE_CAP = 100;
public static final String CACHE_NAME = "sample";

@Bean(name = CACHE_NAME)
CacheManager cacheManager() {
CaffeineCacheManager cm = new CaffeineCacheManager();
cm.setCaffeine(Caffeine.newBuilder().expireAfterAccess(1, TimeUnit.MINUTES)
.recordStats()
.initialCapacity(CACHE_CAP)
.maximumSize(CACHE_CAP));
cm.setCacheNames(Lists.newArrayList(CACHE_NAME));
return cm;
}
}

CustomKeyGenerator.java

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
package pub.wii.cook.springboot.cache;

import org.springframework.cache.interceptor.KeyGenerator;
import org.springframework.stereotype.Component;

import java.lang.reflect.Method;
import java.util.Arrays;

@Component
public class CustomKeyGenerator implements KeyGenerator {
@SuppressWarnings("NullableProblems")
@Override
public Object generate(Object o, Method method, Object... objects) {
return o.getClass().getSimpleName() + ":" + method.getName() + ":" + Arrays.toString(objects);
}
}

CacheableService.java

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
package pub.wii.cook.springboot.cache;

import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;

import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.UUID;

import static pub.wii.cook.springboot.cache.CustomCacheManager.CACHE_NAME;

@Service
public class CacheableService {
private static final Random random = new Random();

@Cacheable(
cacheManager = CACHE_NAME,
cacheNames = CACHE_NAME,
keyGenerator = "customKeyGenerator"
)
public List<Object> cache(String key) {
List<Object> res = new ArrayList<>();
int size = random.nextInt(10) + 1;
for (int i = 0; i < size; ++i) {
res.add(UUID.randomUUID().toString());
}
return res;
}
}

CacheController.java

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
package pub.wii.cook.springboot.controller;

import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import pub.wii.cook.springboot.cache.CacheableService;

import javax.annotation.Resource;
import java.util.List;

@RestController
@RequestMapping("cache")
public class CacheController {

@Resource
CacheableService cacheableService;

@RequestMapping(value = "get",
method = RequestMethod.GET,
produces = MediaType.APPLICATION_JSON_VALUE)
@ResponseBody
public ResponseEntity<List<Object>> get(@RequestParam("key") String key) {
return ResponseEntity.ok(cacheableService.cache(key));
}
}

示例

1
2
$ curl "http://localhost:8080/cache/get?key=wii"
["4b2dee35-781b-42cd-b594-8994025fa36e","6b4432c3-f16f-45c6-b56e-334054588a65","57b7f89a-8dd6-4500-98ab-b14973a88971"]

不缓存空结果

只需要在 @Cacheable 添加 unless = "#result == null or #result.size() == 0" 选项即可。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
@Service
public class CacheableService {
private static final Random random = new Random();

@Cacheable(
cacheManager = CACHE_NAME,
cacheNames = CACHE_NAME,
keyGenerator = "customKeyGenerator",
unless = "#result == null or #result.size() == 0"
)
public List<Object> cache(String key) {
List<Object> res = new ArrayList<>();
int size = random.nextInt(10) + 1;
for (int i = 0; i < size; ++i) {
res.add(UUID.randomUUID().toString());
}
return res;
}
}

单测

相较于其他单测方式,直接测 Cache 内有没有缓存数据更直接。

CacheableServiceTest.java

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
package pub.wii.cook.springboot.cache;

import com.google.common.collect.Lists;
import lombok.SneakyThrows;
import org.junit.jupiter.api.Test;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.cache.interceptor.KeyGenerator;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
import pub.wii.cook.springboot.config.CookSpringBootConfiguration;

import javax.annotation.Resource;

import java.util.ArrayList;
import java.util.List;

import static org.junit.jupiter.api.Assertions.*;
import static pub.wii.cook.springboot.cache.CustomCacheManager.CACHE_NAME;

@ContextConfiguration(classes = CookSpringBootConfiguration.class)
@SpringJUnitConfig(classes = CookSpringBootConfiguration.class)
class CacheableServiceTest {

interface ICache {
List<Integer> cacheWithEmpty(List<Integer> echo);

List<Integer> cacheWithoutEmpty(List<Integer> echo);
}

@Configuration
@EnableCaching
static class Config {
static class ICacheImpl implements ICache {

@Cacheable(
cacheManager = CACHE_NAME,
cacheNames = CACHE_NAME,
keyGenerator = "customKeyGenerator"
)
@Override
public List<Integer> cacheWithEmpty(List<Integer> echo) {
return echo;
}

@Cacheable(
cacheManager = CACHE_NAME,
cacheNames = CACHE_NAME,
keyGenerator = "customKeyGenerator",
unless = "#result == null or #result.size() == 0"
)
@Override
public List<Integer> cacheWithoutEmpty(List<Integer> echo) {
return echo;
}
}

@Bean
ICache iCache() {
return new ICacheImpl();
}
}

@Resource(name = CACHE_NAME)
CacheManager cacheManager;

@Resource
ICache iCache;

@SuppressWarnings("ConstantConditions")
@SneakyThrows
@Test
void cache() {
List<Integer> nonEmpty = Lists.newArrayList(1, 2, 3);
List<Integer> empty = new ArrayList<>();
List<Integer> nil = null;
Cache cache = cacheManager.getCache(CACHE_NAME);
KeyGenerator kg = new CustomKeyGenerator();
assertNotNull(cache);

iCache.cacheWithEmpty(nonEmpty);
iCache.cacheWithEmpty(empty);
iCache.cacheWithEmpty(nil);
iCache.cacheWithoutEmpty(nonEmpty);
iCache.cacheWithoutEmpty(empty);
iCache.cacheWithoutEmpty(nil);

assertEquals(cache.get(genKeyWithEmpty(kg, nonEmpty)).get(), nonEmpty);
assertEquals(cache.get(genKeyWithEmpty(kg, empty)).get(), empty);
assertEquals(cache.get(genKeyWithEmpty(kg, nil)).get(), nil);
assertEquals(cache.get(genKeyWithoutEmpty(kg, nonEmpty)).get(), nonEmpty);
assertEquals(cache.get(genKeyWithoutEmpty(kg, empty)), nil);
assertEquals(cache.get(genKeyWithoutEmpty(kg, nil)), nil);
}

@SneakyThrows
Object genKeyWithEmpty(KeyGenerator kg, List<Integer> arg) {
// 第一个参数不要用 iCache, iCache 是通过反射机制设置的对象, 有可能是一个 Proxy
// 获取 class name 的时候可能会得到奇怪的值, 导致 key 匹配不上
return kg.generate(new Config.ICacheImpl(),
ICache.class.getMethod("cacheWithEmpty", List.class), arg);
}

@SneakyThrows
Object genKeyWithoutEmpty(KeyGenerator kg, List<Integer> arg) {
// 第一个参数不要用 iCache, iCache 是通过反射机制设置的对象, 有可能是一个 Proxy
// 获取 class name 的时候可能会得到奇怪的值, 导致 key 匹配不上
return kg.generate(new Config.ICacheImpl(),
ICache.class.getMethod("cacheWithoutEmpty", List.class), arg);
}
}

CookSpringBootConfiguration.java

1
2
3
4
5
6
7
8
9
10
11
package pub.wii.cook.springboot.config;

import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.EnableAspectJAutoProxy;
import org.springframework.context.annotation.aspectj.EnableSpringConfigured;

@EnableSpringConfigured
@EnableAspectJAutoProxy
@ComponentScan(basePackages = {"pub.wii.cook"}, lazyInit = true)
public class CookSpringBootConfiguration {
}

成长

经验之谈

  • 临渊羡鱼,不如退而结网
  • 凡事预则立,不预则废
  • 保持定力,一步一步慢慢来(尤其对于工作、长期目标)
  • 莫等闲,白了少年头,空悲切。岳飞 《满江红》

应该具备的能力

  • 保持身体健康的能力
  • 保持理性的能力
    • 保持心情愉悦的能力
      • 脱离消极、沮丧的能力
    • 保持独立思考的能力
      • 能清晰且辩证地了解自己的价值观
      • 有独处的能力
    • 保持积极乐观心态的能力
    • 学会接受那些接受不了、但又不得不接受的变故的能力
  • 保持感性的能力
    • 有感知被爱的能力
    • 有爱人的能力

TIPS

  • 道阻且长,行则必至

elastic search

安装

官网给了丰富的安装方式。

Centos 7

添加仓库

1
2
3
4
5
6
7
8
9
$ sudo vim /etc/yum.repos.d/elasticsearch.repo
[elasticsearch]
name=Elasticsearch repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=0
autorefresh=1
type=rpm-md

安装

1
$ sudo yum install --enablerepo=elasticsearch elasticsearch

systemd 配置

1
2
$ sudo systemctl daemon-reload
$ sudo systemctl enable elasticsearch.service

启动

1
sudo systemctl start elasticsearch.service

配置

添加用户

文档参考这里

对于使用 yum 在 centos 7 安装的 es 来说,可执行程序在 /usr/share/elasticsearch/bin 下。

1
2
3
4
5
6
7
8
9
10
# 命令格式
$ bin/elasticsearch-users
([useradd <username>] [-p <password>] [-r <roles>]) |
([list] <username>) |
([passwd <username>] [-p <password>]) |
([roles <username>] [-a <roles>] [-r <roles>]) |
([userdel <username>])

# 创建管理员
$ bin/elasticsearch-users useradd wii -p <password> -r superuser

角色 内置角色在这里

  • superuser : 管理员,所有权限

安装 Kibana

yum 安装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# install public signing key
$ rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch

# 添加源
$ vim /etc/yum.repo.d/kibana.repo
[kibana-7.x]
name=Kibana repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

# 安装
$ yum install kibana
$ chkconfig --add kibana

# systemd
$ sudo /bin/systemctl daemon-reload
$ sudo /bin/systemctl enable kibana.service

启动 Kibana

1
$ sudo systemctl start kibana.service

配置

1
2
3
# file: /etc/kibana/kibana.yml
# 添加如下内容
server.host: "0.0.0.0"

访问

http://host:5601

使用

数据

工具

在 Kibana 首页,打开 Dev Tools,或者直接访问 http://host:5601/app/dev_tools#/console

image-20210922115803345

左侧为请求,右侧为响应。

请求的第一行,为请求方法及 URL,下方为数据

写入数据

查询

1
2
3
4
5
6
GET index_article/_search
{
"query": {
"match_all": {}
}
}

curl

参数

1
2
-X	指定请求方法
-H 添加 Header,添加多个可通过重复指定

示例

1
2
3
4
5
6
7
8
# 格式
curl [options] [URL...]

# 指定方法
curl -X POST [URL...]

# 添加 Header
curl -H "Content-Type: application/json" [URL...]

nginx 日志分析

日志文件合并

1
2
# 对于 gz 格式日志文件,使用 gzcat
$ gzcat access-*.gz > access-all.log

分析

goaccess

安装

使用 brew 或下载

1
$ brew install goaccess

日志格式匹配

nginx 日志格式配置需要转换为 goaccess 可识别的日志格式,下面是示例。

nginx 配置 goaccess 格式配置
$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" "$request_body" "$upstream_addr" $upstream_status $upstream_response_time $request_time log-format %h %^ [%d:%t %^] "%r" %s %b "%R" "%u" "%^" "%^" "%^" %^ %^ %T
date-format %d/%b/%Y
time-format %T
$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time $upstream_addr $upstream_response_time $pipe %h %^[%d:%t %^] "%r" %s %b "%R" "%u" %T %^

日志格式配置

分析前,需要配置日志格式。日志格式可以在 ~/.goaccessrc 文件配置,也可以在命令行中指定。

1
2
3
4
5
6
7
8
# ~/.goaccessrc
log-format %h %^[%d:%t %^] "%r" %s %b "%R" "%u" "%^" "%^" "%^" %^ %^ %T
date-format %d/%b/%Y
time-format %T


# 使用命令行指定
goaccess access-all.log --log-format='%h %^[%d:%t %^] "%r" %s %b "%R" "%u" "%^" "%^" "%^" %^ %^ %T' --date-format='%d/%b/%Y' --time-format='%T'

离线分析

1
2
3
4
5
# 使用 ~/.goaccessrc 文件配置日志格式
$ LANG="en_US.UTF-8" goaccess access-all.log -o report.html

# 使用命令行指定日志格式
$ LANG="en_US.UTF-8" goaccess access-all.log --log-format='%h %^[%d:%t %^] "%r" %s %b "%R" "%u" "%^" "%^" "%^" %^ %^ %T' --date-format='%d/%b/%Y' --time-format='%T' -o report.html

如果出错,可以尝试去掉 LANG="en_US.UTF-8"

spring boot maven

单个可执行 jar 包

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.3.0</version>
<configuration>
<archive>
<manifest>
<mainClass>pub.wii.cook.springboot.CookSpringBootApplication</mainClass>
</manifest>
</archive>
<descriptors> <!-- 会被忽略 -->
<descriptor>src/assemble/assembly.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>make-assembly</id> <!-- this is used for inheritance merges -->
<phase>package</phase> <!-- bind to the packaging phase -->
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>

assemble.xml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0
http://maven.apache.org/xsd/assembly-2.1.0.xsd">
<id>jar-with-dependencies</id>
<formats>
<format>jar</format>
</formats>

<includeBaseDirectory>false</includeBaseDirectory>
<dependencySets>
<dependencySet>
<outputDirectory>/</outputDirectory>
<useProjectArtifact>true</useProjectArtifact>
<unpack>true</unpack>
<scope>runtime</scope>
</dependencySet>
</dependencySets>

<containerDescriptorHandlers>
<containerDescriptorHandler>
<handlerName>metaInf-services</handlerName>
</containerDescriptorHandler>
<containerDescriptorHandler>
<handlerName>metaInf-spring</handlerName>
</containerDescriptorHandler>
<containerDescriptorHandler>
<handlerName>plexus</handlerName>
</containerDescriptorHandler>
</containerDescriptorHandlers>
</assembly>