flink

Protobuf

设置自定义序列化

引入依赖

1
2
3
4
5
<dependency>
<groupId>com.twitter</groupId>
<artifactId>chill-protobuf</artifactId>
<version>0.10.0</version>
</dependency>

设置自定义序列化类

1
2
3
import com.twitter.chill.protobuf.ProtobufSerializer

env.getConfig.registerTypeWithKryoSerializer(classOf[PbMessage], classOf[ProtobufSerializer])

参数

1
2
3
4
5
6
7
8
9
10
-m    address of job manager which to connect
-d detached mode
-yd yarn detached mode
-ynm yarn application name
-yn number of yarn container to allocate = Number of Task Managers
-ys Number of slots per TaskManager
-yjm Memory for JobManager Container with optional unit (default: MB)
-ytm Memory per TaskManager Container with optional unit (default: MB)
-p The parallelism with which to run the program.
-c class to run

cassandra - cql

语法

CQL

Datacenter

1
2
use system;
select data_center from local;

Role / User

创建

1
2
3
4
5
CREATE ROLE [IF NOT EXISTS] role_name 
[WITH SUPERUSER = true | false
| LOGIN = true | false
| PASSWORD = 'password'
| OPTIONS = option_map]

示例

1
2
CREATE ROLE IF NOT EXISTS 'root' WITH SUPERUSER = true AND LOGIN = false;
CREATE ROLE IF NOT EXISTS 'root' WITH SUPERUSER = true AND PASSWORD = 'root';

修改权限

1
2
3
4
5
ALTER ROLE role_name 
[WITH [PASSWORD = 'password']
[LOGIN = true | false]
[SUPERUSER = true | false]
[OPTIONS = map_literal]]

示例

1
ALTER ROLE root WITH PASSWORD = 'NewPassword';

Key Space

查询

1
2
# 列出所有 keysapces
desc keyspaces;

创建

1
2
3
4
5
6
7
CREATE  KEYSPACE [IF NOT EXISTS] keyspace_name 
WITH REPLICATION = {
'class' : 'SimpleStrategy', 'replication_factor' : N }
| 'class' : 'NetworkTopologyStrategy',
'dc1_name' : N [, ...]
}
[AND DURABLE_WRITES = true|false] ;

示例

1
2
3
4
CREATE  KEYSPACE IF NOT EXISTS "trace" WITH REPLICATION = {
'class' : 'NetworkTopologyStrategy',
'vg' : 2
}

Table

查询

1
desc <keyspace-name>;
1
2
use <keyspace-name>;
desc tables;

创建

1
2
3
4
5
6
7
CREATE TABLE [IF NOT EXISTS] [keyspace_name.]table_name ( 
column_definition [, ...]
PRIMARY KEY (column_name [, column_name ...])
[WITH table_options
| CLUSTERING ORDER BY (clustering_column_name order])
| ID = 'table_hash_tag'
| COMPACT STORAGE]

数据类型参考这里

删除

1
DROP TABLE [IF EXISTS] keyspace_name.table_name

修改

添加列

1
2
3
4
5
6
ALTER TABLE [keyspace_name.] table_name 
[ALTER column_name TYPE cql_type]
[ADD (column_definition_list)]
[DROP column_list | COMPACT STORAGE ]
[RENAME column_name TO column_name]
[WITH table_properties];

示例

1
ALTER TABLE trace.ranker ADD rid TEXT;

Query

1
2
3
// 查询一条
select * from ks.table limit 1;
select * from ks.table where id='value' limit 1;

kudu - usage

安装

新版本安装参考这里

centos

build

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#!/bin/bash
sudo yum -y install autoconf automake curl cyrus-sasl-devel cyrus-sasl-gssapi \
cyrus-sasl-plain flex gcc gcc-c++ gdb git java-1.8.0-openjdk-devel \
krb5-server krb5-workstation libtool make openssl-devel patch pkgconfig \
redhat-lsb-core rsync unzip vim-common which
sudo yum -y install centos-release-scl-rh
sudo yum -y install devtoolset-8
git clone https://github.com/apache/kudu
cd kudu
build-support/enable_devtoolset.sh thirdparty/build-if-necessary.sh
mkdir -p build/release
cd build/release
../../build-support/enable_devtoolset.sh \
../../thirdparty/installed/common/bin/cmake \
-DCMAKE_BUILD_TYPE=release \
../..
make -j4

对于 centos 7 默认 gcc 是 4.8,用 yum install gcc72-c++ 升级到 gcc-g++ 7.2

package

1
2
3
4
# 下载 repo 配置
$ wget http://archive.cloudera.com/kudu/redhat/7/x86_64/kudu/cloudera-kudu.repo -P /etc/yum.repos.d/
# 安装
$ yum update & yum install kudu

linux - crontab

命令格式

1
2
3
MIN HOUR DOM MON DOW CMD
# DOM: day of month
# DOW: day of week

查看

1
$ crontab -l

编辑

1
$ crontab -e

启动时运行

1
@reboot /home/ubuntu/app/consul/start.sh

时间格式

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
MIN HOUR DOM MON DOW CMD
* * * * * ...

# 特殊符号
* 任意值
, 多值分隔符
- 范围分隔符
/ 间隔符 (step values)

# 示例
* * * * * 每分钟
30 08 10 06 * 每年的 6 月 10 号 8 点 30 分
30 * * * * 每小时的 30 分
00 11,16 * * * 每天的 11:00 和 16:00
00 09-18 * * * 每天的 9、10 ... 18 点 00 分
00 09-18 * * 1-5 周一至周五每天的 9、10 ... 18 点 00 分
*/10 * * * * 每十分钟

设置 PATH

1
2
$ crontab -e # 开头添加如下内容
PATH=/bin:/sbin:/usr/bin:/usr/sbin:/home/ubuntu/.local/bin

python - generate wheel

安装

1
$ pip3 install wheel setuptools

创建 setup.py

1
2
3
4
5
6
7
from setuptools import setup

setup(
name='abtest',
version='1.0',
packages=['.abtest'],
)

setup.py 示例

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
import os.path

from setuptools import setup

REQUIREMENTS = 'requirements.txt'
requires = []
if os.path.isfile(REQUIREMENTS):
with open(REQUIREMENTS, 'r') as f:
requires = f.read().splitlines()

# VERSION i.e. 1.3
with open('VERSION', 'r') as f:
v = f.read()
setup(
name='abtest',
version=v.strip(),
packages=['abtest', 'data', 'model', 'utils'], # 指定 package
install_requires=requires # 指定依赖基本版本
)

打包工程

1
python setup.py bdist_wheel --universal