Blog

vim 重复之前的 :命令
执行:替换或其他操作后想要再次执行
在记录里搜索命令并执行
```
q:
然后上下滑动，选取要重复执行的命令
```
或者
使用 @: 来重复上次的 : 命令
https://vim.fandom.com/wiki/Repeat_last_colon_command
2020 年 3 月 31 日

ldapsearch 测试 ldap 筛选

ldapsearch -D "uid=svc-stp,cn=users,cn=accounts,dc=ipa,dc=mycorp,dc=com" -b "cn=users,cn=accounts,dc=ipa,dc=mycorp,dc=com"  -h haipa5.dc.mycorp.com -p 389 -w "password" -s sub  > all

ldapsearch -D "uid=svc-stp,cn=users,cn=accounts,dc=ipa,dc=mycorp,dc=com" -b "cn=users,cn=accounts,dc=ipa,dc=mycorp,dc=com"  -h haipa5.dc.mycorp.com -p 389 -w "password" -s sub  '(&(objectclass=posixAccount)(memberOf=cn=ipausers,cn=groups,cn=accounts,dc=ipa,dc=mycorp,dc=com)(!(nsAccountLock=TRUE)))'

ldapsearch -D "uid=svc-stp,cn=users,cn=accounts,dc=ipa,dc=mycorp,dc=com" -b "cn=users,cn=accounts,dc=ipa,dc=mycorp,dc=com"  -h haipa5.dc.mycorp.com -p 389 -w "password" -s sub  '(&(uid=*)(!(ipaUserAuthType=*))(!(nsAccountLock=TRUE)))'

2020 年 3 月 28 日

Ceph 笔记

# 列出所有的pool
ceph osd lspools
# 创建一个 pool
ceph osd pool create kong 3
# 列出pool中的所有镜像
rbd ls kong
# 创建pool
ceph osd pool create
# 创建一个大小为1G的镜像
rbd create -p tenant_1 xm3gyumgqupmexxmewoome
# 显示一个镜像的详细信息
rbd info -p tenant_1 –image xm3gyumgqupmexxmewoome
# 删除一个镜像
rdb rm -p tenant_1 xm3gyumgqupmexxmewoome

cat /etc/ceph/ceph.client.admin.keyring
ceph auth list
ceph osd pool ls detail
ceph osd pool delete kong
ceph osd pool delete kong kong –yes-i-really-really-mean-it

# k8s查看最近创建的容器
kubectl get pods -n 1 | awk ‘$5 ~ /s|m/ {print}’ | sort -V -k 5

# docker删除exit的容器
docker ps -a | grep Exited | awk ‘{print $1}’ | xargs docker rm

2020 年 3 月 21 日

k8s flannel vxlan

用到的知识:

linux bridge
linux network namespace
flannel backend
iptables

host 上的 flannel 配置文件

[root@k8snode028 bin]# cat /etc/cni/net.d/10-flannel.conflist 
{
  "name":"cni0",
  "cniVersion":"0.3.1",
  "plugins":[
    {
      "type":"flannel",
      "delegate":{
        "forceAddress":true,
        "hairpinMode": true,
        "isDefaultGateway":true
      }
    },
    {
      "type":"portmap",
      "capabilities":{
        "portMappings":true
      }
    }
  ]
}

host 上的 flannel 相关 bin 文件

ls /opt/cni/bin/
bridge  cnitool  dhcp  flannel  host-local  ipvlan  loopback  macvlan  noop  portmap  ptp  tuning  vlan

查看 namespace

ip netns list

查看 link

ip link

创建 red 的 namespace，并查看 link

ip netns exec red ip link

kubernetes host 上的 interface, ifconfig

...
cni0: flags=4163  mtu 1450
        inet 10.27.9.1  netmask 255.255.255.0  broadcast 0.0.0.0
...

flannel.1: flags=4163  mtu 1450
        inet 10.27.9.0  netmask 255.255.255.255  broadcast 0.0.0.0
...

kubernetes pod 内的 arp route 显示

[root@backend-6b7b58986c-jdnhn spring]# route  -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.27.9.1       0.0.0.0         UG    0      0        0 eth0
10.27.0.0       10.27.9.1       255.255.0.0     UG    0      0        0 eth0
10.27.9.0       0.0.0.0         255.255.255.0   U     0      0        0 eth0

node 节点上的 bridge 信息

yum install bridge-utils -y
[root@k8snode028 ~]# brctl show
bridge name	bridge id		STP enabled	interfaces
cni0		8000.0a580a1d0d01	no		veth0985a0b8
							veth1253dd53
							veth127dc912
							veth15b89d85
							veth30b9e182
							veth3a6a0f27
							veth500fcaf1
							veth5f994152
							veth6374a5e2
							veth67f0cb2e
							veth69772276
							veth6a95a62c
							veth6c8897b0
							veth781dd728
							veth88eddba5
							veth8d28dfe8
							veth8fd03c3e
							veth8fe5c104
							vetha529244f
							vethb2b2134b
							vethc070358a
							vethccd69c78
							vethd6f2ae04
							vethdbaec209
							vethe0e0bd3a
							vethe3939f54
							vethec7d63a5
							vethefdaa81a
							vethf836c26b
							vethfb0f86bc
							vethfec66a08
docker0		8000.0242fbd0e139	no

node 节点上的路由信息

[root@k8snode028 network-scripts]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.18.19.1      0.0.0.0         UG    0      0        0 bond0
10.18.19.0      0.0.0.0         255.255.255.0   U     0      0        0 bond0
10.27.0.0       10.27.0.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.1.0       10.27.1.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.2.0       10.27.2.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.3.0       10.27.3.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.4.0       10.27.4.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.5.0       10.27.5.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.6.0       10.27.6.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.7.0       10.27.7.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.8.0       10.27.8.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.9.0       0.0.0.0         255.255.255.0   U     0      0        0 cni0
10.27.10.0      10.27.10.0      255.255.255.0   UG    0      0        0 flannel.1
10.27.11.0      10.27.11.0      255.255.255.0   UG    0      0        0 flannel.1
10.27.12.0      10.27.12.0      255.255.255.0   UG    0      0        0 flannel.1
10.27.13.0      10.27.13.0      255.255.255.0   UG    0      0        0 flannel.1
10.27.14.0      10.27.14.0      255.255.255.0   UG    0      0        0 flannel.1
169.254.0.0     0.0.0.0         255.255.0.0     U     1006   0        0 bond0
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0

node 节点上查看 bridge (相当于本机虚拟交换机)

ip link show type bridge 或 bridge link show
7: docker0:  mtu 1500 qdisc noqueue state UP mode DEFAULT group default 
    link/ether 02:42:1c:85:aa:91 brd ff:ff:ff:ff:ff:ff
12: cni0:  mtu 1450 qdisc noqueue state UP mode DEFAULT group default qlen 1000
    link/ether 0a:58:0a:1b:09:01 brd ff:ff:ff:ff:ff:ff

node 节点上查看所有的 veth

ip link show type veth
25: veth7443cee@if24:  mtu 1500 qdisc noqueue master docker0 state UP mode DEFAULT group default 
    link/ether 2e:03:af:2d:5e:46 brd ff:ff:ff:ff:ff:ff link-netnsid 5
...

node 上的 flannel 容器里的信息

[root@k8snode028 bin]# docker exec 1a84ab994e5d cat /run/flannel/subnet.env
FLANNEL_NETWORK=10.27.0.0/16
FLANNEL_SUBNET=10.27.9.1/24
FLANNEL_MTU=1450
FLANNEL_IPMASQ=true

从一台 node pod 到另一台 node pod 网段(10.29.13.0)，查询找到 mac 地址

arp -e | grep 10.29.13.0
10.29.13.0               ether   42:32:74:de:24:50   CM                    flannel.1

再通过 mac 地址找到是哪台目标 node

bridge fdb show | grep 42:32:74:de:24:50
42:32:74:de:24:50 dev flannel.1 dst 10.18.19.129 self permanent

参考:
https://xuxinkun.github.io/2019/06/05/flannel-vxlan/

2020 年 3 月 14 日

使用联通线路 ssh 拉取 github 代码
联通线路访问比 github 快多了

laptop => 联通ssh server => [email protected]

cat ~/.ssh/config
```
Host github.com
ProxyCommand ssh -l jpuyy -q ddns.of.liantong.yyhome.dmzhost.org -p 22 nc %h %p
```
2020 年 3 月 10 日
接入 k8s 遇到的问题

从传统容器接入 k8s 过程中遇到的问题：

1. 某 sdk 要升级（低版本的会导致 istio 容器挂掉）
报错为 Caused by: java.io.IOException: Cannot bind to URL [rmi:///jmxrmi]: javax.naming.CommunicationException [Root exception is java.rmi.ConnectIOException: error during JRMP connection establishment; nested exception is

2. 客户端 HTTP 对外请求被 envoy rule deny 400( bad request ), 原因 HTTP header 里出现了空的 key:value，客户端修复后，问题消失。以下是抓包，见 Content-Type 和 Accept 中间的一行
14:08:37.918970 IP 10.18.19.98.51604 > lb008-dev.http: Flags [P.], seq 1:489, ack 1, win 229, options [nop,nop,TS val 1596856343 ecr 1593089157], length 488: HTTP: POST /ws/rs/domain/domain/init HTTP/1.1
E….J@.?…
..b
..7…P. .].*……+\…..
_…^…POST /ws/rs/domain/domain/init HTTP/1.1
Content-Type: application/json
:
Accept: application/json
api-uuid: 02ac3ebe-f212-4ca8-998e-4a4ab576018c
api-control-request-type: ANONYMOUS
User-Agent: Apache CXF 3.1.4
Cache-Control: no-cache
Pragma: no-cache
Host: uniauthserver-dev
Connection: keep-alive
Content-Length: 407
解决：修复上面 : k,v 都是空的行

3. 如果想要使用 jaeger 进行分布式 tracing，可以参考 https://istio.io/zh/docs/tasks/telemetry/distributed-tracing/overview/

4. kiali 出现 unknown 的调用链：没有通过 service mesh 的调用，会显示 unknown

5. k8snode kernel 版本问题
kernel版本过低会导致docker报错，kernel:unregister_netdevice: waiting for eth0 to become free. Usage count = 1
会导致系统cpu占用过高，docker容器都会卡住。
Observed kernel versions with this issue
RHEL7 3.10.0-862
4.15.0
4.20.0
Kernel versions claimed not triggering this issue
RHEL7 3.10.0-957.10.1
4.19.12
4.17.0
4.17.11
Related kernel commits
torvalds/linux@f186ce6 – since 4.12
torvalds/linux@4ee806d – since 4.15
torvalds/linux@ee60ad2 – since 5.1

另一个表现为 kubectl get pods –all-namespace -o wide 发现 pods 长时间一直 Terminating，删不掉

解决： yum update ( 升级 kernel 和操作系统至最新版 kernel 3.10.0-957.21.3.el7)

6. 请求的 url 出现 no healthy upstream( http 503 错误) 检查是否发布成功

7. 请求 url 出现 404 （业务发布是成功的) ，检查 k8s 内部的 virtual service 和 ingress gateway 是否配置正确

8. node 程序因为 k8s 注入的环境变量太多（k8s服务发现机制)，导致 node process.env 长度太长，报错启动失败。

目前遇到问题的有 frontend-main, market-solution-activity-web。还没找到不改程序的解决办法。改程序的解决办法是只取自己需的 process.env https://zhuanlan.zhihu.com/p/74056339

[2019-07-30 16:54:13] PM2 error: Trace: { Error: spawn E2BIG
at exports._errnoException (util.js:1024:11)
at ChildProcess.spawn (internal/child_process.js:325:11)
at exports.spawn (child_process.js:493:9)
at exports.fork (child_process.js:99:10)
at createWorkerProcess (internal/cluster/master.js:127:10)
at EventEmitter.cluster.fork (internal/cluster/master.js:161:25)
at Object.nodeApp (/opt/nodeapp/node_modules/pm2/lib/God/ClusterMode.js:52:21)
at Object.executeApp (/opt/nodeapp/node_modules/pm2/lib/God.js:159:9)
at inject (/opt/nodeapp/node_modules/pm2/lib/God.js:418:18)
at Object.injectVariables (/opt/nodeapp/node_modules/pm2/lib/God.js:530:10) code: ‘E2BIG’, errno: ‘E2BIG’, syscall: ‘spawn’ }
at Object.God.logAndGenerateError (/opt/nodeapp/node_modules/pm2/lib/God/Methods.js:36:15)
at Object.nodeApp (/opt/nodeapp/node_modules/pm2/lib/God/ClusterMode.js:54:11)
at Object.executeApp (/opt/nodeapp/node_modules/pm2/lib/God.js:159:9)
at inject (/opt/nodeapp/node_modules/pm2/lib/God.js:418:18)
at Object.injectVariables (/opt/nodeapp/node_modules/pm2/lib/God.js:530:10)
at /opt/nodeapp/node_modules/pm2/lib/God.js:416:9
at /opt/nodeapp/node_modules/pm2/node_modules/async/dist/async.js:1135:9
at replenish (/opt/nodeapp/node_modules/pm2/node_modules/async/dist/async.js:1011:17)
at /opt/nodeapp/node_modules/pm2/node_modules/async/dist/async.js:1016:9
at _asyncMap (/opt/nodeapp/node_modules/pm2/node_modules/async/dist/async.js:1133:5)
[2019-07-30 16:54:13] PM2 error: spawn E2BIG

9. 用 flannel + host-gw 阿里云不支持自己定义的 route ，否则需要手动添加路由，换为 vxlan

[[email protected] kubespray]# ansible all -i inventory/k8s_prod_aliyun-cn-shanghai-b_006/inventory.ini -m shell -a “ping -c 3 10.36.3.4”
[WARNING]: Invalid characters were found in group names but not replaced, use -vvvv to see details
k8snode034-prod.aliyun-cn-shanghai-b | CHANGED | rc=0 >>
PING 10.36.3.4 (10.36.3.4) 56(84) bytes of data.
64 bytes from 10.36.3.4: icmp_seq=1 ttl=64 time=0.066 ms
64 bytes from 10.36.3.4: icmp_seq=2 ttl=64 time=0.068 ms
64 bytes from 10.36.3.4: icmp_seq=3 ttl=64 time=0.067 ms
— 10.36.3.4 ping statistics —
3 packets transmitted, 3 received, 0% packet loss, time 1999ms
rtt min/avg/max/mdev = 0.066/0.067/0.068/0.000 ms
k8smaster016-prod.aliyun-cn-shanghai-b | FAILED | rc=1 >>
PING 10.36.3.4 (10.36.3.4) 56(84) bytes of data.
— 10.36.3.4 ping statistics —
3 packets transmitted, 0 received, 100% packet loss, time 2000msnon-zero return code
k8smaster015-prod.aliyun-cn-shanghai-b | FAILED | rc=1 >>
PING 10.36.3.4 (10.36.3.4) 56(84) bytes of data.
— 10.36.3.4 ping statistics —
3 packets transmitted, 0 received, 100% packet loss, time 1999msnon-zero return code
k8smaster014-prod.aliyun-cn-shanghai-b | FAILED | rc=1 >>
PING 10.36.3.4 (10.36.3.4) 56(84) bytes of data.
— 10.36.3.4 ping statistics —
3 packets transmitted, 0 received, 100% packet loss, time 2000msnon-zero return code

10. 有的应用需要自己拨 vpn 连到其他网络，有状态，不能接入

11. k8s里面，java应用通过 Runtime.getRuntime().availableProcessors()拿到的核数为1，这样使用这个设置线程池的大小会变成1，按照之前docker的情况应该市返回宿主机核数

12. 有的暂时不接入 k8s ，暴露端口为 tcp (非 http )，发布系统生成的 istio 配置均为 http，后续考虑

13. pod STATUS CreateContainerConfigError

2020 年 3 月 8 日

Blog

vim 重复之前的 :命令

ldapsearch 测试 ldap 筛选

Ceph 笔记

k8s flannel vxlan

使用联通线路 ssh 拉取 github 代码

接入 k8s 遇到的问题