1、集群安装
参考：https://www.cnblogs.com/a120608yby/p/17295938.html
2、业务及管理网络创建
参考：https://www.cnblogs.com/a120608yby/p/17140953.html
3、Ceph网络创建
# 配置ceph多网卡绑定
# vim /etc/network/interfaces
...
auto bond1
iface bond1 inet manual
ovs_bridge vmbr1
ovs_type OVSBond
ovs_bonds enp2s0f2 enp2s0f3
ovs_options bond_mode=balance-tcp lacp=active other_config:lacp-time=fast
...
# 创建ceph网络
# vim /etc/network/interfaces
...
auto ceph
iface ceph inet static
address 10.10.10.101/24
ovs_type OVSIntPort
ovs_bridge vmbr0
ovs_options tag=111
...
# 创建OVS Bridge Ceph网络
# vim /etc/network/interfaces
auto vmbr1
iface vmbr1 inet manual
ovs_type OVSBridge
ovs_ports bond1 ceph
# 最终配置
# vim /etc/network/interfaces
# network interface settings; autogenerated
# Please do NOT modify this file directly, unless you know what
# you're doing.
#
# If you want to manage parts of the network configuration manually,
# please utilize the 'source' or 'source-directory' directives to do
# so.
# PVE will preserve these directives, but will NOT read its network
# configuration from sourced files, so do not attempt to move any of
# the PVE managed interfaces into external files!
auto lo
iface lo inet loopback
auto enp2s0f0
iface enp2s0f0 inet manual
auto enp2s0f1
iface enp2s0f1 inet manual
auto enp2s0f2
iface enp2s0f2 inet manual
auto enp2s0f3
iface enp2s0f3 inet manual
iface enp11s0 inet manual
iface enp12s0 inet manual
auto mgt
iface mgt inet static
address 192.168.0.101/24
gateway 192.168.0.254
ovs_type OVSIntPort
ovs_bridge vmbr0
ovs_options tag=110
#mgt
auto ceph
iface ceph inet static
address 10.10.10.101/24
ovs_type OVSIntPort
ovs_bridge vmbr1
ovs_options tag=111
#ceph
auto bond0
iface bond0 inet manual
ovs_bridge vmbr0
ovs_type OVSBond
ovs_bonds enp2s0f0 enp2s0f1
ovs_options bond_mode=balance-tcp lacp=active other_config:lacp-time=fast
auto bond1
iface bond1 inet manual
ovs_bridge vmbr1
ovs_type OVSBond
ovs_bonds enp2s0f2 enp2s0f3
ovs_options bond_mode=balance-tcp lacp=active other_config:lacp-time=fast
#ceph
auto vmbr0
iface vmbr0 inet manual
ovs_type OVSBridge
ovs_ports bond0 mgt
auto vmbr1
iface vmbr1 inet manual
ovs_type OVSBridge
ovs_ports bond1 ceph
4、华为交换机配置
interface Eth-Trunk1 description link_to_pve01 port link-type trunk undo port trunk allow-pass vlan 1 port trunk allow-pass vlan 2 to 4094 mode lacp interface Eth-Trunk2 description link_to_pve01_ceph port link-type access port default vlan 111 mode lacp interface GigabitEthernet0/0/7 description link_to_pve01 eth-trunk 1 interface GigabitEthernet0/0/8 description link_to_pve01 eth-trunk 1 interface GigabitEthernet0/0/9 description link_to_pve01_ceph eth-trunk 2 interface GigabitEthernet0/0/10 description link_to_pve01_ceph eth-trunk 2
5、配置ceph安装源
# vim /etc/apt/sources.list.d/ceph.list deb https://mirrors.ustc.edu.cn/proxmox/debian/ceph-quincy bullseye main
6、ceph安装
pveceph install
7、初始化ceph配置
pveceph init --network 10.10.10.0/24
8、创建Monitor（三个节点）
pveceph mon create
9、创建Manager（另外两个节点）
pveceph mgr create
10、创建OSD（三个节点）
pveceph osd create /dev/sdb pveceph osd create /dev/sdc
11、创建Pool（其中一个节点）
pveceph pool create vm --add_storages
12、配置PG Autoscaler
ceph mgr module enable pg_autoscaler
13、创建CephFS（三个节点）
# 创建服务 pveceph mds create # 增加配置 # cat /etc/pve/ceph.conf [global] ... mds standby replay = true ... # 创建cephfs pveceph fs create --pg_num 128 --add-storage
参考：https://pve.proxmox.com/wiki/Deploy_Hyper-Converged_Ceph_Cluster