Notes 基于 IP 哈希的路由

基于 IP 哈希的路由的运行方式是获取源和目标 IP 地址，然后在每个数据包上执行数学计算以确定要使用成组中的哪个上行链路。由于负载平衡基于源/目标 IP 地址，因此单个虚拟机与多个 IP 地址通信可以跨成组中的所有网络适配器平衡其负载，并更好的利用可用带宽。

基于 IP 哈希的路由负载平衡需要多个已组合到 EtherChannel（有时也称为聚合绑定、端口通道或中继）中的物理交换机端口。这确保了同一哈希算法用于返回相反方向的流量。 The switch must be set to perform 802.3ad link aggregation in static mode ON

优点：

* 单个虚拟机与多个虚拟机通信的情况下的性能已提高。理论上，虚拟机可使用超过一个物理网卡所提供的带宽。 * 网络适配器冗余：如果网卡出现故障，并且链路状态已断开，则成组中的其余网卡将继续传递流量。为了使链路状态正常运行，物理交换机和 ESXi 主机都必须将链路识别为断开。如果存在出入，则无法将流量故障切换到成组中的其他适配器。

缺点：

* 物理交换机配置较不灵活，并且需要为静态 EtherChannel 绑定配置物理交换机端口。有关详细信息，请参见 Sample configuration of EtherChannel / Link Aggregation Control Protocol (LACP) with ESXi/ESX and Cisco/HP switches (1004048)。 * 网卡组仅可使用一个物理交换机，因为大多数交换机都不支持跨多个物理交换机的 EtherChannel 绑定。这将防止物理硬件冗余。

注意：存在一些例外情况，因为某些“堆栈”交换机或模块交换机可跨多个物理交换机或模块执行此操作。Cisco 的 VPC（虚拟端口通道）技术也可在受支持的交换机上解决此问题。有关详细信息，请联系您的硬件供应商。 IRF，堆叠技术。

* IP 哈希不支持信号探测。仅可将链路状态用作故障检测方法。如果链路在其没有断开的情况下出现故障，则无法避免 vSwitch 上的网络通信问题。 * 标准 vSwitch 仅支持静态模式 EtherChannel。LACP 仅在 vSphere 5.1 的 vSphere Distributed Switch 中或 Cisco Nexus 1000V 中受支持。

Cisco switch setting

建议在 ESXi/ESX 网络环境中实施虚拟局域网 (VLAN)，这是因为：

它将 ESXi/ESX 集成到预先存在的网络中 它可保障网络流量 它可降低网络流量拥塞

Port Channel/Aggr

configure terminal
interface GigabitEthernet1/11
description VMware_MGT
switchport mode access
switchport access vlan x 
spanning-tree portfast

Port Channel/Aggr

First Enable LACP

configure terminal
switch (config)# feature lacp 
show system internal clis feature
show port-channel summary
show port-channel database
show port-channel load-balance

!
configure terminal
interface port-channel 1
description VMware_VM_Net
switchport trunk encapsulation dot1q
switchport mode trunk
!

interface GigabitEthernet1/13
switchport trunk encapsulation dot1q
switchport mode trunk
switchport trunk allowed vlan x,y,z
spanning-tree portfast trunk 
channel-group 1 mode on

Oracle RAC

建议创建+CRS磁盘组(OCR和表决文件)使用正常冗余(Normal)。

建议将其他所有ASM磁盘组设置为外部冗余(Extenal)。

建议将+DATA、+FRA和+LOG ASM磁盘组分开，位于不同的阵列，以减轻和平衡I/O负担。

SAN_01 & SAN_03

switchdisable , cfgshow, cfgdisable , cfgclear, configure and set the fabric.ops parametres similerly as the existing one except the DOMAIN ID(it should be unique 1-239, by default it is 1 in each new SW), cfgsave.

Inter Switch Link (ISL) Setting

1. 升级光纤交换机firmware版本
2. 初始化配制，设置新交换机管理IP并分别修改domain id为1和3。
3. 完成光纤连接。
4. 确认ISL情况

  switchshow
  islshow
  fabricshow

创建别名

   1 alicreate "VPLEX_E1_A1_F0",""
   2 alicreate "VPLEX_E1_B1_F0",""
   3 alicreate "VPLEX_E1_A0_F2",""
   4 alicreate "VPLEX_E1_B0_F2",""
   5 alicreate "ST01_A0",""
   6 alicreate "ST01_B0",""
   7 alicreate "ST02_A0",""
   8 alicreate "ST02_B0",""
   9 alicreate "TA_SW01_F1",""
  10 alicreate "TA_SW02_F1",""
  11 
  12 alicreate "VPLEX_E1_A0_F0",""
  13 alicreate "VPLEX_E1_B0_F0",""
  14 alicreate "VPLEX_E1_A1_F2",""
  15 alicreate "VPLEX_E1_B1_F2",""
  16 alicreate "ST03_A0",""
  17 alicreate "ST03_B0",""
  18 alicreate "ST04_A0",""
  19 alicreate "ST04_B0",""
  20 alicreate "ESXi01_F1",""
  21 alicreate "ESXi02_F1",""
  22 alicreate "ESXi03_F1",""
  23 alicreate "ESXi04_F1",""
  24 alicreate "DB_RAC01_F1",""
  25 alicreate "DB_RAC02_F1",""

ZONE划分

   1 # Front Zone
   2 # VPLEX_E1_A0 with APPs
   3 zonecreate "VPLEX_DB_RAC01","DB_RAC01_F1,VPLEX_E1_A0_F0,VPLEX_E1_B0_F0"
   4 zonecreate "VPLEX_DB_RAC02","DB_RAC02_F1,VPLEX_E1_A0_F0,VPLEX_E1_B0_F0"
   5 zonecreate "VPLEX_ESXi01","ESXi01_F1,VPLEX_E1_A0_F0,VPLEX_E1_B0_F0"
   6 zonecreate "VPLEX_ESXi02","ESXi02_F1,VPLEX_E1_A0_F0,VPLEX_E1_B0_F0"
   7 zonecreate "VPLEX_ESXi03","ESXi03_F1,VPLEX_E1_A0_F0,VPLEX_E1_B0_F0"
   8 zonecreate "VPLEX_ESXi04","ESXi04_F1,VPLEX_E1_A0_F0,VPLEX_E1_B0_F0"
   9 
  10 zonecreate "VPLEX_TA_SW01","TA_SW01_F1,VPLEX_E1_A0_F2,VPLEX_E1_B0_F2"
  11 zonecreate "VPLEX_TA_SW02","TA_SW02_F1,VPLEX_E1_A0_F2,VPLEX_E1_B0_F2"
  12 # Back Zone
  13 # VPLEX_E1_A1 with VNX5200
  14 zonecreate "VPLEX_E1_A1_F0_ST01","VPLEX_E1_A1_F0,ST01_A0,ST01_B0"
  15 zonecreate "VPLEX_E1_A1_F0_ST02","VPLEX_E1_A1_F0,ST02_A0,ST02_B0"
  16 zonecreate "VPLEX_E1_A1_F0_ST03","VPLEX_E1_A1_F0,ST03_A0,ST03_B0"
  17 zonecreate "VPLEX_E1_A1_F0_ST04","VPLEX_E1_A1_F0,ST03_A0,ST04_B0"
  18 
  19 zonecreate "VPLEX_E1_A1_F2_ST01","VPLEX_E1_A1_F2,ST01_A0,ST01_B0"
  20 zonecreate "VPLEX_E1_A1_F2_ST02","VPLEX_E1_A1_F2,ST02_A0,ST02_B0"
  21 zonecreate "VPLEX_E1_A1_F2_ST03","VPLEX_E1_A1_F2,ST03_A0,ST03_B0"
  22 zonecreate "VPLEX_E1_A1_F2_ST04","VPLEX_E1_A1_F2,ST04_A0,ST04_B0"
  23 
  24 zonecreate "VPLEX_E1_B1_F0_ST01","VPLEX_E1_B1_F0,ST01_A0,ST01_B0"
  25 zonecreate "VPLEX_E1_B1_F0_ST02","VPLEX_E1_B1_F0,ST02_A0,ST02_B0"
  26 zonecreate "VPLEX_E1_B1_F0_ST03","VPLEX_E1_B1_F0,ST03_A0,ST03_B0"
  27 zonecreate "VPLEX_E1_B1_F0_ST04","VPLEX_E1_B1_F0,ST04_A0,ST04_B0"
  28 
  29 zonecreate "VPLEX_E1_B1_F2_ST01","VPLEX_E1_B1_F2,ST01_A0,ST01_B0"
  30 zonecreate "VPLEX_E1_B1_F2_ST02","VPLEX_E1_B1_F2,ST02_A0,ST02_B0"
  31 zonecreate "VPLEX_E1_B1_F2_ST03","VPLEX_E1_B1_F2,ST03_A0,ST03_B0"
  32 zonecreate "VPLEX_E1_B1_F2_ST04","VPLEX_E1_B1_F2,ST04_A0,ST04_B0"

VPLEX_E2_A1 with VNX5200 (预留)

zonecreate "VPLEX_E2_A0_F0_ST01","VPLEX_E2_A1_F0,ST01_A0,ST01_B0"
zonecreate "VPLEX_E2_A0_F0_ST02","VPLEX_E2_A1_F0,ST02_A0,ST02_B0"
zonecreate "VPLEX_E2_A0_F0_ST03","VPLEX_E2_A1_F0,ST03_A0,ST03_B0"
zonecreate "VPLEX_E2_A0_F0_ST04","VPLEX_E2_A1_F0,ST04_A0,ST04_B0"

zonecreate "VPLEX_E2_A0_F2_ST01","VPLEX_E2_A1_F2,ST01_A0,ST01_B0"
zonecreate "VPLEX_E2_A0_F2_ST02","VPLEX_E2_A1_F2,ST02_A0,ST02_B0"
zonecreate "VPLEX_E2_A0_F2_ST03","VPLEX_E2_A1_F2,ST03_A0,ST03_B0"
zonecreate "VPLEX_E2_A0_F2_ST04","VPLEX_E2_A1_F2,ST03_A0,ST03_B0"

zonecreate "VPLEX_E2_B0_F0_ST01","VPLEX_E2_B1_F0,ST01_A0,ST01_B0"
zonecreate "VPLEX_E2_B0_F0_ST02","VPLEX_E2_B1_F0,ST02_A0,ST02_B0"
zonecreate "VPLEX_E2_B0_F0_ST03","VPLEX_E2_B1_F0,ST03_A0,ST03_B0"
zonecreate "VPLEX_E2_B0_F0_ST04","VPLEX_E2_B1_F0,ST04_A0,ST04_B0"

zonecreate "VPLEX_E2_B0_F2_ST01","VPLEX_E2_B1_F2,ST01_A0,ST01_B0"
zonecreate "VPLEX_E2_B0_F2_ST02","VPLEX_E2_B1_F2,ST02_A0,ST02_B0"
zonecreate "VPLEX_E2_B0_F2_ST03","VPLEX_E2_B1_F2,ST03_A0,ST03_B0"
zonecreate "VPLEX_E2_B0_F2_ST04","VPLEX_E2_B1_F2,ST04_A0,ST04_B0"

创建并激活CONFIG

# sed -n '/VPLEX_E1/p' < /tmp/tmphSAXQp.moin | grep zonecre | awk -F"\"" '{ print $2 }' | sort | fmt | tr '\n' ' ' | tr ' ' ','

cfgcreate "ZHCIQ_SAN_A","VPLEX_DB_RAC01,VPLEX_DB_RAC02,VPLEX_E1_A1_F0_ST01,VPLEX_E1_A1_F0_ST02,VPLEX_E1_A1_F0_ST03,VPLEX_E1_A1_F0_ST04,VPLEX_E1_A1_F2_ST01,VPLEX_E1_A1_F2_ST02,VPLEX_E1_A1_F2_ST03,VPLEX_E1_A1_F2_ST04,VPLEX_E1_B1_F0_ST01,VPLEX_E1_B1_F0_ST02,VPLEX_E1_B1_F0_ST03,VPLEX_E1_B1_F0_ST04,VPLEX_E1_B1_F2_ST01,VPLEX_E1_B1_F2_ST02,VPLEX_E1_B1_F2_ST03,VPLEX_E1_B1_F2_ST04,VPLEX_ESXi01,VPLEX_ESXi02,VPLEX_ESXi03,VPLEX_ESXi04,VPLEX_TA_SW01,VPLEX_TA_SW02"

# 
cfgenable ZHCIQ_SAN_A
cfgsave ZHCIQ_SAN_A

SAN_02 & SAN_04

Inter Switch Link (ISL) Setting

1. 升级光纤交换机firmware版本
2. 初始化配制，设置新交换机管理IP并分别修改domain id为2和4。
3. 完成光纤连接。
4. 确认ISL情况

  switchshow
  islshow
  fabricshow

创建别名

   1 alicreate "VPLEX_E1_A1_F1",""
   2 alicreate "VPLEX_E1_B1_F1",""
   3 alicreate "VPLEX_E1_A0_F3",""
   4 alicreate "VPLEX_E1_B0_F3",""
   5 alicreate "ST01_A1",""
   6 alicreate "ST01_B1",""
   7 alicreate "ST02_A1",""
   8 alicreate "ST02_B1",""
   9 alicreate "TA_SW01_F2",""
  10 alicreate "TA_SW02_F2",""
  11 
  12 alicreate "VPLEX_E1_A0_F1",""
  13 alicreate "VPLEX_E1_B0_F1",""
  14 alicreate "VPLEX_E1_A1_F3",""
  15 alicreate "VPLEX_E1_B1_F3",""
  16 alicreate "ST03_A1",""
  17 alicreate "ST03_B1",""
  18 alicreate "ST04_A1",""
  19 alicreate "ST04_B1",""
  20 alicreate "ESXi01_F2",""
  21 alicreate "ESXi02_F2",""
  22 alicreate "ESXi03_F2",""
  23 alicreate "ESXi04_F2",""
  24 alicreate "DB_RAC01_F2",""
  25 alicreate "DB_RAC02_F2",""

ZONE划分

   1 # Front Zone
   2 # VPLEX_E1_A0 with APPs
   3 zonecreate "VPLEX_DB_RAC01","DB_RAC01_F2,VPLEX_E1_A0_F1,VPLEX_E1_B0_F1"
   4 zonecreate "VPLEX_DB_RAC02","DB_RAC02_F2,VPLEX_E1_A0_F1,VPLEX_E1_B0_F1"
   5 
   6 zonecreate "VPLEX_ESXi01","ESXi01_F2,VPLEX_E1_A0_F1,VPLEX_E1_B0_F1"
   7 zonecreate "VPLEX_ESXi02","ESXi02_F2,VPLEX_E1_A0_F1,VPLEX_E1_B0_F1"
   8 zonecreate "VPLEX_ESXi03","ESXi03_F2,VPLEX_E1_A0_F1,VPLEX_E1_B0_F1"
   9 zonecreate "VPLEX_ESXi04","ESXi04_F2,VPLEX_E1_A0_F1,VPLEX_E1_B0_F1"
  10 
  11 zonecreate "VPLEX_TA_SW01","TA_SW01_F2,VPLEX_E1_A0_F3,VPLEX_E1_B0_F3"
  12 zonecreate "VPLEX_TA_SW02","TA_SW02_F2,VPLEX_E1_A0_F3,VPLEX_E1_B0_F3"
  13 # Back Zone
  14 # VPLEX_E1_A1 with VNX5200
  15 zonecreate "VPLEX_E1_A1_F1_ST01","VPLEX_E1_A1_F1,ST01_A1,ST01_B1"
  16 zonecreate "VPLEX_E1_A1_F1_ST02","VPLEX_E1_A1_F1,ST02_A1,ST02_B1"
  17 zonecreate "VPLEX_E1_A1_F1_ST03","VPLEX_E1_A1_F1,ST03_A1,ST03_B1"
  18 zonecreate "VPLEX_E1_A1_F1_ST04","VPLEX_E1_A1_F1,ST04_A1,ST04_B1"
  19 
  20 zonecreate "VPLEX_E1_A1_F3_ST01","VPLEX_E1_A1_F3,ST01_A1,ST01_B1"
  21 zonecreate "VPLEX_E1_A1_F3_ST02","VPLEX_E1_A1_F3,ST02_A1,ST02_B1"
  22 zonecreate "VPLEX_E1_A1_F3_ST03","VPLEX_E1_A1_F3,ST03_A1,ST03_B1"
  23 zonecreate "VPLEX_E1_A1_F3_ST04","VPLEX_E1_A1_F3,ST04_A1,ST04_B1"
  24 
  25 zonecreate "VPLEX_E1_B1_F1_ST01","VPLEX_E1_B1_F1,ST01_A1,ST01_B1"
  26 zonecreate "VPLEX_E1_B1_F1_ST02","VPLEX_E1_B1_F1,ST02_A1,ST02_B1"
  27 zonecreate "VPLEX_E1_B1_F1_ST03","VPLEX_E1_B1_F1,ST03_A1,ST03_B1"
  28 zonecreate "VPLEX_E1_B1_F1_ST04","VPLEX_E1_B1_F1,ST04_A1,ST04_B1"
  29 
  30 zonecreate "VPLEX_E1_B1_F3_ST01","VPLEX_E1_B1_F3,ST01_A1,ST01_B1"
  31 zonecreate "VPLEX_E1_B1_F3_ST02","VPLEX_E1_B1_F3,ST02_A1,ST02_B1"
  32 zonecreate "VPLEX_E1_B1_F3_ST03","VPLEX_E1_B1_F3,ST03_A1,ST03_B1"
  33 zonecreate "VPLEX_E1_B1_F3_ST04","VPLEX_E1_B1_F3,ST04_A1,ST04_B1"

VPLEX_E2_A1 with VNX5200 (预留)

zonecreate "VPLEX_E2_A1_F1_ST01","VPLEX_E2_A1_F1,ST01_A1,ST01_B1"
zonecreate "VPLEX_E2_A1_F1_ST02","VPLEX_E2_A1_F1,ST02_A1,ST02_B1"
zonecreate "VPLEX_E2_A1_F1_ST03","VPLEX_E2_A1_F1,ST03_A1,ST03_B1"
zonecreate "VPLEX_E2_A1_F1_ST03","VPLEX_E2_A1_F1,ST03_A1,ST03_B1"

zonecreate "VPLEX_E2_A1_F3_ST01","VPLEX_E2_A1_F3,ST01_A1,ST01_B1"
zonecreate "VPLEX_E2_A1_F3_ST02","VPLEX_E2_A1_F3,ST02_A1,ST02_B1"
zonecreate "VPLEX_E2_A1_F3_ST03","VPLEX_E2_A1_F3,ST03_A1,ST03_B1"
zonecreate "VPLEX_E2_A1_F3_ST04","VPLEX_E2_A1_F3,ST04_A1,ST04_B1"

zonecreate "VPLEX_E2_B1_F1_ST01","VPLEX_E2_B1_F1,ST01_A1,ST01_B1"
zonecreate "VPLEX_E2_B1_F1_ST02","VPLEX_E2_B1_F1,ST02_A1,ST02_B1"
zonecreate "VPLEX_E2_B1_F1_ST03","VPLEX_E2_B1_F1,ST03_A1,ST03_B1"
zonecreate "VPLEX_E2_B1_F1_ST04","VPLEX_E2_B1_F1,ST04_A1,ST04_B1"

zonecreate "VPLEX_E2_B1_F3_ST01","VPLEX_E2_B1_F3,ST01_A1,ST01_B1"
zonecreate "VPLEX_E2_B1_F3_ST02","VPLEX_E2_B1_F3,ST02_A1,ST02_B1"
zonecreate "VPLEX_E2_B1_F3_ST03","VPLEX_E2_B1_F3,ST03_A1,ST03_B1"
zonecreate "VPLEX_E2_B1_F3_ST04","VPLEX_E2_B1_F3,ST04_A1,ST04_B1"

创建并激活CONFIG

# sed -n '/VPLEX_E1/p' < /tmp/tmphSAXQp.moin | grep zonecre | awk -F"\"" '{ print $2 }' | sort | fmt | tr '\n' ' ' | tr ' ' ','

cfgcreate "ZHCIQ_SAN_B","VPLEX_DB_RAC01,VPLEX_DB_RAC02,VPLEX_E1_A1_F1_ST01,VPLEX_E1_A1_F1_ST02,VPLEX_E1_A1_F1_ST03,VPLEX_E1_A1_F1_ST04,VPLEX_E1_A1_F3_ST01,VPLEX_E1_A1_F3_ST02,VPLEX_E1_A1_F3_ST03,VPLEX_E1_A1_F3_ST04,VPLEX_E1_B1_F1_ST01,VPLEX_E1_B1_F1_ST02,VPLEX_E1_B1_F1_ST03,VPLEX_E1_B1_F1_ST04,VPLEX_E1_B1_F3_ST01,VPLEX_E1_B1_F3_ST02,VPLEX_E1_B1_F3_ST03,VPLEX_E1_B1_F3_ST04,VPLEX_ESXi01,VPLEX_ESXi02,VPLEX_ESXi03,VPLEX_ESXi04,VPLEX_TA_SW01,VPLEX_TA_SW02"
cfgenable ZHCIQ_SAN_B
cfgsave

Host HBAs

10:00:00:90:fa:b4:2f:16 ESXi_01_F1  
10:00:00:90:fa:b4:2f:17 ESXi_01_F2 
10:00:00:90:fa:b4:2e:be ESXi_02_F1 
10:00:00:90:fa:b4:2e:bf ESXi_02_F2 
21:00:00:24:ff:13:83:d5 ESXi_05_F1 
21:00:00:24:ff:13:82:ef ESXi_05_F2 
21:00:00:24:ff:14:23:72 ESXi_06_F1 
21:00:00:24:ff:14:22:ad ESXi_06_F2 
10:00:00:90:fa:c0:c1:e4 ESXi_07_F1 
10:00:00:90:fa:c0:c2:00 ESXi_07_F2 
10:00:00:90:fa:c0:c2:15 ESXi_08_F1 
10:00:00:90:fa:c0:c2:ca ESXi_08_F2 

# 一共6台ESXi

10:00:00:90:fa:c0:c1:e9 DB_RAC02_F2 
10:00:00:90:fa:c0:c1:60 DB_RAC02_F1 
10:00:00:90:fa:c0:c2:ce DB_RAC01_F1 
10:00:00:90:fa:c0:c2:d7 DB_RAC01_F2