# compute node config /etc/nova/nova.conf
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 |
network_api_class = nova.network.api.API security_group_api = nova firewall_driver = nova.virt.libvirt.firewall.IptablesFirewallDriver network_manager = nova.network.manager.FlatDHCPManager network_size = 254 allow_same_net_traffic = False multi_host = True send_arp_for_ha = True share_dhcp_address = True force_dhcp_release = True flat_network_bridge = br100 flat_interface = eth0 public_interface = eth0 |
* flat_network_bridge = br100 สร้าง bridge interface
* flat_interface = eth0 นำ eth1 add interface เข้า bridge
* public_interface = eth0 เป็น interface ต่อ external nework
% brctl show
|
1 2 3 4 5 |
bridge name bridge id STP enabled interfaces br100 8000.525400a9206e no eth0 vnet0 |
% ip addr show dev br100
|
1 2 3 4 5 6 7 8 |
4: br100: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default link/ether 52:54:00:a9:20:6e brd ff:ff:ff:ff:ff:ff inet 192.168.213.105/25 brd 192.168.213.127 scope global br100 valid_lft forever preferred_lft forever inet6 fe80::f84f:c7ff:fe98:6a60/64 scope link valid_lft forever preferred_lft forever |
* เมื่อทำการสร้าง network และ instance ip gateway ของ nework จะถูกนำมา add บน br100
* add default route บน compute node เพือให้ vm ออก extenal network ในกรณีไม่ต้องการ nat ( floating ip )
|
1 2 3 4 |
ip route add default via 192.168.213.254 dev br100 default via 192.168.213.126 dev br100 |
* ป้องกัน ip gateway ที่ add บน bridge interface ส่ง arp reply ซึ่งทำให้ external nework ไม่สามารถ connect มาที่ ip gateway ได้
แต่ vm สามารถ connect gateway บน bridge ได้ eth0 เป็น flat_nework
* ip ที่ add on br100 (จากตัวอย่าง 192.168.213.105) ip นี้จะมีอยู่ทุกๆ compute node ซึ่ง External ไม่สามารถ connect เข้ามาได้
และ DNSMASQ จะมี process ที่แยกกันอยู่แต่ละ compute node , และ แต่ละ intance จะรับ DHCP จาก compute node ของตัวเอง
เท่านั้นจะไม่รับข้าม compute node.
|
1 2 3 4 5 6 7 8 9 10 11 12 13 |
%ebtables -L Bridge chain: INPUT, entries: 1, policy: ACCEPT -p ARP -i eth0 --arp-ip-dst 192.168.213.105 -j DROP Bridge chain: FORWARD, entries: 2, policy: ACCEPT -p IPv4 -o eth0 --ip-proto udp --ip-dport 67:68 -j DROP -p IPv4 -i eth0 --ip-proto udp --ip-dport 67:68 -j DROP Bridge chain: OUTPUT, entries: 1, policy: ACCEPT -p ARP -o eth0 --arp-ip-src 192.168.213.105 -j DROP |
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 |
%ebtables -t nat -L Bridge chain: PREROUTING, entries: 1, policy: ACCEPT -i vnet0 -j libvirt-I-vnet0 Bridge chain: OUTPUT, entries: 0, policy: ACCEPT Bridge chain: POSTROUTING, entries: 1, policy: ACCEPT -o vnet0 -j libvirt-O-vnet0 Bridge chain: libvirt-I-vnet0, entries: 5, policy: ACCEPT -j I-vnet0-mac -p IPv4 -j I-vnet0-ipv4-ip -p IPv4 -j I-vnet0-ipv4 -p ARP -j I-vnet0-arp-mac -p ARP -j I-vnet0-arp-ip Bridge chain: libvirt-O-vnet0, entries: 2, policy: ACCEPT -p IPv4 -j O-vnet0-ipv4 -p IPv6 -j O-vnet0-ipv6 Bridge chain: I-vnet0-mac, entries: 2, policy: ACCEPT -s fa:16:3e:7a:98:9d -j RETURN -j DROP Bridge chain: I-vnet0-ipv4-ip, entries: 3, policy: ACCEPT -p IPv4 --ip-src 0.0.0.0 --ip-proto udp -j RETURN -p IPv4 --ip-src 192.168.213.106 -j RETURN -j DROP Bridge chain: I-vnet0-ipv4, entries: 1, policy: ACCEPT -p IPv4 --ip-src 0.0.0.0 --ip-dst 255.255.255.255 --ip-proto udp --ip-sport 68 --ip-dport 67 -j ACCEPT Bridge chain: O-vnet0-ipv4, entries: 1, policy: ACCEPT -p IPv4 --ip-src 192.168.213.105 --ip-proto udp --ip-sport 67 --ip-dport 68 -j ACCEPT Bridge chain: O-vnet0-ipv6, entries: 1, policy: ACCEPT -s fa:16:3e:7a:98:9d -d 33:33:0:0:0:0/ff:ff:0:0:0:0 -j DROP Bridge chain: I-vnet0-arp-mac, entries: 2, policy: ACCEPT -p ARP --arp-mac-src fa:16:3e:7a:98:9d -j RETURN -j DROP Bridge chain: I-vnet0-arp-ip, entries: 2, policy: ACCEPT -p ARP --arp-ip-src 192.168.213.106 -j RETURN -j DROP |
* 192.168.213.105 ip gateway on brige interface
* 192.168.213.106 ip vm
* fa:16:3e:7a:98:9d mac adress vm
ขั้นตอนการสร้าง VM ผ่าน NOVA
1 . create network
|
1 2 3 4 |
nova network-create demo-net --bridge br100 --multi-host T --fixed-range-v4 192.168.213.0/25 \ --allowed-start 192.168.213.105 --allowed-end 192.168.213.110 |
|
1 2 3 4 5 6 7 8 |
nova net-list +--------------------------------------+----------+------------------+ | ID | Label | CIDR | +--------------------------------------+----------+------------------+ | 2c732c09-0d7a-4154-8cdd-05cf454bf63a | demo-net | 192.168.213.0/25 | +--------------------------------------+----------+------------------+ |
2. List image
|
1 2 3 4 5 6 7 8 9 |
% nova image-list +--------------------------------------+---------------------+--------+--------+ | ID | Name | Status | Server | +--------------------------------------+---------------------+--------+--------+ | e5272b3c-d607-4d47-adf6-aac17616ef13 | cirros-0.3.3-x86_64 | ACTIVE | | +--------------------------------------+---------------------+--------+--------+ |
3. List flavor-list
|
1 2 3 4 5 6 7 8 9 10 11 12 13 |
nova flavor-list +----+-----------+-----------+------+-----------+------+-------+-------------+-----------+ | ID | Name | Memory_MB | Disk | Ephemeral | Swap | VCPUs | RXTX_Factor | Is_Public | +----+-----------+-----------+------+-----------+------+-------+-------------+-----------+ | 1 | m1.tiny | 512 | 1 | 0 | | 1 | 1.0 | True | | 2 | m1.small | 2048 | 20 | 0 | | 1 | 1.0 | True | | 3 | m1.medium | 4096 | 40 | 0 | | 2 | 1.0 | True | | 4 | m1.large | 8192 | 80 | 0 | | 4 | 1.0 | True | | 5 | m1.xlarge | 16384 | 160 | 0 | | 8 | 1.0 | True | | 6 | s1.tiny | 256 | 1 | 0 | | 1 | 1.0 | True | +----+-----------+-----------+------+-----------+------+-------+-------------+-----------+ |
4. Allow ping and ssh
|
1 2 3 4 |
nova secgroup-add-rule default icmp -1 -1 0.0.0.0/0 nova secgroup-add-rule default tcp 22 22 0.0.0.0/0 |
5. Create instance
|
1 2 3 4 5 6 7 8 9 10 11 12 |
% nova boot --flavor s1.tiny --image cirros-0.3.3-x86_64 \ --nic net-id=2c732c09-0d7a-4154-8cdd-05cf454bf63a --security-group default app00 % nova list +--------------------------------------+-------+--------+------------+-------------+--------------------------+ | ID | Name | Status | Task State | Power State | Networks | +--------------------------------------+-------+--------+------------+-------------+--------------------------+ | 9edbca0d-25b3-4b64-af4a-9a11aff2d862 | app00 | ACTIVE | - | Running | demo-net=192.168.213.106 | +--------------------------------------+-------+--------+------------+-------------+--------------------------+ |
ทดสอบ การสร้าง Multi Subnet
สร้าง subnet
|
1 2 3 4 |
nova network-create mypri00 --bridge br100 --multi-host T \ --fixed-range-v4 172.16.0.0/24 --allowed-start 172.16.0.100 --allowed-end 172.16.0.200 |
create instance
|
1 2 3 4 |
nova boot --flavor s1.tiny --image cirros-0.3.3-x86_64 \ --nic net-id=f3939dc6-8495-483f-b10e-03ccbf243e3c --security-group default app01 |
|
1 2 3 4 5 6 7 8 9 10 |
% nova list +--------------------------------------+-------+--------+------------+-------------+--------------------------+ | ID | Name | Status | Task State | Power State | Networks | +--------------------------------------+-------+--------+------------+-------------+--------------------------+ | 9edbca0d-25b3-4b64-af4a-9a11aff2d862 | app00 | ACTIVE | - | Running | demo-net=192.168.213.106 | | 8f12a240-9547-424c-b6aa-400a1c82319f | app01 | ACTIVE | - | Running | mypri00=172.16.0.101 | +--------------------------------------+-------+--------+------------+-------------+--------------------------+ |
|
1 2 3 4 5 6 7 8 9 10 11 12 |
% ip addr show dev br100 4: br100: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default link/ether 52:54:00:a9:20:6e brd ff:ff:ff:ff:ff:ff inet 172.16.0.100/24 brd 172.16.0.255 scope global br100 valid_lft forever preferred_lft forever inet 192.168.213.105/25 brd 192.168.213.127 scope global br100 valid_lft forever preferred_lft forever inet6 fe80::f84f:c7ff:fe98:6a60/64 scope link valid_lft forever preferred_lft forever |
ปัญหาที่พบ DNSMASQ ไม่สามารถแจก IP อีก range ได้ แต่ ip address add br100 ปกติ
|
1 2 3 4 5 6 7 |
/usr/sbin/dnsmasq --strict-order --bind-interfaces --conf-file= --pid-file=/var/lib/nova/networks/nova-br100.pid --dhcp-optsfile=/var/lib/nova/networks/nova-br100.opts --listen-address=192.168.213.105 --except-interface=lo --dhcp-range=set:demo-net,192.168.213.106,static,255.255.255.128,86400s --dhcp-lease-max=128 --dhcp-hostsfile=/var/lib/nova/networks/nova-br100.conf --dhcp-script=/usr/bin/nova-dhcpbridge --no-hosts --leasefile-ro --domain=novalocal --addn-hosts=/var/lib/nova/networks/nova-br100.hosts |
|
1 2 3 4 5 |
% cat /var/lib/nova/networks/nova-br100.conf fa:16:3e:ea:21:82,app01.novalocal,172.16.0.101root@compute01:/var/log/nova |
ทดสอบ การสร้าง Multi Subnet โดยสร้าง bridge interface
สร้าง network
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 |
% nova network-create mypri00 --bridge br100 --multi-host T \ --fixed-range-v4 192.168.213.0/25 --allowed-start 192.168.213.105 \ --allowed-end 192.168.213.110 % nova network-create mypri01 --bridge br200 --multi-host T \ --fixed-range-v4 172.16.0.0/24 --allowed-start 172.16.0.100 \ --allowed-end 172.16.0.200 +--------------------------------------+---------+------------------+ | ID | Label | CIDR | +--------------------------------------+---------+------------------+ | 7197f489-8cc4-4834-85af-758d1a77890f | mypri00 | 192.168.213.0/25 | | 703aa9c0-a5c1-46ec-b693-3c5e682ebd51 | mypri01 | 172.16.0.0/24 | +--------------------------------------+---------+------------------+ |
สร้าง instance
|
1 2 3 4 5 6 7 |
nova boot --flavor m1.tiny --image cirros-0.3.3-x86_64 --nic net-id=7197f489-8cc4-4834-85af-758d1a77890f \ --security-group default app00 nova boot --flavor m1.tiny --image cirros-0.3.3-x86_64 --nic net-id=703aa9c0-a5c1-46ec-b693-3c5e682ebd51 \ --security-group default app01 |
ผลลัพธ์
* มีการสร้าง bridge interface br200 เพิ่มขึ้นมา
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 |
4: br100: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default link/ether 52:54:00:a9:20:6e brd ff:ff:ff:ff:ff:ff inet 192.168.213.105/25 brd 192.168.213.127 scope global br100 valid_lft forever preferred_lft forever inet6 fe80::40d3:73ff:fe0f:b938/64 scope link valid_lft forever preferred_lft forever 6: br200: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default link/ether fe:16:3e:60:66:ae brd ff:ff:ff:ff:ff:ff inet 172.16.0.100/24 brd 172.16.0.255 scope global br200 valid_lft forever preferred_lft forever inet6 fe80::f42b:e8ff:fe30:e556/64 scope link valid_lft forever preferred_lft forever |
แต่ eth0 bridge กับ br100 ตาม config ด้านบน
|
1 2 3 4 5 6 7 8 |
% brctl show bridge name bridge id STP enabled interfaces br100 8000.525400a9206e no eth0 vnet0 br200 8000.fe163e6066ae no vnet1 |
dnsmasq สามารถทำงานแจก ip ได้ทั้ง สอง subnet
|
1 2 3 4 5 |
--listen-address=172.16.0.100 --except-interface=lo --dhcp-range=set:mypri01,172.16.0.101,static,255.255.255.0,86400s --listen-address=192.168.213.105 --except-interface=lo --dhcp-range=set:mypri00,192.168.213.106,static,255.255.255.128,86400s |
ด้วยวิธีการนี้ ปัญหา คือ vm ที่ bridge กับ br200 ไม่สามารถออก external network ได้ รวมทั้ง ไม่สามารถ connect vm ข้าม compute node ได้