mirror of https://github.com/EdgeVPNio/cni.git
tested build
saumitraaditya
parent
c5d290ee4d
commit
9bd0388b7b
45
README.md
45
README.md
|
|
@ -1,2 +1,45 @@
|
|||
# evioPlugin
|
||||
Kubernetes CNI pluigin to integrate EdgeVPN overlay with K8s
|
||||
CNI plugin for EdgeVPN - Enhanced version of bridge plugin to work with OVS and utility to allocate IP address range
|
||||
to nodes and generate config file for the plugin.
|
||||
Dependency on github.com/containernetworking tag v0.7.5
|
||||
```
|
||||
mkdir -p $GOPATH/src/github.com/containernetworking
|
||||
cd $GOPATH/src/github.com/containernetworking
|
||||
git clone https://github.com/containernetworking/plugins.git
|
||||
cd plugins
|
||||
git checkout tags/v0.7.5
|
||||
cd plugins/main/
|
||||
git clone https://github.com/EdgeVPNio/evioPlugin.git
|
||||
```
|
||||
|
||||
Now need to download a few dependencies.
|
||||
```
|
||||
go get github.com/j-keck/arping
|
||||
go get github.com/vishvananda/netlink
|
||||
go get github.com/digitalocean/go-openvswitch/ovs
|
||||
go get go.etcd.io/etcd/clientv3
|
||||
```
|
||||
|
||||
We will need three executables, for the plugin, config-gen evioUtililty and for
|
||||
host-local IPAM plugin.
|
||||
```
|
||||
# assuming you are in $GOPATH/src/github.com/containernetworking/plugins/plugins/main/evioPlugin/src
|
||||
go build evioPlugin.go
|
||||
go build evioUtilities.go
|
||||
cd $GOPATH/src/github.com/containernetworking/plugins/plugins/ipam/host-local
|
||||
go build
|
||||
```
|
||||
|
||||
Now we need to copy all three executables to the deployment folder.
|
||||
```
|
||||
cd $GOPATH/src/github.com/containernetworking/plugins/plugins/main/evioPlugin/deployment
|
||||
cp $GOPATH/src/github.com/containernetworking/plugins/plugins/ipam/host-local/host-local .
|
||||
cp $GOPATH/src/github.com/containernetworking/plugins/plugins/main/evioPlugin/src/evioPlugin .
|
||||
cp $GOPATH/src/github.com/containernetworking/plugins/plugins/main/evioPlugin/src/evioUtilities .
|
||||
```
|
||||
|
||||
Build the docker image.
|
||||
|
||||
```
|
||||
docker build -t evio_plugin:0.0 -f evioCNI.dockerfile .
|
||||
```
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
#!/bin/sh
|
||||
cp evioPlugin /opt/cni/bin
|
||||
mv /opt/cni/bin/host-local /opt/cni/bin/bkp-host-local
|
||||
cp host-local /opt/cni/bin
|
||||
./evioUtilities
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
FROM partlab/ubuntu-golang
|
||||
|
||||
WORKDIR /root/
|
||||
COPY ./evioUtilities .
|
||||
COPY ./evioPlugin .
|
||||
COPY ./host-local .
|
||||
COPY ./cmd.sh .
|
||||
RUN chmod +x cmd.sh
|
||||
RUN chmod +x evioUtilities
|
||||
RUN chmod +x evioPlugin
|
||||
RUN chmod +x host-local
|
||||
|
||||
CMD ["./cmd.sh"]
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
---
|
||||
apiVersion: policy/v1beta1
|
||||
kind: PodSecurityPolicy
|
||||
metadata:
|
||||
name: psp.evio.unprivileged
|
||||
annotations:
|
||||
seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
|
||||
seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
|
||||
apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
|
||||
apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
|
||||
spec:
|
||||
privileged: false
|
||||
volumes:
|
||||
- configMap
|
||||
- secret
|
||||
- emptyDir
|
||||
- hostPath
|
||||
allowedHostPaths:
|
||||
- pathPrefix: "/etc/cni/net.d"
|
||||
- pathPrefix: "/etc/kube-evio"
|
||||
- pathPrefix: "/run/evio"
|
||||
readOnlyRootFilesystem: false
|
||||
# Users and groups
|
||||
runAsUser:
|
||||
rule: RunAsAny
|
||||
supplementalGroups:
|
||||
rule: RunAsAny
|
||||
fsGroup:
|
||||
rule: RunAsAny
|
||||
# Privilege Escalation
|
||||
allowPrivilegeEscalation: false
|
||||
defaultAllowPrivilegeEscalation: false
|
||||
# Capabilities
|
||||
allowedCapabilities: ['NET_ADMIN']
|
||||
defaultAddCapabilities: []
|
||||
requiredDropCapabilities: []
|
||||
# Host namespaces
|
||||
hostPID: false
|
||||
hostIPC: false
|
||||
hostNetwork: true
|
||||
hostPorts:
|
||||
- min: 0
|
||||
max: 65535
|
||||
# SELinux
|
||||
seLinux:
|
||||
# SELinux is unused in CaaSP
|
||||
rule: 'RunAsAny'
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: evio
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: store-credentials
|
||||
namespace: kube-system
|
||||
data: # base64 encoded
|
||||
username: ZXZwblVzZXI=
|
||||
password: cGFzcw==
|
||||
---
|
||||
kind: ConfigMap
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: kube-evio-cfg
|
||||
namespace: kube-system
|
||||
labels:
|
||||
tier: node
|
||||
app: evio
|
||||
data:
|
||||
# number of bits for host = nodePrefix - subnet
|
||||
# in sample config below 26-16 = 8, can have 2^8 hosts
|
||||
# each with 2^8 pods.
|
||||
net-conf.json: |
|
||||
{
|
||||
"cniVersion": "0.3.1",
|
||||
"name": "k8s-pod-network",
|
||||
"type": "evioPlugin",
|
||||
"bridge": "evioB1B111B",
|
||||
"bridgeType": "OVS",
|
||||
"isGateway": true,
|
||||
"isDefaultGateway": true,
|
||||
"isIPMasq": true,
|
||||
"nodeBits": "8",
|
||||
"podCIDR": "10.244.0.16/16",
|
||||
"dataStore": "0.0.0.0:42000",
|
||||
"auth": false
|
||||
}
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: kube-evio-ds-amd64
|
||||
namespace: kube-system
|
||||
labels:
|
||||
tier: node
|
||||
app: evio
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: evio
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: node
|
||||
app: evio
|
||||
spec:
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/os
|
||||
operator: In
|
||||
values:
|
||||
- linux
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values:
|
||||
- amd64
|
||||
hostNetwork: true
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
effect: NoSchedule
|
||||
serviceAccountName: evio
|
||||
initContainers:
|
||||
- name: install-cni
|
||||
image: evio_plugin:0.0
|
||||
volumeMounts:
|
||||
- name: cni
|
||||
mountPath: /etc/cni/net.d
|
||||
- name: evio-cfg
|
||||
mountPath: /etc/evioCNI/
|
||||
- name: plugin
|
||||
mountPath: /opt/cni/bin/
|
||||
- name: store-credentials
|
||||
mountPath: /etc/credentials/
|
||||
containers:
|
||||
- name: dumdum
|
||||
image: gcr.io/kubernetes-e2e-test-images/dnsutils:1.3
|
||||
command:
|
||||
- sleep
|
||||
args:
|
||||
- "3600"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "100m"
|
||||
memory: "50Mi"
|
||||
limits:
|
||||
cpu: "100m"
|
||||
memory: "50Mi"
|
||||
securityContext:
|
||||
privileged: false
|
||||
capabilities:
|
||||
add: ["NET_ADMIN"]
|
||||
volumes:
|
||||
- name: cni
|
||||
hostPath:
|
||||
path: /etc/cni/net.d
|
||||
- name: plugin
|
||||
hostPath:
|
||||
path: /opt/cni/bin/
|
||||
- name: evio-cfg
|
||||
configMap:
|
||||
name: kube-evio-cfg
|
||||
- name: store-credentials
|
||||
secret:
|
||||
secretName: store-credentials
|
||||
|
|
@ -0,0 +1,497 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/containernetworking/cni/pkg/skel"
|
||||
"github.com/containernetworking/cni/pkg/types"
|
||||
"github.com/containernetworking/cni/pkg/types/current"
|
||||
"github.com/containernetworking/cni/pkg/version"
|
||||
"github.com/containernetworking/plugins/pkg/ip"
|
||||
"github.com/containernetworking/plugins/pkg/ipam"
|
||||
"github.com/containernetworking/plugins/pkg/ns"
|
||||
"github.com/containernetworking/plugins/pkg/utils"
|
||||
"github.com/digitalocean/go-openvswitch/ovs"
|
||||
"github.com/j-keck/arping"
|
||||
"github.com/vishvananda/netlink"
|
||||
)
|
||||
|
||||
type NetConf struct {
|
||||
types.NetConf
|
||||
BrName string `json:"bridge"`
|
||||
BrType string `json:"bridgeType"`
|
||||
BrIP string `json:"bridgeIPv4"`
|
||||
IsGW bool `json:"isGateway"`
|
||||
IsDefaultGW bool `json:"isDefaultGateway"`
|
||||
IPMasq bool `json:"isIPMasq"`
|
||||
subnetPath string `json:"subnetPath"`
|
||||
}
|
||||
|
||||
type gwInfo struct {
|
||||
gws []net.IPNet
|
||||
family int
|
||||
defaultRouteFound bool
|
||||
}
|
||||
|
||||
func loadNetConf(bytes []byte) (*NetConf, string, error) {
|
||||
n := &NetConf{}
|
||||
if err := json.Unmarshal(bytes, n); err != nil {
|
||||
return nil, "", fmt.Errorf("failed to load network configuration: %v", err)
|
||||
}
|
||||
log.Printf("%+v", *n)
|
||||
return n, n.CNIVersion, nil
|
||||
}
|
||||
|
||||
func configureBridgeInterface(bridgeName string, gatewayAddress string) error {
|
||||
|
||||
i, err := net.InterfaceByName(bridgeName)
|
||||
if err != nil {
|
||||
log.Println("Bridge interface not found.")
|
||||
return err
|
||||
}
|
||||
ipv4set := false
|
||||
var ip net.IP
|
||||
ipv4 := ""
|
||||
|
||||
addrs, err := i.Addrs()
|
||||
if err != nil {
|
||||
log.Println("Could not get addresses associated with the bridge.")
|
||||
return err
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
switch v := addr.(type) {
|
||||
case *net.IPAddr:
|
||||
ip = v.IP
|
||||
case *net.IPNet:
|
||||
ip = v.IP
|
||||
}
|
||||
log.Printf("Bridge address %s\n", addr.String())
|
||||
if ip.To4() != nil {
|
||||
if strings.Compare(gatewayAddress, addr.String()) == 0 {
|
||||
log.Printf("Interface %s already configured with address %s\n", bridgeName, addr.String())
|
||||
ipv4set = true
|
||||
} else {
|
||||
ipv4 = addr.String()
|
||||
}
|
||||
}
|
||||
}
|
||||
if ipv4set == true {
|
||||
return nil
|
||||
}
|
||||
// need to assign gateway address to switch and bring it up.
|
||||
ipExecutable, err := exec.LookPath("ip")
|
||||
if err != nil {
|
||||
log.Println("failed to locate executable ip")
|
||||
return err
|
||||
}
|
||||
if ipv4 != "" {
|
||||
log.Println(" some other ipv4 address %s is assigned in place of %s ", ipv4, gatewayAddress)
|
||||
}
|
||||
cmdDelIP := &exec.Cmd{
|
||||
Path: ipExecutable,
|
||||
Args: []string{ipExecutable, "addr", "del", ipv4, "dev", bridgeName},
|
||||
}
|
||||
if ipv4 != "" {
|
||||
if output, err := cmdDelIP.Output(); err != nil {
|
||||
log.Println(err.Error())
|
||||
return err
|
||||
} else {
|
||||
log.Println(output)
|
||||
}
|
||||
}
|
||||
cmdSetIP := &exec.Cmd{
|
||||
Path: ipExecutable,
|
||||
Args: []string{ipExecutable, "addr", "add", gatewayAddress, "dev", bridgeName},
|
||||
}
|
||||
if output, err := cmdSetIP.Output(); err != nil {
|
||||
log.Println(err.Error())
|
||||
return err
|
||||
} else {
|
||||
log.Println(output)
|
||||
}
|
||||
|
||||
cmdBringUP := &exec.Cmd{
|
||||
Path: ipExecutable,
|
||||
Args: []string{ipExecutable, "link", "set", bridgeName, "up"},
|
||||
}
|
||||
if output, err := cmdBringUP.Output(); err != nil {
|
||||
log.Println(err.Error())
|
||||
return err
|
||||
} else {
|
||||
log.Println(output)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func setupBridge(netconf *NetConf) (*netlink.Bridge, *current.Interface, error) {
|
||||
br := &netlink.Bridge{
|
||||
LinkAttrs: netlink.LinkAttrs{
|
||||
Name: netconf.BrName,
|
||||
MTU: 1500,
|
||||
TxQLen: -1,
|
||||
},
|
||||
}
|
||||
// this section adds and sets up a linux bridge
|
||||
if netconf.BrType == "LXBR" {
|
||||
err := netlink.LinkAdd(br)
|
||||
if err != nil && err != syscall.EEXIST {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if err := netlink.LinkSetUp(br); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
} else if netconf.BrType == "OVS" {
|
||||
// setting up an ovs bridge
|
||||
ovsClient := ovs.New(ovs.Sudo())
|
||||
if err := ovsClient.VSwitch.AddBridge(br.Attrs().Name); err != nil {
|
||||
log.Fatalf("failed to add OVS bridge: %v", err)
|
||||
}
|
||||
}
|
||||
return br, ¤t.Interface{
|
||||
Name: br.Attrs().Name,
|
||||
Mac: br.Attrs().HardwareAddr.String(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func setupVeth(br *netlink.Bridge, netconf *NetConf, args *skel.CmdArgs) (*current.Interface, *current.Interface, error) {
|
||||
netns, err := ns.GetNS(args.Netns)
|
||||
if err != nil {
|
||||
log.Fatalf("Could not find network namespace %v", err)
|
||||
return nil, nil, fmt.Errorf("Could not find network namespace %v", err)
|
||||
}
|
||||
contIface := ¤t.Interface{}
|
||||
hostIface := ¤t.Interface{}
|
||||
var handler = func(hostNS ns.NetNS) error {
|
||||
hostVeth, containerVeth, err := ip.SetupVeth(args.IfName, 1500, hostNS)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
contIface.Name = containerVeth.Name
|
||||
contIface.Mac = containerVeth.HardwareAddr.String()
|
||||
contIface.Sandbox = netns.Path()
|
||||
hostIface.Name = hostVeth.Name
|
||||
log.Printf("Created pod veth interface: %v\n", containerVeth.Name)
|
||||
return nil
|
||||
}
|
||||
if err := netns.Do(handler); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
hostVeth, err := netlink.LinkByName(hostIface.Name)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
log.Printf("Located host veth interface: %v\n", hostVeth.Attrs().Name)
|
||||
hostIface.Mac = hostVeth.Attrs().HardwareAddr.String()
|
||||
if netconf.BrType == "LXBR" {
|
||||
// below attaches veth interface to the bridge.
|
||||
if err := netlink.LinkSetMaster(hostVeth, br); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
} else if netconf.BrType == "OVS" {
|
||||
// below attaches veth interface to ovs bridge
|
||||
ovsClient := ovs.New(ovs.Sudo())
|
||||
if err := ovsClient.VSwitch.AddPort(br.Attrs().Name, hostIface.Name); err != nil {
|
||||
log.Printf("failed to add %v to bridge %v error %v", hostIface.Name, br.Attrs().Name, err)
|
||||
}
|
||||
log.Printf("Attached %v to %v\n", hostVeth.Attrs().Name, br.Name)
|
||||
}
|
||||
return hostIface, contIface, nil
|
||||
|
||||
}
|
||||
|
||||
func cmdAdd(args *skel.CmdArgs) error {
|
||||
n, cniVersion, err := loadNetConf(args.StdinData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("%v\t %v\t %v", args.ContainerID, args.IfName, args.Netns)
|
||||
log.Println("parsed configuration successfully !")
|
||||
br, brInterface, err := setupBridge(n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("set up bridge %v successfully !\n", br.Name)
|
||||
// set up IP address on the bridge and bring it up.
|
||||
err = configureBridgeInterface(n.BrName, n.BrIP)
|
||||
if err != nil {
|
||||
log.Println("Failure in configuring bridge.")
|
||||
return err
|
||||
}
|
||||
hostInterface, containerInterface, err := setupVeth(br, n, args)
|
||||
if err != nil {
|
||||
log.Println("Failure in setting up Veth interfaces.")
|
||||
return err
|
||||
}
|
||||
log.Println("set up veth interfaces successfully !")
|
||||
|
||||
result := ¤t.Result{CNIVersion: cniVersion, Interfaces: []*current.Interface{brInterface, hostInterface, containerInterface}}
|
||||
r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var success bool = false
|
||||
// release IP in case of failure
|
||||
defer func() {
|
||||
if !success {
|
||||
os.Setenv("CNI_COMMAND", "DEL")
|
||||
ipam.ExecDel(n.IPAM.Type, args.StdinData)
|
||||
os.Setenv("CNI_COMMAND", "ADD")
|
||||
}
|
||||
}()
|
||||
|
||||
// Convert whatever the IPAM result was into the current Result type
|
||||
ipamResult, err := current.NewResultFromResult(r)
|
||||
if err != nil {
|
||||
log.Printf("could not convert IPAM result %+v \n", ipamResult)
|
||||
return err
|
||||
}
|
||||
log.Printf("result from IPAM : %+v\n", ipamResult)
|
||||
result.IPs = ipamResult.IPs
|
||||
result.Routes = ipamResult.Routes
|
||||
|
||||
if len(result.IPs) == 0 {
|
||||
log.Printf("IPAM plugin provided no IP config\n")
|
||||
return errors.New("IPAM plugin returned missing IP config")
|
||||
}
|
||||
netns, err := ns.GetNS(args.Netns)
|
||||
if err != nil {
|
||||
log.Printf("could not find namespace %v", args.Netns)
|
||||
return err
|
||||
}
|
||||
for _, ipc := range result.IPs {
|
||||
ipc.Interface = current.Int(2)
|
||||
}
|
||||
|
||||
gwsV4, gwsV6, err := calcGateways(result, n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Configure the container hardware address and IP address(es)
|
||||
if err := netns.Do(func(_ ns.NetNS) error {
|
||||
contVeth, err := net.InterfaceByName(args.IfName)
|
||||
if err != nil {
|
||||
log.Printf("could not find interface %v", contVeth.Name)
|
||||
return err
|
||||
}
|
||||
// Add the IP to the interface
|
||||
if err := ipam.ConfigureIface(args.IfName, result); err != nil {
|
||||
log.Printf("could not configure IP address on the interface %v", args.IfName)
|
||||
return err
|
||||
}
|
||||
|
||||
// Send a gratuitous arp
|
||||
for _, ipc := range result.IPs {
|
||||
if ipc.Version == "4" {
|
||||
_ = arping.GratuitousArpOverIface(ipc.Address.IP, *contVeth)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
log.Printf("something went wrong while trying to configure IP address on the interface !")
|
||||
return err
|
||||
}
|
||||
if n.IsGW {
|
||||
// Set the IP address(es) on the bridge and enable forwarding
|
||||
for _, gws := range []*gwInfo{gwsV4, gwsV6} {
|
||||
if gws.gws != nil {
|
||||
if err = enableIPForward(gws.family); err != nil {
|
||||
return fmt.Errorf("failed to enable forwarding: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if n.IPMasq {
|
||||
chain := utils.FormatChainName(n.Name, args.ContainerID)
|
||||
comment := utils.FormatComment(n.Name, args.ContainerID)
|
||||
for _, ipc := range result.IPs {
|
||||
if err = ip.SetupIPMasq(ip.Network(&ipc.Address), chain, comment); err != nil {
|
||||
log.Printf("something went wrong while to set up IPMasq %v!", err)
|
||||
return err
|
||||
}
|
||||
log.Printf("IPMasq set up successfully !!")
|
||||
}
|
||||
}
|
||||
if n.BrType == "LXBR" {
|
||||
// commented out because accesses variables set when setting linux bridges
|
||||
l, err := netlink.LinkByName(br.Name)
|
||||
if err != nil {
|
||||
log.Printf("could not lookup %q: %v", br.Name, err)
|
||||
}
|
||||
br, ok := l.(*netlink.Bridge)
|
||||
if !ok {
|
||||
log.Printf("%q already exists but is not a bridge", br.Name)
|
||||
}
|
||||
}
|
||||
log.Printf("result %v\n", result)
|
||||
return types.PrintResult(result, cniVersion)
|
||||
}
|
||||
|
||||
func clearDeadPortsOnBridge(brName string) {
|
||||
ovsClient := ovs.New(ovs.Sudo())
|
||||
listOfPorts, err := ovsClient.VSwitch.ListPorts(brName)
|
||||
// dead interfaces are cleared and interfaces deleted outside cni plugin, just check and remove from
|
||||
// the bridge, happens for previously deleted namespaces not the current one being deleted.
|
||||
if err != nil {
|
||||
log.Printf("Could not list ports on bridge %s, error is %v", brName, err.Error())
|
||||
} else {
|
||||
for _, portName := range listOfPorts {
|
||||
_, err := netlink.LinkByName(portName)
|
||||
if err != nil {
|
||||
if err = ovsClient.VSwitch.DeletePort(brName, portName); err != nil {
|
||||
log.Printf("failed to remove %v from bridge %v error %v", brName, portName, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
func cmdDel(args *skel.CmdArgs) error {
|
||||
n, _, err := loadNetConf(args.StdinData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("%v\t %v\t %v", args.ContainerID, args.IfName, args.Netns)
|
||||
log.Println("Delete: parsed configuration successfully !")
|
||||
if args.Netns == "" {
|
||||
return nil
|
||||
}
|
||||
var ipnets []*net.IPNet
|
||||
err = ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error {
|
||||
var err error
|
||||
ipnets, err = ip.DelLinkByNameAddr(args.IfName)
|
||||
if err != nil && err == ip.ErrLinkNotFound {
|
||||
log.Printf("could not delete NS interface %v", err.Error())
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
})
|
||||
if n.IPMasq {
|
||||
chain := utils.FormatChainName(n.Name, args.ContainerID)
|
||||
comment := utils.FormatComment(n.Name, args.ContainerID)
|
||||
for _, ipn := range ipnets {
|
||||
if err := ip.TeardownIPMasq(ipn, chain, comment); err != nil {
|
||||
log.Printf("could not reset IPMasq %v", err.Error())
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if n.BrType == "OVS" {
|
||||
clearDeadPortsOnBridge(n.BrName)
|
||||
}
|
||||
if n.IPAM.Type != "" {
|
||||
if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil {
|
||||
log.Printf("could not clear out IPAM %v", err.Error())
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
func main() {
|
||||
skel.PluginMain(cmdAdd, cmdDel, version.All)
|
||||
}
|
||||
|
||||
func enableIPForward(family int) error {
|
||||
if family == netlink.FAMILY_V4 {
|
||||
return ip.EnableIP4Forward()
|
||||
}
|
||||
return ip.EnableIP6Forward()
|
||||
}
|
||||
|
||||
func calcGateways(result *current.Result, n *NetConf) (*gwInfo, *gwInfo, error) {
|
||||
|
||||
gwsV4 := &gwInfo{}
|
||||
gwsV6 := &gwInfo{}
|
||||
|
||||
for _, ipc := range result.IPs {
|
||||
|
||||
// Determine if this config is IPv4 or IPv6
|
||||
var gws *gwInfo
|
||||
defaultNet := &net.IPNet{}
|
||||
switch {
|
||||
case ipc.Address.IP.To4() != nil:
|
||||
gws = gwsV4
|
||||
gws.family = netlink.FAMILY_V4
|
||||
defaultNet.IP = net.IPv4zero
|
||||
case len(ipc.Address.IP) == net.IPv6len:
|
||||
gws = gwsV6
|
||||
gws.family = netlink.FAMILY_V6
|
||||
defaultNet.IP = net.IPv6zero
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("Unknown IP object: %v", ipc)
|
||||
}
|
||||
defaultNet.Mask = net.IPMask(defaultNet.IP)
|
||||
|
||||
// All IPs currently refer to the container interface
|
||||
ipc.Interface = current.Int(2)
|
||||
|
||||
if n.IsGW {
|
||||
ipc.Gateway = getGatewayIP(n)
|
||||
}
|
||||
|
||||
// Add a default route for this family using the current
|
||||
// gateway address if necessary.
|
||||
if n.IsDefaultGW && !gws.defaultRouteFound {
|
||||
for _, route := range result.Routes {
|
||||
if route.GW != nil && defaultNet.String() == route.Dst.String() {
|
||||
gws.defaultRouteFound = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !gws.defaultRouteFound {
|
||||
result.Routes = append(
|
||||
result.Routes,
|
||||
&types.Route{Dst: *defaultNet, GW: ipc.Gateway},
|
||||
)
|
||||
gws.defaultRouteFound = true
|
||||
}
|
||||
}
|
||||
|
||||
// Append this gateway address to the list of gateways
|
||||
if n.IsGW {
|
||||
gw := net.IPNet{
|
||||
IP: ipc.Gateway,
|
||||
Mask: ipc.Address.Mask,
|
||||
}
|
||||
gws.gws = append(gws.gws, gw)
|
||||
}
|
||||
}
|
||||
return gwsV4, gwsV6, nil
|
||||
}
|
||||
|
||||
func calcGatewayIP(ipn *net.IPNet) net.IP {
|
||||
nid := ipn.IP.Mask(ipn.Mask)
|
||||
return ip.NextIP(nid)
|
||||
}
|
||||
|
||||
func getGatewayIP(n *NetConf) net.IP {
|
||||
gwIP := net.ParseIP(strings.Split(n.BrIP, "/")[0])
|
||||
log.Printf("Gateway IPv4 address for bridge is %v !!", gwIP.String())
|
||||
return gwIP
|
||||
}
|
||||
|
||||
func init() {
|
||||
// this ensures that main runs only on main thread (thread group leader).
|
||||
// since namespace ops (unshare, setns) are done for a single thread, we
|
||||
// must ensure that the goroutine does not jump from OS thread to thread
|
||||
runtime.LockOSThread()
|
||||
file, err := os.OpenFile("/var/log/evioPlugin.log", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.SetOutput(file)
|
||||
}
|
||||
|
|
@ -0,0 +1,216 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
"go.etcd.io/etcd/clientv3/clientv3util"
|
||||
"go.etcd.io/etcd/clientv3/concurrency"
|
||||
)
|
||||
|
||||
var cli *clientv3.Client
|
||||
|
||||
type ipamInfo struct {
|
||||
IpamType string `json:"type"`
|
||||
Subnet string `json:"subnet"`
|
||||
RangeStart string `json:"rangeStart"`
|
||||
RangeEnd string `json:"rangeEnd"`
|
||||
}
|
||||
|
||||
type InitNetConf struct {
|
||||
CniVersion string `json:"cniVersion"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
BrName string `json:"bridge"`
|
||||
BrType string `json:"bridgeType"`
|
||||
BrIP string `json:"bridgeIPv4"`
|
||||
IsGW bool `json:"isGateway"`
|
||||
IsDefaultGW bool `json:"isDefaultGateway"`
|
||||
IPMasq bool `json:"isIPMasq"`
|
||||
Network ipamInfo `json:"ipam"`
|
||||
PodCIDR string `json:"podCIDR"`
|
||||
NodeBits string `json:"nodeBits"`
|
||||
DataStore string `json:"dataStore"`
|
||||
AuthEnabled bool `json:"auth"`
|
||||
}
|
||||
|
||||
func connectToEtcd(storeAddress string, user string, passkey string) {
|
||||
var err error
|
||||
user = strings.TrimSpace(user)
|
||||
passkey = strings.TrimSpace(passkey)
|
||||
if user != "" {
|
||||
cli, err = clientv3.New(clientv3.Config{
|
||||
Endpoints: []string{storeAddress},
|
||||
DialTimeout: 5 * time.Second,
|
||||
Username: user,
|
||||
Password: passkey,
|
||||
})
|
||||
} else {
|
||||
cli, err = clientv3.New(clientv3.Config{
|
||||
Endpoints: []string{storeAddress},
|
||||
DialTimeout: 5 * time.Second,
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Println("Error:", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
func genRangesForHosts(podCIDR string, nodePrefix int) []string {
|
||||
ip, podNet, _ := net.ParseCIDR(podCIDR)
|
||||
cidrPrefix, _ := strconv.Atoi(strings.Split(podCIDR, "/")[1])
|
||||
baseAddress := ip.Mask(net.CIDRMask(cidrPrefix, 32))
|
||||
log.Println("baseAddress", baseAddress, "podNet", podNet, "cidrPrefix", cidrPrefix)
|
||||
_, ipnet, _ := net.ParseCIDR(baseAddress.String() + "/" + strconv.Itoa(nodePrefix))
|
||||
baseMask := ipnet.Mask
|
||||
maxMask := net.CIDRMask(32, 32)
|
||||
mm := binary.BigEndian.Uint32(maxMask)
|
||||
bm := binary.BigEndian.Uint32(baseMask)
|
||||
addRange := mm - bm
|
||||
log.Println("Range is ", addRange)
|
||||
aR := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(aR, addRange)
|
||||
fmt.Println(aR)
|
||||
ba := binary.BigEndian.Uint32(baseAddress)
|
||||
log.Println("baseAddress", baseAddress, "ba", ba)
|
||||
firstNet := ba + addRange
|
||||
log.Println("firstNet in BigEndian format", firstNet)
|
||||
numRanges := math.Pow(2, float64(nodePrefix-cidrPrefix))
|
||||
hostRanges := make([]string, 0, int(numRanges))
|
||||
for addressBound, i := firstNet, 0; i < int(numRanges); i++ {
|
||||
bS := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(bS, addressBound)
|
||||
bridgeAddress := net.IP(bS).Mask(baseMask)
|
||||
bridgeAddress[3]++
|
||||
firstAddress := net.IP(bS).Mask(baseMask) // do not want to copy bridgeAddress
|
||||
firstAddress[3] += 2
|
||||
lastAddress := net.IP(bS).To4()
|
||||
lastAddress[3]--
|
||||
addressBound = addressBound + addRange + 1
|
||||
hostRanges = append(hostRanges, firstAddress.String()+","+lastAddress.String()+","+bridgeAddress.String()+"/"+strconv.Itoa(cidrPrefix))
|
||||
}
|
||||
return hostRanges
|
||||
}
|
||||
|
||||
func getNodeSubnet(podCIDR string, nodePrefix int) string {
|
||||
var selectedRange string
|
||||
candidates := genRangesForHosts(podCIDR, nodePrefix)
|
||||
s, _ := concurrency.NewSession(cli)
|
||||
defer s.Close()
|
||||
lock := concurrency.NewMutex(s, "/evpn/evio-lock/")
|
||||
ctx := context.TODO()
|
||||
if err := lock.Lock(ctx); err != nil {
|
||||
log.Println("Could not get Lock on store")
|
||||
}
|
||||
kv := clientv3.NewKV(cli)
|
||||
keyPrefix := "/evpn/"
|
||||
for _, hostRange := range candidates {
|
||||
resp, err := kv.Txn(context.TODO()).
|
||||
If(clientv3util.KeyMissing(keyPrefix + hostRange)).
|
||||
Then(clientv3.OpPut(keyPrefix+hostRange, "ok")).
|
||||
Commit()
|
||||
if resp.Succeeded {
|
||||
log.Printf("Successfully reserved hostRange %v on Store", hostRange)
|
||||
selectedRange = hostRange
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("%v", err)
|
||||
}
|
||||
}
|
||||
if err := lock.Unlock(ctx); err != nil {
|
||||
log.Println("Failed to release lock on store")
|
||||
}
|
||||
if len(selectedRange) == 0 {
|
||||
log.Println("Could not reserve any range on the Store.")
|
||||
}
|
||||
return selectedRange
|
||||
}
|
||||
|
||||
func loadBasicConf(bytes []byte) (*InitNetConf, error) {
|
||||
initialNetConf := &InitNetConf{}
|
||||
if err := json.Unmarshal(bytes, initialNetConf); err != nil {
|
||||
log.Printf("ERROR")
|
||||
return nil, fmt.Errorf("failed to inital load network configuration: %v", err)
|
||||
}
|
||||
log.Printf("%+v", *initialNetConf)
|
||||
initialNetConf.Network.Subnet = initialNetConf.PodCIDR
|
||||
initialNetConf.Network.IpamType = "host-local"
|
||||
initialNetConf.Network.RangeStart = ""
|
||||
initialNetConf.Network.RangeEnd = ""
|
||||
if initialNetConf.AuthEnabled {
|
||||
var username, password string
|
||||
log.Printf("Auth enabled\n")
|
||||
b, err := ioutil.ReadFile("/etc/credentials/username")
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
}
|
||||
username = string(b)
|
||||
b, err = ioutil.ReadFile("/etc/credentials/password")
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
}
|
||||
password = string(b)
|
||||
connectToEtcd(initialNetConf.DataStore, username, password)
|
||||
} else {
|
||||
connectToEtcd(initialNetConf.DataStore, "", "")
|
||||
}
|
||||
log.Printf("%+v", *initialNetConf)
|
||||
return initialNetConf, nil
|
||||
}
|
||||
|
||||
func SetUpNodeAddressRange() {
|
||||
cniFile := "/etc/cni/net.d/10-evio.conf"
|
||||
//check if prefix file already exists.
|
||||
_, err := os.Stat(cniFile)
|
||||
if os.IsNotExist(err) {
|
||||
jsonFile, err := os.Open("/etc/evioCNI/net-conf.json")
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
log.Println("Successfully Opened file")
|
||||
// defer the closing of our jsonFile so that we can parse it later on
|
||||
defer jsonFile.Close()
|
||||
byteValue, _ := ioutil.ReadAll(jsonFile)
|
||||
icf, _ := loadBasicConf(byteValue)
|
||||
podCIDR := icf.Network.Subnet
|
||||
podPrefix, _ := strconv.Atoi(strings.Split(podCIDR, "/")[1])
|
||||
nodeBits, _ := strconv.Atoi(icf.NodeBits)
|
||||
nodePrefix := podPrefix + nodeBits
|
||||
if 32-nodePrefix < 2 {
|
||||
log.Println("Not sufficient addresses for pods on host, allocate fewer bits for nodes")
|
||||
return
|
||||
}
|
||||
hostRange := strings.Split(getNodeSubnet(podCIDR, nodePrefix), ",")
|
||||
// manipulate RangeStart, first address in range reserved for gateway switch.
|
||||
icf.Network.RangeStart = hostRange[0]
|
||||
icf.Network.RangeEnd = hostRange[1]
|
||||
icf.BrIP = hostRange[2]
|
||||
log.Printf("startAddress %s, endAddress %s", icf.Network.RangeStart, icf.Network.RangeEnd)
|
||||
log.Printf("%+v", *icf)
|
||||
file, err := json.MarshalIndent(icf, "", " ")
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
_ = ioutil.WriteFile(cniFile, file, 0644)
|
||||
} else {
|
||||
log.Printf("CNI file %s already exists", cniFile)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
SetUpNodeAddressRange()
|
||||
}
|
||||
Loading…
Reference in New Issue