From 9bd0388b7ba61f792f388ea10f2c8c1ecfc2b92e Mon Sep 17 00:00:00 2001 From: saumitraaditya Date: Tue, 25 Aug 2020 15:39:42 -0700 Subject: [PATCH] tested build --- README.md | 45 ++- deployment/cmd.sh | 5 + deployment/evioCNI.dockerfile | 13 + deployment/evioCNI.yml | 170 ++++++++++++ src/evioPlugin.go | 497 ++++++++++++++++++++++++++++++++++ src/evioUtilities.go | 216 +++++++++++++++ 6 files changed, 945 insertions(+), 1 deletion(-) create mode 100644 deployment/cmd.sh create mode 100644 deployment/evioCNI.dockerfile create mode 100644 deployment/evioCNI.yml create mode 100644 src/evioPlugin.go create mode 100644 src/evioUtilities.go diff --git a/README.md b/README.md index facbf8f..20f3e5a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,45 @@ # evioPlugin -Kubernetes CNI pluigin to integrate EdgeVPN overlay with K8s +CNI plugin for EdgeVPN - Enhanced version of bridge plugin to work with OVS and utility to allocate IP address range +to nodes and generate config file for the plugin. +Dependency on github.com/containernetworking tag v0.7.5 +``` +mkdir -p $GOPATH/src/github.com/containernetworking +cd $GOPATH/src/github.com/containernetworking +git clone https://github.com/containernetworking/plugins.git +cd plugins +git checkout tags/v0.7.5 +cd plugins/main/ +git clone https://github.com/EdgeVPNio/evioPlugin.git +``` + + Now need to download a few dependencies. + ``` + go get github.com/j-keck/arping + go get github.com/vishvananda/netlink + go get github.com/digitalocean/go-openvswitch/ovs + go get go.etcd.io/etcd/clientv3 + ``` + + We will need three executables, for the plugin, config-gen evioUtililty and for + host-local IPAM plugin. + ``` + # assuming you are in $GOPATH/src/github.com/containernetworking/plugins/plugins/main/evioPlugin/src + go build evioPlugin.go + go build evioUtilities.go + cd $GOPATH/src/github.com/containernetworking/plugins/plugins/ipam/host-local + go build + ``` + + Now we need to copy all three executables to the deployment folder. + ``` + cd $GOPATH/src/github.com/containernetworking/plugins/plugins/main/evioPlugin/deployment + cp $GOPATH/src/github.com/containernetworking/plugins/plugins/ipam/host-local/host-local . + cp $GOPATH/src/github.com/containernetworking/plugins/plugins/main/evioPlugin/src/evioPlugin . + cp $GOPATH/src/github.com/containernetworking/plugins/plugins/main/evioPlugin/src/evioUtilities . + ``` + + Build the docker image. + +``` +docker build -t evio_plugin:0.0 -f evioCNI.dockerfile . +``` diff --git a/deployment/cmd.sh b/deployment/cmd.sh new file mode 100644 index 0000000..5ef4bf9 --- /dev/null +++ b/deployment/cmd.sh @@ -0,0 +1,5 @@ +#!/bin/sh +cp evioPlugin /opt/cni/bin +mv /opt/cni/bin/host-local /opt/cni/bin/bkp-host-local +cp host-local /opt/cni/bin +./evioUtilities diff --git a/deployment/evioCNI.dockerfile b/deployment/evioCNI.dockerfile new file mode 100644 index 0000000..4265d33 --- /dev/null +++ b/deployment/evioCNI.dockerfile @@ -0,0 +1,13 @@ +FROM partlab/ubuntu-golang + +WORKDIR /root/ +COPY ./evioUtilities . +COPY ./evioPlugin . +COPY ./host-local . +COPY ./cmd.sh . +RUN chmod +x cmd.sh +RUN chmod +x evioUtilities +RUN chmod +x evioPlugin +RUN chmod +x host-local + +CMD ["./cmd.sh"] diff --git a/deployment/evioCNI.yml b/deployment/evioCNI.yml new file mode 100644 index 0000000..335eaa0 --- /dev/null +++ b/deployment/evioCNI.yml @@ -0,0 +1,170 @@ +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: psp.evio.unprivileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default + seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default + apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default + apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default +spec: + privileged: false + volumes: + - configMap + - secret + - emptyDir + - hostPath + allowedHostPaths: + - pathPrefix: "/etc/cni/net.d" + - pathPrefix: "/etc/kube-evio" + - pathPrefix: "/run/evio" + readOnlyRootFilesystem: false + # Users and groups + runAsUser: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + fsGroup: + rule: RunAsAny + # Privilege Escalation + allowPrivilegeEscalation: false + defaultAllowPrivilegeEscalation: false + # Capabilities + allowedCapabilities: ['NET_ADMIN'] + defaultAddCapabilities: [] + requiredDropCapabilities: [] + # Host namespaces + hostPID: false + hostIPC: false + hostNetwork: true + hostPorts: + - min: 0 + max: 65535 + # SELinux + seLinux: + # SELinux is unused in CaaSP + rule: 'RunAsAny' +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: evio + namespace: kube-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: store-credentials + namespace: kube-system +data: # base64 encoded + username: ZXZwblVzZXI= + password: cGFzcw== +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-evio-cfg + namespace: kube-system + labels: + tier: node + app: evio +data: + # number of bits for host = nodePrefix - subnet + # in sample config below 26-16 = 8, can have 2^8 hosts + # each with 2^8 pods. + net-conf.json: | + { + "cniVersion": "0.3.1", + "name": "k8s-pod-network", + "type": "evioPlugin", + "bridge": "evioB1B111B", + "bridgeType": "OVS", + "isGateway": true, + "isDefaultGateway": true, + "isIPMasq": true, + "nodeBits": "8", + "podCIDR": "10.244.0.16/16", + "dataStore": "0.0.0.0:42000", + "auth": false + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-evio-ds-amd64 + namespace: kube-system + labels: + tier: node + app: evio +spec: + selector: + matchLabels: + app: evio + template: + metadata: + labels: + tier: node + app: evio + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: kubernetes.io/arch + operator: In + values: + - amd64 + hostNetwork: true + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: evio + initContainers: + - name: install-cni + image: evio_plugin:0.0 + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: evio-cfg + mountPath: /etc/evioCNI/ + - name: plugin + mountPath: /opt/cni/bin/ + - name: store-credentials + mountPath: /etc/credentials/ + containers: + - name: dumdum + image: gcr.io/kubernetes-e2e-test-images/dnsutils:1.3 + command: + - sleep + args: + - "3600" + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: false + capabilities: + add: ["NET_ADMIN"] + volumes: + - name: cni + hostPath: + path: /etc/cni/net.d + - name: plugin + hostPath: + path: /opt/cni/bin/ + - name: evio-cfg + configMap: + name: kube-evio-cfg + - name: store-credentials + secret: + secretName: store-credentials diff --git a/src/evioPlugin.go b/src/evioPlugin.go new file mode 100644 index 0000000..5e6a7d9 --- /dev/null +++ b/src/evioPlugin.go @@ -0,0 +1,497 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "net" + "os" + "os/exec" + "runtime" + "strings" + "syscall" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/cni/pkg/version" + "github.com/containernetworking/plugins/pkg/ip" + "github.com/containernetworking/plugins/pkg/ipam" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/containernetworking/plugins/pkg/utils" + "github.com/digitalocean/go-openvswitch/ovs" + "github.com/j-keck/arping" + "github.com/vishvananda/netlink" +) + +type NetConf struct { + types.NetConf + BrName string `json:"bridge"` + BrType string `json:"bridgeType"` + BrIP string `json:"bridgeIPv4"` + IsGW bool `json:"isGateway"` + IsDefaultGW bool `json:"isDefaultGateway"` + IPMasq bool `json:"isIPMasq"` + subnetPath string `json:"subnetPath"` +} + +type gwInfo struct { + gws []net.IPNet + family int + defaultRouteFound bool +} + +func loadNetConf(bytes []byte) (*NetConf, string, error) { + n := &NetConf{} + if err := json.Unmarshal(bytes, n); err != nil { + return nil, "", fmt.Errorf("failed to load network configuration: %v", err) + } + log.Printf("%+v", *n) + return n, n.CNIVersion, nil +} + +func configureBridgeInterface(bridgeName string, gatewayAddress string) error { + + i, err := net.InterfaceByName(bridgeName) + if err != nil { + log.Println("Bridge interface not found.") + return err + } + ipv4set := false + var ip net.IP + ipv4 := "" + + addrs, err := i.Addrs() + if err != nil { + log.Println("Could not get addresses associated with the bridge.") + return err + } + for _, addr := range addrs { + switch v := addr.(type) { + case *net.IPAddr: + ip = v.IP + case *net.IPNet: + ip = v.IP + } + log.Printf("Bridge address %s\n", addr.String()) + if ip.To4() != nil { + if strings.Compare(gatewayAddress, addr.String()) == 0 { + log.Printf("Interface %s already configured with address %s\n", bridgeName, addr.String()) + ipv4set = true + } else { + ipv4 = addr.String() + } + } + } + if ipv4set == true { + return nil + } + // need to assign gateway address to switch and bring it up. + ipExecutable, err := exec.LookPath("ip") + if err != nil { + log.Println("failed to locate executable ip") + return err + } + if ipv4 != "" { + log.Println(" some other ipv4 address %s is assigned in place of %s ", ipv4, gatewayAddress) + } + cmdDelIP := &exec.Cmd{ + Path: ipExecutable, + Args: []string{ipExecutable, "addr", "del", ipv4, "dev", bridgeName}, + } + if ipv4 != "" { + if output, err := cmdDelIP.Output(); err != nil { + log.Println(err.Error()) + return err + } else { + log.Println(output) + } + } + cmdSetIP := &exec.Cmd{ + Path: ipExecutable, + Args: []string{ipExecutable, "addr", "add", gatewayAddress, "dev", bridgeName}, + } + if output, err := cmdSetIP.Output(); err != nil { + log.Println(err.Error()) + return err + } else { + log.Println(output) + } + + cmdBringUP := &exec.Cmd{ + Path: ipExecutable, + Args: []string{ipExecutable, "link", "set", bridgeName, "up"}, + } + if output, err := cmdBringUP.Output(); err != nil { + log.Println(err.Error()) + return err + } else { + log.Println(output) + } + return nil +} + +func setupBridge(netconf *NetConf) (*netlink.Bridge, *current.Interface, error) { + br := &netlink.Bridge{ + LinkAttrs: netlink.LinkAttrs{ + Name: netconf.BrName, + MTU: 1500, + TxQLen: -1, + }, + } + // this section adds and sets up a linux bridge + if netconf.BrType == "LXBR" { + err := netlink.LinkAdd(br) + if err != nil && err != syscall.EEXIST { + return nil, nil, err + } + + if err := netlink.LinkSetUp(br); err != nil { + return nil, nil, err + } + } else if netconf.BrType == "OVS" { + // setting up an ovs bridge + ovsClient := ovs.New(ovs.Sudo()) + if err := ovsClient.VSwitch.AddBridge(br.Attrs().Name); err != nil { + log.Fatalf("failed to add OVS bridge: %v", err) + } + } + return br, ¤t.Interface{ + Name: br.Attrs().Name, + Mac: br.Attrs().HardwareAddr.String(), + }, nil +} + +func setupVeth(br *netlink.Bridge, netconf *NetConf, args *skel.CmdArgs) (*current.Interface, *current.Interface, error) { + netns, err := ns.GetNS(args.Netns) + if err != nil { + log.Fatalf("Could not find network namespace %v", err) + return nil, nil, fmt.Errorf("Could not find network namespace %v", err) + } + contIface := ¤t.Interface{} + hostIface := ¤t.Interface{} + var handler = func(hostNS ns.NetNS) error { + hostVeth, containerVeth, err := ip.SetupVeth(args.IfName, 1500, hostNS) + if err != nil { + return err + } + contIface.Name = containerVeth.Name + contIface.Mac = containerVeth.HardwareAddr.String() + contIface.Sandbox = netns.Path() + hostIface.Name = hostVeth.Name + log.Printf("Created pod veth interface: %v\n", containerVeth.Name) + return nil + } + if err := netns.Do(handler); err != nil { + return nil, nil, err + } + hostVeth, err := netlink.LinkByName(hostIface.Name) + if err != nil { + return nil, nil, err + } + log.Printf("Located host veth interface: %v\n", hostVeth.Attrs().Name) + hostIface.Mac = hostVeth.Attrs().HardwareAddr.String() + if netconf.BrType == "LXBR" { + // below attaches veth interface to the bridge. + if err := netlink.LinkSetMaster(hostVeth, br); err != nil { + return nil, nil, err + } + } else if netconf.BrType == "OVS" { + // below attaches veth interface to ovs bridge + ovsClient := ovs.New(ovs.Sudo()) + if err := ovsClient.VSwitch.AddPort(br.Attrs().Name, hostIface.Name); err != nil { + log.Printf("failed to add %v to bridge %v error %v", hostIface.Name, br.Attrs().Name, err) + } + log.Printf("Attached %v to %v\n", hostVeth.Attrs().Name, br.Name) + } + return hostIface, contIface, nil + +} + +func cmdAdd(args *skel.CmdArgs) error { + n, cniVersion, err := loadNetConf(args.StdinData) + if err != nil { + return err + } + + log.Printf("%v\t %v\t %v", args.ContainerID, args.IfName, args.Netns) + log.Println("parsed configuration successfully !") + br, brInterface, err := setupBridge(n) + if err != nil { + return err + } + log.Printf("set up bridge %v successfully !\n", br.Name) + // set up IP address on the bridge and bring it up. + err = configureBridgeInterface(n.BrName, n.BrIP) + if err != nil { + log.Println("Failure in configuring bridge.") + return err + } + hostInterface, containerInterface, err := setupVeth(br, n, args) + if err != nil { + log.Println("Failure in setting up Veth interfaces.") + return err + } + log.Println("set up veth interfaces successfully !") + + result := ¤t.Result{CNIVersion: cniVersion, Interfaces: []*current.Interface{brInterface, hostInterface, containerInterface}} + r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData) + if err != nil { + return err + } + var success bool = false + // release IP in case of failure + defer func() { + if !success { + os.Setenv("CNI_COMMAND", "DEL") + ipam.ExecDel(n.IPAM.Type, args.StdinData) + os.Setenv("CNI_COMMAND", "ADD") + } + }() + + // Convert whatever the IPAM result was into the current Result type + ipamResult, err := current.NewResultFromResult(r) + if err != nil { + log.Printf("could not convert IPAM result %+v \n", ipamResult) + return err + } + log.Printf("result from IPAM : %+v\n", ipamResult) + result.IPs = ipamResult.IPs + result.Routes = ipamResult.Routes + + if len(result.IPs) == 0 { + log.Printf("IPAM plugin provided no IP config\n") + return errors.New("IPAM plugin returned missing IP config") + } + netns, err := ns.GetNS(args.Netns) + if err != nil { + log.Printf("could not find namespace %v", args.Netns) + return err + } + for _, ipc := range result.IPs { + ipc.Interface = current.Int(2) + } + + gwsV4, gwsV6, err := calcGateways(result, n) + if err != nil { + return err + } + // Configure the container hardware address and IP address(es) + if err := netns.Do(func(_ ns.NetNS) error { + contVeth, err := net.InterfaceByName(args.IfName) + if err != nil { + log.Printf("could not find interface %v", contVeth.Name) + return err + } + // Add the IP to the interface + if err := ipam.ConfigureIface(args.IfName, result); err != nil { + log.Printf("could not configure IP address on the interface %v", args.IfName) + return err + } + + // Send a gratuitous arp + for _, ipc := range result.IPs { + if ipc.Version == "4" { + _ = arping.GratuitousArpOverIface(ipc.Address.IP, *contVeth) + } + } + return nil + }); err != nil { + log.Printf("something went wrong while trying to configure IP address on the interface !") + return err + } + if n.IsGW { + // Set the IP address(es) on the bridge and enable forwarding + for _, gws := range []*gwInfo{gwsV4, gwsV6} { + if gws.gws != nil { + if err = enableIPForward(gws.family); err != nil { + return fmt.Errorf("failed to enable forwarding: %v", err) + } + } + } + } + + if n.IPMasq { + chain := utils.FormatChainName(n.Name, args.ContainerID) + comment := utils.FormatComment(n.Name, args.ContainerID) + for _, ipc := range result.IPs { + if err = ip.SetupIPMasq(ip.Network(&ipc.Address), chain, comment); err != nil { + log.Printf("something went wrong while to set up IPMasq %v!", err) + return err + } + log.Printf("IPMasq set up successfully !!") + } + } + if n.BrType == "LXBR" { + // commented out because accesses variables set when setting linux bridges + l, err := netlink.LinkByName(br.Name) + if err != nil { + log.Printf("could not lookup %q: %v", br.Name, err) + } + br, ok := l.(*netlink.Bridge) + if !ok { + log.Printf("%q already exists but is not a bridge", br.Name) + } + } + log.Printf("result %v\n", result) + return types.PrintResult(result, cniVersion) +} + +func clearDeadPortsOnBridge(brName string) { + ovsClient := ovs.New(ovs.Sudo()) + listOfPorts, err := ovsClient.VSwitch.ListPorts(brName) + // dead interfaces are cleared and interfaces deleted outside cni plugin, just check and remove from + // the bridge, happens for previously deleted namespaces not the current one being deleted. + if err != nil { + log.Printf("Could not list ports on bridge %s, error is %v", brName, err.Error()) + } else { + for _, portName := range listOfPorts { + _, err := netlink.LinkByName(portName) + if err != nil { + if err = ovsClient.VSwitch.DeletePort(brName, portName); err != nil { + log.Printf("failed to remove %v from bridge %v error %v", brName, portName, err) + } + } + } + } +} +func cmdDel(args *skel.CmdArgs) error { + n, _, err := loadNetConf(args.StdinData) + if err != nil { + return err + } + + log.Printf("%v\t %v\t %v", args.ContainerID, args.IfName, args.Netns) + log.Println("Delete: parsed configuration successfully !") + if args.Netns == "" { + return nil + } + var ipnets []*net.IPNet + err = ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error { + var err error + ipnets, err = ip.DelLinkByNameAddr(args.IfName) + if err != nil && err == ip.ErrLinkNotFound { + log.Printf("could not delete NS interface %v", err.Error()) + return nil + } + return err + }) + if n.IPMasq { + chain := utils.FormatChainName(n.Name, args.ContainerID) + comment := utils.FormatComment(n.Name, args.ContainerID) + for _, ipn := range ipnets { + if err := ip.TeardownIPMasq(ipn, chain, comment); err != nil { + log.Printf("could not reset IPMasq %v", err.Error()) + return err + } + } + } + if n.BrType == "OVS" { + clearDeadPortsOnBridge(n.BrName) + } + if n.IPAM.Type != "" { + if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil { + log.Printf("could not clear out IPAM %v", err.Error()) + return err + } + } + + return nil +} + + +func main() { + skel.PluginMain(cmdAdd, cmdDel, version.All) +} + +func enableIPForward(family int) error { + if family == netlink.FAMILY_V4 { + return ip.EnableIP4Forward() + } + return ip.EnableIP6Forward() +} + +func calcGateways(result *current.Result, n *NetConf) (*gwInfo, *gwInfo, error) { + + gwsV4 := &gwInfo{} + gwsV6 := &gwInfo{} + + for _, ipc := range result.IPs { + + // Determine if this config is IPv4 or IPv6 + var gws *gwInfo + defaultNet := &net.IPNet{} + switch { + case ipc.Address.IP.To4() != nil: + gws = gwsV4 + gws.family = netlink.FAMILY_V4 + defaultNet.IP = net.IPv4zero + case len(ipc.Address.IP) == net.IPv6len: + gws = gwsV6 + gws.family = netlink.FAMILY_V6 + defaultNet.IP = net.IPv6zero + default: + return nil, nil, fmt.Errorf("Unknown IP object: %v", ipc) + } + defaultNet.Mask = net.IPMask(defaultNet.IP) + + // All IPs currently refer to the container interface + ipc.Interface = current.Int(2) + + if n.IsGW { + ipc.Gateway = getGatewayIP(n) + } + + // Add a default route for this family using the current + // gateway address if necessary. + if n.IsDefaultGW && !gws.defaultRouteFound { + for _, route := range result.Routes { + if route.GW != nil && defaultNet.String() == route.Dst.String() { + gws.defaultRouteFound = true + break + } + } + if !gws.defaultRouteFound { + result.Routes = append( + result.Routes, + &types.Route{Dst: *defaultNet, GW: ipc.Gateway}, + ) + gws.defaultRouteFound = true + } + } + + // Append this gateway address to the list of gateways + if n.IsGW { + gw := net.IPNet{ + IP: ipc.Gateway, + Mask: ipc.Address.Mask, + } + gws.gws = append(gws.gws, gw) + } + } + return gwsV4, gwsV6, nil +} + +func calcGatewayIP(ipn *net.IPNet) net.IP { + nid := ipn.IP.Mask(ipn.Mask) + return ip.NextIP(nid) +} + +func getGatewayIP(n *NetConf) net.IP { + gwIP := net.ParseIP(strings.Split(n.BrIP, "/")[0]) + log.Printf("Gateway IPv4 address for bridge is %v !!", gwIP.String()) + return gwIP +} + +func init() { + // this ensures that main runs only on main thread (thread group leader). + // since namespace ops (unshare, setns) are done for a single thread, we + // must ensure that the goroutine does not jump from OS thread to thread + runtime.LockOSThread() + file, err := os.OpenFile("/var/log/evioPlugin.log", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + log.Fatal(err) + } + log.SetOutput(file) +} diff --git a/src/evioUtilities.go b/src/evioUtilities.go new file mode 100644 index 0000000..19ee3e7 --- /dev/null +++ b/src/evioUtilities.go @@ -0,0 +1,216 @@ +package main + +import ( + "context" + "encoding/binary" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "math" + "net" + "os" + "strconv" + "strings" + "time" + + "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/clientv3/clientv3util" + "go.etcd.io/etcd/clientv3/concurrency" +) + +var cli *clientv3.Client + +type ipamInfo struct { + IpamType string `json:"type"` + Subnet string `json:"subnet"` + RangeStart string `json:"rangeStart"` + RangeEnd string `json:"rangeEnd"` +} + +type InitNetConf struct { + CniVersion string `json:"cniVersion"` + Name string `json:"name"` + Type string `json:"type"` + BrName string `json:"bridge"` + BrType string `json:"bridgeType"` + BrIP string `json:"bridgeIPv4"` + IsGW bool `json:"isGateway"` + IsDefaultGW bool `json:"isDefaultGateway"` + IPMasq bool `json:"isIPMasq"` + Network ipamInfo `json:"ipam"` + PodCIDR string `json:"podCIDR"` + NodeBits string `json:"nodeBits"` + DataStore string `json:"dataStore"` + AuthEnabled bool `json:"auth"` +} + +func connectToEtcd(storeAddress string, user string, passkey string) { + var err error + user = strings.TrimSpace(user) + passkey = strings.TrimSpace(passkey) + if user != "" { + cli, err = clientv3.New(clientv3.Config{ + Endpoints: []string{storeAddress}, + DialTimeout: 5 * time.Second, + Username: user, + Password: passkey, + }) + } else { + cli, err = clientv3.New(clientv3.Config{ + Endpoints: []string{storeAddress}, + DialTimeout: 5 * time.Second, + }) + } + if err != nil { + fmt.Println("Error:", err) + os.Exit(2) + } +} +func genRangesForHosts(podCIDR string, nodePrefix int) []string { + ip, podNet, _ := net.ParseCIDR(podCIDR) + cidrPrefix, _ := strconv.Atoi(strings.Split(podCIDR, "/")[1]) + baseAddress := ip.Mask(net.CIDRMask(cidrPrefix, 32)) + log.Println("baseAddress", baseAddress, "podNet", podNet, "cidrPrefix", cidrPrefix) + _, ipnet, _ := net.ParseCIDR(baseAddress.String() + "/" + strconv.Itoa(nodePrefix)) + baseMask := ipnet.Mask + maxMask := net.CIDRMask(32, 32) + mm := binary.BigEndian.Uint32(maxMask) + bm := binary.BigEndian.Uint32(baseMask) + addRange := mm - bm + log.Println("Range is ", addRange) + aR := make([]byte, 4) + binary.BigEndian.PutUint32(aR, addRange) + fmt.Println(aR) + ba := binary.BigEndian.Uint32(baseAddress) + log.Println("baseAddress", baseAddress, "ba", ba) + firstNet := ba + addRange + log.Println("firstNet in BigEndian format", firstNet) + numRanges := math.Pow(2, float64(nodePrefix-cidrPrefix)) + hostRanges := make([]string, 0, int(numRanges)) + for addressBound, i := firstNet, 0; i < int(numRanges); i++ { + bS := make([]byte, 4) + binary.BigEndian.PutUint32(bS, addressBound) + bridgeAddress := net.IP(bS).Mask(baseMask) + bridgeAddress[3]++ + firstAddress := net.IP(bS).Mask(baseMask) // do not want to copy bridgeAddress + firstAddress[3] += 2 + lastAddress := net.IP(bS).To4() + lastAddress[3]-- + addressBound = addressBound + addRange + 1 + hostRanges = append(hostRanges, firstAddress.String()+","+lastAddress.String()+","+bridgeAddress.String()+"/"+strconv.Itoa(cidrPrefix)) + } + return hostRanges +} + +func getNodeSubnet(podCIDR string, nodePrefix int) string { + var selectedRange string + candidates := genRangesForHosts(podCIDR, nodePrefix) + s, _ := concurrency.NewSession(cli) + defer s.Close() + lock := concurrency.NewMutex(s, "/evpn/evio-lock/") + ctx := context.TODO() + if err := lock.Lock(ctx); err != nil { + log.Println("Could not get Lock on store") + } + kv := clientv3.NewKV(cli) + keyPrefix := "/evpn/" + for _, hostRange := range candidates { + resp, err := kv.Txn(context.TODO()). + If(clientv3util.KeyMissing(keyPrefix + hostRange)). + Then(clientv3.OpPut(keyPrefix+hostRange, "ok")). + Commit() + if resp.Succeeded { + log.Printf("Successfully reserved hostRange %v on Store", hostRange) + selectedRange = hostRange + break + } + if err != nil { + log.Printf("%v", err) + } + } + if err := lock.Unlock(ctx); err != nil { + log.Println("Failed to release lock on store") + } + if len(selectedRange) == 0 { + log.Println("Could not reserve any range on the Store.") + } + return selectedRange +} + +func loadBasicConf(bytes []byte) (*InitNetConf, error) { + initialNetConf := &InitNetConf{} + if err := json.Unmarshal(bytes, initialNetConf); err != nil { + log.Printf("ERROR") + return nil, fmt.Errorf("failed to inital load network configuration: %v", err) + } + log.Printf("%+v", *initialNetConf) + initialNetConf.Network.Subnet = initialNetConf.PodCIDR + initialNetConf.Network.IpamType = "host-local" + initialNetConf.Network.RangeStart = "" + initialNetConf.Network.RangeEnd = "" + if initialNetConf.AuthEnabled { + var username, password string + log.Printf("Auth enabled\n") + b, err := ioutil.ReadFile("/etc/credentials/username") + if err != nil { + log.Println(err.Error()) + } + username = string(b) + b, err = ioutil.ReadFile("/etc/credentials/password") + if err != nil { + log.Println(err.Error()) + } + password = string(b) + connectToEtcd(initialNetConf.DataStore, username, password) + } else { + connectToEtcd(initialNetConf.DataStore, "", "") + } + log.Printf("%+v", *initialNetConf) + return initialNetConf, nil +} + +func SetUpNodeAddressRange() { + cniFile := "/etc/cni/net.d/10-evio.conf" + //check if prefix file already exists. + _, err := os.Stat(cniFile) + if os.IsNotExist(err) { + jsonFile, err := os.Open("/etc/evioCNI/net-conf.json") + if err != nil { + fmt.Println(err) + } + log.Println("Successfully Opened file") + // defer the closing of our jsonFile so that we can parse it later on + defer jsonFile.Close() + byteValue, _ := ioutil.ReadAll(jsonFile) + icf, _ := loadBasicConf(byteValue) + podCIDR := icf.Network.Subnet + podPrefix, _ := strconv.Atoi(strings.Split(podCIDR, "/")[1]) + nodeBits, _ := strconv.Atoi(icf.NodeBits) + nodePrefix := podPrefix + nodeBits + if 32-nodePrefix < 2 { + log.Println("Not sufficient addresses for pods on host, allocate fewer bits for nodes") + return + } + hostRange := strings.Split(getNodeSubnet(podCIDR, nodePrefix), ",") + // manipulate RangeStart, first address in range reserved for gateway switch. + icf.Network.RangeStart = hostRange[0] + icf.Network.RangeEnd = hostRange[1] + icf.BrIP = hostRange[2] + log.Printf("startAddress %s, endAddress %s", icf.Network.RangeStart, icf.Network.RangeEnd) + log.Printf("%+v", *icf) + file, err := json.MarshalIndent(icf, "", " ") + if err != nil { + fmt.Println(err) + return + } + _ = ioutil.WriteFile(cniFile, file, 0644) + } else { + log.Printf("CNI file %s already exists", cniFile) + return + } +} + +func main() { + SetUpNodeAddressRange() +}