From ed806de15d71a1086d262366a510d3bc7a10dbc0 Mon Sep 17 00:00:00 2001 From: Ken Subratie Date: Thu, 25 May 2023 14:39:19 -0400 Subject: [PATCH] Update build tools for v23 --- config/config_gen.py | 111 +++++ config/template.py | 92 ++++ debian-package/evio/DEBIAN/postinst | 14 +- debian-package/evio/opt/evio/evio-start | 3 +- docker-image/evio-node.Dockerfile | 66 ++- scripts/ans-deploy-evio20.yml | 18 - scripts/ans-deploy-evio21.yml | 20 - scripts/ans-deploy-evioarm.yml | 18 - scripts/ans-remove-evio.yml | 9 - scripts/deploy-evio-docker.sh | 46 ++ scripts/evio-control.py | 13 +- scripts/evio-tools-v23.sh | 365 ++++++++++++++++ scripts/overlay-test.sh | 48 -- scripts/ping-hosts | 5 - scripts/update-limits.sh | 0 setup-evt.sh | 7 +- testbed/template-config.json | 86 ---- testbed/testbed.py | 555 ------------------------ versioning/Versioning.py | 146 ------- versioning/tool_config.py | 25 +- versioning/versioning.py | 202 +++++++++ 21 files changed, 924 insertions(+), 925 deletions(-) create mode 100755 config/config_gen.py create mode 100644 config/template.py delete mode 100644 scripts/ans-deploy-evio20.yml delete mode 100644 scripts/ans-deploy-evio21.yml delete mode 100644 scripts/ans-deploy-evioarm.yml delete mode 100644 scripts/ans-remove-evio.yml create mode 100644 scripts/deploy-evio-docker.sh create mode 100755 scripts/evio-tools-v23.sh delete mode 100755 scripts/overlay-test.sh delete mode 100644 scripts/ping-hosts mode change 100755 => 100644 scripts/update-limits.sh delete mode 100644 testbed/template-config.json delete mode 100755 testbed/testbed.py delete mode 100644 versioning/Versioning.py create mode 100644 versioning/versioning.py diff --git a/config/config_gen.py b/config/config_gen.py new file mode 100755 index 0000000..944aa83 --- /dev/null +++ b/config/config_gen.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +try: + import simplejson as json +except ImportError: + import json + +import argparse +import ipaddress +import os +import pathlib +import pickle +import random +import shutil +import subprocess +import sys +import time +from copy import deepcopy + +from template import BASE_TEMAPLTE, NODE_ID, UPDATES + + +class ConfigGen: + def __init__( + self, + ): + parser = argparse.ArgumentParser( + description="Generates EdgeVPN config files for a range of nodes", + fromfile_prefix_chars="@", + ) + parser.add_argument( + "-t", + "--template", + action="store", + dest="template", + type=str, + help="Fully qualified filename for configuration template used to generate the others", + ) + parser.add_argument( + "-u", + "--updates", + action="store", + dest="updates", + help="Fully qualified filename for configuration parameters to merge/overwrite on the template ", + ) + parser.add_argument( + "-o", + "--output_dir", + action="store", + dest="output_dir", + type=pathlib.Path, + help="Directory for generated configuration files", + ) + + parser.add_argument( + "-r", + "--range", + action="store", + dest="range", + nargs=2, + required=True, + help="Specifies the range of files to genereate", + ) + self.args = parser.parse_args() + self.range_end = int(self.args.range[1]) + self.range_start = int(self.args.range[0]) + if not self.args.output_dir: + self.output_dir = os.path.abspath(".") + else: + self.args.output_dir.mkdir(exist_ok=True) + self.output_dir = str(self.args.output_dir) + self.config_file_base = "{0}/config-".format(self.output_dir) + + def gen(self): + config = deepcopy(UPDATES) + node_id = NODE_ID + for val in range(self.range_start, self.range_end): + rng_str = "{0:03}".format(val) + cfg_file = f"{self.config_file_base}{rng_str}.json" + node_id = "{0}{1}{2}{1}{3}".format( + node_id[:4], rng_str, node_id[7:29], node_id[32:] + ) + overlays = config["Broker"].get("Overlays", []) + config["Broker"]["NodeId"] = node_id + for olid in overlays: + netwk = config["BridgeController"]["Overlays"][olid]["NetDevice"][ + "AppBridge" + ].pop("NetworkAddress") + netwk = ipaddress.IPv4Network(netwk) + node_ip = str(netwk[val]) + config["BridgeController"]["Overlays"][olid]["NetDevice"]["AppBridge"][ + "IP4" + ] = node_ip + config["BridgeController"]["Overlays"][olid]["NetDevice"]["AppBridge"][ + "PrefixLen" + ] = netwk.prefixlen + config["Topology"]["Overlays"][olid]["LocationId"] = val + with open(cfg_file, "w") as cfg_fle: + json.dump(config, cfg_fle, indent=2) + cfg_fle.flush() + config = deepcopy(UPDATES) + + print("{0} config file(s) generated".format(self.range_end - self.range_start)) + + +def main(): # pylint: disable=too-many-return-statements + cfg = ConfigGen() + cfg.gen() + + +if __name__ == "__main__": + main() diff --git a/config/template.py b/config/template.py new file mode 100644 index 0000000..8ad5163 --- /dev/null +++ b/config/template.py @@ -0,0 +1,92 @@ +BASE_TEMAPLTE = { + "Broker": {"Overlays": ["_OVERLAYNAME_"]}, + "Signal": { + "Overlays": { + "_OVERLAYNAME_": { + "HostAddress": "A.B.C.D", + "AuthenticationMethod": "PASSWORD", + "Port": "5222", + "Username": "test1@openfire.local", + "Password": "password_test1", + } + } + }, + "LinkManager": { + "Stun": ["stun.l.google.com:19302", "stun1.l.google.com:19302"], + "Overlays": { + "_OVERLAYNAME_": { + "IgnoredNetInterfaces": ["flannel.1", "cni0", "docker0", "ovs-system"] + } + }, + }, + "BridgeController": { + "BoundedFlood": {"Overlays": {"_OVERLAYNAME_": {}}}, + "Overlays": {"_OVERLAYNAME_": {}}, + }, +} + + +UPDATES = { + "Broker": { + "Overlays": ["PiK8kEG"], + "NodeId": "", + "LogLevel": "DEBUG", + }, + "Signal": { + "Overlays": { + "PiK8kEG": { + "HostAddress": "trial.edgevpn.io", + "AuthenticationMethod": "PASSWORD", + "Port": "5222", + "Username": "PiK8kEG_node9@trial.edgevpn.io", + "Password": "ztgxxfunbisptvrw", + } + } + }, + "Topology": { + "StateTracingEnabled": True, + "Overlays": {"PiK8kEG": {"LocationId": 0, "EncryptionRequired": False}}, + }, + "LinkManager": { + "Stun": ["stun.l.google.com:19302", "stun1.l.google.com:19302"], + "Turn": [ + { + "Address": "trial.edgevpn.io:3478", + "User": "PiK8kEG_node9", + "Password": "ztgxxfunbisptvrw", + } + ], + "IgnoredNetInterfaces": [ + "flannel.1", + "cni0", + "docker0", + "nodelocaldns", + "kube-ipvs0", + "ovs-system", + "nebula1", + ], + }, + "GeneveTunnel": {"Overlays": {"PiK8kEG": {"EndPointAddress": "*.*.*.*"}}}, + "BridgeController": { + "BoundedFlood": { + "LogLevel": "INFO", + "StateTracingEnabled": True, + "Overlays": {"PiK8kEG": {}}, + }, + "Overlays": { + "PiK8kEG": { + "NetDevice": { + "MTU": 1410, + "AppBridge": {"MTU": 1350, "NetworkAddress": "10.10.100.0/24"}, + } + } + }, + }, + "UsageReport": { + "Enabled": True, + "TimerInterval": 3600, + "WebService": "https://qdscz6pg37.execute-api.us-west-2.amazonaws.com/default/EvioUsageReport", + }, +} + +NODE_ID = "a100###ffffffffffffffffffffff###" diff --git a/debian-package/evio/DEBIAN/postinst b/debian-package/evio/DEBIAN/postinst index 54a8c7f..1fa5ed4 100755 --- a/debian-package/evio/DEBIAN/postinst +++ b/debian-package/evio/DEBIAN/postinst @@ -1,10 +1,12 @@ #!/bin/bash -cd /opt/evio -python3.8 -m venv venv -source venv/bin/activate -pip3 --cache-dir /var/cache/evio/ install wheel -pip3 --cache-dir /var/cache/evio/ install eventlet==0.30.2 psutil slixmpp requests simplejson ryu pyroute2 -deactivate +if [ ! -d /opt/evio/venv ]; then + cd /opt/evio + python3.9 -m venv venv + source venv/bin/activate + pip3 --cache-dir /var/cache/evio/ install wheel + pip3 --cache-dir /var/cache/evio/ install eventlet==0.30.2 slixmpp requests simplejson ryu pyroute2 keyring + deactivate +fi systemctl daemon-reload systemctl enable evio \ No newline at end of file diff --git a/debian-package/evio/opt/evio/evio-start b/debian-package/evio/opt/evio/evio-start index 747c7fe..7d0acd0 100755 --- a/debian-package/evio/opt/evio/evio-start +++ b/debian-package/evio/opt/evio/evio-start @@ -2,9 +2,8 @@ evio_start() { cd /opt/evio - ./tincan & source venv/bin/activate - python Controller.py -c /etc/opt/evio/config.json & + python evio_controller.py -c /etc/opt/evio/config.json & } evio_start diff --git a/docker-image/evio-node.Dockerfile b/docker-image/evio-node.Dockerfile index 4a82e42..7d696fc 100755 --- a/docker-image/evio-node.Dockerfile +++ b/docker-image/evio-node.Dockerfile @@ -1,7 +1,65 @@ -FROM edgevpnio/evio-base:1.2 -WORKDIR /root/ -COPY ./evio.deb . -RUN apt-get install -y ./evio.deb && rm -rf /var/lib/apt/lists/* && apt-get autoclean +FROM ubuntu:jammy AS base +RUN echo 'root:root' | chpasswd +RUN printf '#!/bin/sh\nexit 0' > /usr/sbin/policy-rc.d +RUN apt-get update && \ + apt-get install -y systemd systemd-sysv dbus dbus-user-session +FROM base AS evio-base +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC +RUN apt-get update && \ + apt-get install -y \ + openvswitch-switch \ + iputils-ping \ + fping \ + libffi-dev \ + software-properties-common \ + iproute2 \ + bridge-utils \ + tcpdump && \ + add-apt-repository ppa:deadsnakes/ppa && \ + apt-get update +RUN apt-get install -y \ + python3.9 \ + python3.9-dev \ + python3.9-venv \ + python3-pip \ + python3-wheel + +WORKDIR /opt/evio/ +ENV VIRTUAL_ENV=/opt/evio/venv +RUN python3.9 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN pip3 install --upgrade pip && \ + pip3 --cache-dir /var/cache/evio/ \ + install wheel && \ + pip3 --cache-dir /var/cache/evio/ \ + install eventlet==0.30.2 psutil \ + slixmpp requests simplejson \ + pyroute2 keyring ryu +#RUN systemctl mask getty@tty1.service + +FROM evio-base +WORKDIR /var/cache/evio +ARG TARGETPLATFORM +ARG TARGETARCH +ARG BUILDPLATFORM +COPY ./evio_$TARGETARCH.deb . +WORKDIR /root +RUN apt-get install -y /var/cache/evio/evio_$TARGETARCH.deb && \ + rm -rf /var/lib/apt/lists/* \ + /var/cache/evio/ && \ + apt-get autoclean CMD ["/sbin/init"] +ARG DATE +ARG VERSION +ENV CREATED=$DATE +ENV VERSION=$VERSION + +LABEL org.opencontainers.image.created=$DATE \ + org.opencontainers.image.authors="ACIS Lab" \ + org.opencontainers.image.url="https://edgevpn.io" \ + org.opencontainers.image.source="https://github.com/EdgeVPNio/evio" \ + org.opencontainers.image.version=$VERSION \ + org.opencontainers.image.title="Virtualized overlay networking for the fog" diff --git a/scripts/ans-deploy-evio20.yml b/scripts/ans-deploy-evio20.yml deleted file mode 100644 index aaf3e2a..0000000 --- a/scripts/ans-deploy-evio20.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- hosts: evio20 - tasks: - - name: Copying evio deb-pak - copy: - src: /home/kcratie/workspace/out/evio_20.12.5_amd64.deb - dest: /tmp/ - - name: Installing evio - apt: - deb: /tmp/evio_20.12.5_amd64.deb - become: yes - become_method: sudo - - name: Starting evio daemon - service: - name: evio - state: started - become: yes - become_method: sudo diff --git a/scripts/ans-deploy-evio21.yml b/scripts/ans-deploy-evio21.yml deleted file mode 100644 index 75f7e92..0000000 --- a/scripts/ans-deploy-evio21.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- hosts: evio21 - become: yes - become_method: sudo - tasks: - - name: Copying evio deb-pak - copy: - src: /home/kcratie/workspace/out/evio_21.6.0.125-dev_amd64.deb - dest: /tmp/ - - name: Installing evio - apt: - deb: /tmp/evio_21.6.0.125-dev_amd64.deb - - name: Copying evio configuration - copy: - src: /home/kcratie/workspace/configs21/config-{{inventory_hostname}}.json - dest: /etc/opt/evio/config.json - - name: Starting evio daemon - service: - name: evio - state: restarted \ No newline at end of file diff --git a/scripts/ans-deploy-evioarm.yml b/scripts/ans-deploy-evioarm.yml deleted file mode 100644 index 428883b..0000000 --- a/scripts/ans-deploy-evioarm.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- hosts: evioarm - tasks: - - name: Copying evio deb-pak - copy: - src: /home/kcratie/workspace/out/evio_20.12.2_arm64.deb - dest: /home/kcratie/workspace/ - - name: Installing evio - apt: - deb: /home/kcratie/workspace/evio_20.12.2_arm64.deb - become: yes - become_method: sudo - - name: Starting evio daemon - service: - name: evio - state: started - become: yes - become_method: sudo diff --git a/scripts/ans-remove-evio.yml b/scripts/ans-remove-evio.yml deleted file mode 100644 index 01bef24..0000000 --- a/scripts/ans-remove-evio.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- hosts: evio21 - tasks: - - name: Removing evio - apt: - name: evio - state: absent - become: yes - become_method: sudo \ No newline at end of file diff --git a/scripts/deploy-evio-docker.sh b/scripts/deploy-evio-docker.sh new file mode 100644 index 0000000..4eb53e8 --- /dev/null +++ b/scripts/deploy-evio-docker.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +export NODE_IDX=097 + +sudo apt-get update && sudo apt-get upgrade -y +# Install prereqs and openvswitch kernel module on raspberry-pi. +# linux-modules-extra-raspi is currently needed for rpi4/ubuntu server 22.04 +sudo apt-get install -y \ + ca-certificates \ + curl \ + gnupg \ + lsb-release \ + linux-modules-extra-raspi + +# auto load openvswitch kernel module at boot +echo openvswitch | sudo tee -a /etc/modules > /dev/null + +# disable openvsiwtch on host if it is installed +sudo systemctl disable openvswitch-switch + +# this file causes a conflict so remove if exists +sudo rm /etc/apt/sources.list.d/download_docker_com_linux_ubuntu.list + +# Install docker +sudo apt-get remove docker docker-engine docker.io containerd runc +sudo apt-get remove containerd.io docker-ce-cli docker-ce-rootless-extras docker-ce +sudo mkdir -m 0755 -p /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +sudo chmod a+r /etc/apt/keyrings/docker.gpg && sudo apt-get update && \ +sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +# Add the user to docker group +me=$USER && sudo usermod -aG docker $me + +sudo reboot +# pull latest edgevpn image +docker pull edgevpnio/evio-node:latest + +mkdir /home/$USER/.evio +cp /etc/opt/evio/config.json /home/$USER/.evio/config-$NODE_IDX.json +docker run -d -v /home/$USER/.evio/config-097.json:/etc/opt/evio/config.json -v /var/log/evio/:/var/log/evio/ --restart always --privileged --name evio-node --network host edgevpnio/evio-node:latest \ No newline at end of file diff --git a/scripts/evio-control.py b/scripts/evio-control.py index dd93c95..472210f 100755 --- a/scripts/evio-control.py +++ b/scripts/evio-control.py @@ -14,7 +14,7 @@ except ImportError: config_file="/etc/opt/evio/config.json" nid_file="/var/opt/evio/nid" log_file="/var/log/evio/evio-control.log" -server_addr="x.x.x.x:5802" +server_addr=None logger=None def runcmd(cmd): @@ -46,10 +46,19 @@ def node_id(): with open(nid_file) as f: return f.readline().strip('\n') - +def server_address(): + with open(config_file) as f: + config = json.load(f) + if "OverlayVisualizer" in config: + return config["OverlayVisualizer"].get("WebServiceAddress") + def main(): setup_logger() nid = node_id() + server_addr = server_address() + if server_addr is None: + logger.info("No web server address configured") + return url=f'http://{server_addr}/eviocontrol/?nodeid={nid}' resp=requests.get(url).json() logger.info(f"Server response: {resp}") diff --git a/scripts/evio-tools-v23.sh b/scripts/evio-tools-v23.sh new file mode 100755 index 0000000..0172410 --- /dev/null +++ b/scripts/evio-tools-v23.sh @@ -0,0 +1,365 @@ +#!/bin/bash +#v23 + +set -e + +EVIO_REPO=https://github.com/EdgeVPNio/evio.git +EXT_REPO=https://github.com/EdgeVPNio/external.git +TOOLS_REPO=https://github.com/EdgeVPNio/tools.git +PORTAL_REPO=https://github.com/EdgeVPNio/portal.git +PY=python3.9 +WorkspaceRoot=$(readlink -e ../..) + +EVIO_DIR=$WorkspaceRoot/EdgeVPNio/evio/evio +TINCAN_DIR=$WorkspaceRoot/EdgeVPNio/tincan +EXT_DIR=$WorkspaceRoot/EdgeVPNio/external +TOOLS_DIR=$WorkspaceRoot/EdgeVPNio/tools +PORTAL_DIR=$WorkspaceRoot/EdgeVPNio/portal +OUT_DIR=$WorkspaceRoot/out +BuildWRTC=$TOOLS_DIR/scripts/build_webrtc.sh +GetArchives=$TOOLS_DIR/scripts/get_archives.sh +GetInclude=$TOOLS_DIR/scripts/get_include.sh +BuildTincan=$TOOLS_DIR/scripts/build_tincan.sh +Versioning=$TOOLS_DIR/versioning/versioning.py +VER=0 +PLATFORM="" +BUILD_TYPE="" +ARCH="" +DATE="$(date '+%Y-%m-%d %T%z')" + +function display_env +{ + echo DATE=$DATE + echo VER=$VER + echo PLATFORM=$PLATFORM + echo BUILD_TYPE=$BUILD_TYPE + echo ARCH=$ARCH + echo WorkspaceRoot=$WorkspaceRoot + echo OUT_DIR=$OUT_DIR + echo EVIO_DIR=$EVIO_DIR + echo EXT_DIR=$EXT_DIR + echo TOOLS_DIR=$TOOLS_DIR +} + +function check_env +{ + + envlist="VER PLATFORM BUILD_TYPE ARCH" + + for env in $envlist + do + echo ${!env} + if [ -z ${!env} ]; then + echo "Environment ${env} is not set, exiting." + exit 1 + + fi + done +} + +function update_repo +{ + rv=$(git -C $1 rev-parse --is-inside-work-tree) + if [ "$rv" == "true" -o "$rv" == "false" ]; then + git -C "$(dirname $1)" pull "$2" + else + git -C "$(dirname $1)" clone "$2" + fi +} + +function sync_repos +{ + update_repo $EVIO_DIR $EVIO_REPO + update_repo $EXT_DIR $EXT_REPO + update_repo $TOOLS_DIR $TOOLS_REPO + update_repo $PORTAL_DIR $PORTAL_REPO + +} + +function copy_evio_files +{ + echo target dir= "$1" + cp -r $EVIO_DIR/evio_controller.py "$1"/ && \ + cp $EVIO_DIR/template-config.json "$1"/config.json && \ + cp -r $EVIO_DIR/broker/ $EVIO_DIR/controllers/ "$1"/ && \ + chmod 0775 "$1"/broker/ && \ + chmod 0664 "$1"/broker/* && \ + chmod 0775 "$1"/controllers/ && \ + chmod 0664 "$1"/controllers/* && \ + chmod 0664 "$1"/evio_controller.py && \ + chmod 0664 "$1"/config.json && \ + cp $TINCAN_DIR/out/"$PLATFORM"/"$BUILD_TYPE"/tincan "$1"/ && \ + chmod 0775 "$1"/tincan +} + +function make_debpak +{ + display_env + check_env + + sudo rm -rf $EVIO_DIR/controllers/__pycache__/ $EVIO_DIR/broker/__pycache__/ + wd=$(pwd) + mkdir -p $TOOLS_DIR/debian-package/evio/etc/opt/evio + # copy evio and tincan files to debpak directory + cp -r $EVIO_DIR/evio_controller.py $TOOLS_DIR/debian-package/evio/opt/evio/ && \ + cp $EVIO_DIR/template-config.json $TOOLS_DIR/debian-package/evio/etc/opt/evio/config.json && \ + cp -r $EVIO_DIR/controllers/ $EVIO_DIR/broker/ $TOOLS_DIR/debian-package/evio/opt/evio/ && \ + chmod 0775 $TOOLS_DIR/debian-package/evio/opt/evio/broker/ && \ + chmod 0664 $TOOLS_DIR/debian-package/evio/opt/evio/broker/* && \ + chmod 0775 $TOOLS_DIR/debian-package/evio/opt/evio/controllers/ && \ + chmod 0664 $TOOLS_DIR/debian-package/evio/opt/evio/controllers/* && \ + chmod 0664 $TOOLS_DIR/debian-package/evio/opt/evio/evio_controller.py && \ + chmod 0664 $TOOLS_DIR/debian-package/evio/etc/opt/evio/config.json && \ + cp $TINCAN_DIR/out/"$PLATFORM"/"$BUILD_TYPE"/tincan $TOOLS_DIR/debian-package/evio/opt/evio/ && \ + chmod 0775 $TOOLS_DIR/debian-package/evio/opt/evio/tincan + if [ $? -ne 0 ]; then + exit 1 + fi + mkdir -p $OUT_DIR + # invoke deb-gen to create debian installer package + cd $TOOLS_DIR/debian-package/ && \ + ./deb-gen $OUT_DIR $ARCH $VER $1 + if [ $? -ne 0 ]; then + exit 1 + fi + # remove previously copied files + sudo rm -rf evio/opt/evio/broker \ + evio/opt/evio/controllers \ + evio/opt/evio/tincan \ + evio/opt/evio/template-config.json \ + evio/opt/evio/evio_controller.py \ + evio/etc/opt/evio/config.json + cd $wd +} + +function install_testbed_deps +{ + sudo bash -c " + apt-get update -y && \ + apt-get install -y openvswitch-switch \ + $PY $PY-venv $PY-dev python3-pip \ + apt-transport-https \ + ca-certificates \ + curl git \ + software-properties-common && \ + + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + add-apt-repository \"deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable\" && \ + apt-cache policy docker-ce && \ + apt-get install -y containerd.io \ + docker-ce-cli \ + docker-ce && \ + groupadd -f docker && \ + usermod -a -G docker $USER && \ + newgrp docker \ + " + exit + groups | grep docker + echo "If docker is not listed above, you must logout and relogin for docker group membership to take effect." +} + +function make_dkrimg +{ + display_env + check_env + DPK_NAME=evio_$VER"_"$ARCH.deb + echo Building evio node docker image using $DPK_NAME + cp $OUT_DIR/$DPK_NAME $TOOLS_DIR/docker-image/evio"_"$ARCH.deb && \ + docker build --build-arg TARGETARCH="$ARCH" -f $TOOLS_DIR/docker-image/evio-node.Dockerfile -t edgevpnio/evio-node:"$VER" $TOOLS_DIR/docker-image + rm $TOOLS_DIR/docker-image/evio"_"$ARCH.deb +} + +function buildx_dkrimg +{ + display_env + DPK_NAME_AMD=evio_$VER"_"amd64.deb + DPK_NAME_ARM=evio_$VER"_"arm64.deb + cp $OUT_DIR/$DPK_NAME_AMD $TOOLS_DIR/docker-image/evio"_"amd64.deb && \ + cp $OUT_DIR/$DPK_NAME_ARM $TOOLS_DIR/docker-image/evio"_"arm64.deb && \ + docker buildx use evibuilder && \ + docker buildx build --push \ + --build-arg VERSION="$VER" \ + --build-arg DATE="$DATE" \ + --platform linux/amd64,linux/arm64 \ + -f $TOOLS_DIR/docker-image/evio-node.Dockerfile \ + -t edgevpnio/evio-node:$VER \ + -t edgevpnio/evio-node:latest \ + $TOOLS_DIR/docker-image + rm $TOOLS_DIR/docker-image/evio"_"amd64.deb $TOOLS_DIR/docker-image/evio"_"arm64.deb +} + +# function do_clean +# { +# #TODO(ken): Unit test +# # tincan +# wd=$(pwd) + +# echo rm -ri $TINCAN_DIR/out +# echo rm -ri $OUT_DIR +# #debian pak +# cd $TOOLS_DIR/debian-package/ && \ +# rm -ri ./*.deb \ +# evio/opt/evio/broker \ +# evio/opt/evio/controllers \ +# evio/opt/evio/tincan \ +# evio/opt/evio/template-config.json \ +# evio/opt/evio/evio_controller.py +# # docker-image +# cd .. +# rm -f docker-image/*.deb +# docker rmi edgevpnio/evio-node:"$VER" +# docker rmi $(docker images -q --filter "dangling=true") +# # testbed +# cd $TOOLS_DIR/testbed +# rm -rf config log cert venv +# cd $wd +# } + +function build_webrtc() +{ + display_env + check_env + wd=$(pwd) + cd $WorkspaceRoot + chmod +x $BuildWRTC $GetArchives $GetInclude + $BuildWRTC -t $PLATFORM -b $BUILD_TYPE + if [ $? -eq 0 ]; then + cd $WorkspaceRoot + $GetArchives -t $PLATFORM -b $BUILD_TYPE + if [ $? -eq 0 ]; then + cd $WorkspaceRoot + $GetInclude + fi + fi + cd $wd +} + +function build_tincan { + display_env + check_env + cd $WorkspaceRoot + chmod +x $BuildTincan + git -C "$EXT_DIR" checkout $PLATFORM + $BuildTincan -t $PLATFORM -b $BUILD_TYPE + if [ $? -ne 0 ]; then + exit 1 + fi +} + +function do_build +{ + build_tincan && \ + make_debpak && \ + make_dkrimg +} + +function do_build_debian_x64_release +{ + $PY $Versioning --workspace_root=$WorkspaceRoot --gen_version_files + VER=$($PY $Versioning --version) + PLATFORM="debian-x64" + BUILD_TYPE="release" + ARCH="amd64" + do_build +} + +function do_build_debian_arm64_release +{ + $PY $Versioning --workspace_root=$WorkspaceRoot --gen_version_files + VER=$($PY $Versioning --version) + PLATFORM="debian-arm64" + BUILD_TYPE="release" + ARCH="arm64" + do_build +} + +function do_build_debian_arm_release +{ + $PY $Versioning --workspace_root=$WorkspaceRoot --gen_version_files + VER=$($PY $Versioning --version) + PLATFORM="debian-arm" + BUILD_TYPE="release" + ARCH="armhf" + do_build +} + +source venv/bin/activate +case $1 in + sync) + sync_repos + ;; + setup_testbed) + install_testbed_deps + ;; + bldnum) + if [ -z "$2" ]; then + cat /var/tmp/evio_build_number + echo "" + elif [ "$2" == "set" ]; then + if [ ! -z "$3" ]; then + echo $2 > /var/tmp/evio_build_number + else + echo "No value for build number specified" + fi + elif [ "$2" == "next" ]; then + $PY $Versioning --next_build_num + $PY $Versioning --workspace_root=$WorkspaceRoot --gen_version_files -verbose + fi + ;; + build) + VER=$($PY $Versioning --version) + $PY $Versioning --workspace_root=$WorkspaceRoot --gen_version_files -verbose + if [ "$3" == "amd64" ]; then + PLATFORM="debian-x64" + ARCH="amd64" + elif [ "$3" == "arm64" ]; then + PLATFORM="debian-arm64" + ARCH="arm64" + elif [ "$3" == "arm" ]; then + PLATFORM="debian-arm" + ARCH="armhf" + fi + if [ "$4" == "rel" ]; then + BUILD_TYPE="release" + elif [ "$4" == "dbg" ]; then + BUILD_TYPE="debug" + fi + if [ "$2" == "debpak" ]; then + make_debpak $5 + elif [ "$2" == "dkrimg" ]; then + make_dkrimg + elif [ "$2" == "webrtc" ]; then + build_webrtc + elif [ "$2" == "tincan" ]; then + build_tincan + fi + ;; + buildx) + VER=$($PY $Versioning --version) + $PY $Versioning --workspace_root=$WorkspaceRoot --gen_version_files + BUILD_TYPE="release" + buildx_dkrimg + ;; + build_debx64_rel) + do_build_debian_x64_release + ;; + build_debarm64_rel) + do_build_debian_arm64_release + ;; + build_debarm_rel) + do_build_debian_arm_release + ;; + fbr) + $PY $Versioning --next_build_num + $PY $Versioning --workspace_root=$WorkspaceRoot --gen_version_files -verbose + VER=$($PY $Versioning --version) + PLATFORM="debian-x64" + ARCH="amd64" + BUILD_TYPE="release" + make_debpak + sudo apt install -y "$OUT_DIR"/evio_"$VER"_"$ARCH".deb + ;; + *) + echo "no match on input -> $1" + ;; +esac diff --git a/scripts/overlay-test.sh b/scripts/overlay-test.sh deleted file mode 100755 index a45d8d3..0000000 --- a/scripts/overlay-test.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -pingresult=/var/log/evio/ping-result.log -pingstat=/var/log/evio/ping-stat.log -pingfails=/var/log/evio/ping-failures.log -timestamp=$(date +"%D %T %Z %z") - -function icmp_test -{ - ping -c1 $2 > /dev/null - #iperf -c $1 -e -yC -n3G | tee -a /var/log/evio/$1-iperf-results.log - echo -e "\n############## $HOSTNAME-->>$1 - $timestamp ##############\n" 2>&1 | tee -a $pingresult - ping -c25 -n $2 | xargs -n1 -i bash -c 'echo `date +"%Y-%m-%d %H:%M:%S"`" {}"' 2>&1 | tee -a $pingresult - echo -e "\n############## $HOSTNAME-->>$1 - $timestamp ##############\n" 2>&1 | tee -a $pingstat - tail -n 2 $pingresult 2>&1 | tee -a $pingstat -} - -function ping_hosts -{ - while read ips - do - icmp_test $ips - if [ "$?" -ne "0" ]; then - echo "$(date +"%D %T %Z %z"): ICMP Failure to host $ips" | tee -a $pingfails - fi - done < $1 -} - -function run_all_tests -{ - ping_hosts $1 -} - -case $1 in - run_all) - run_all_tests $2 - ;; - ph) - ping_hosts $2 - ;; - icmp) - icmp_test $2 $3 - ;; - *) - echo "no match on input -> $1" - ;; -esac - diff --git a/scripts/ping-hosts b/scripts/ping-hosts deleted file mode 100644 index 667821c..0000000 --- a/scripts/ping-hosts +++ /dev/null @@ -1,5 +0,0 @@ -nd-40 10.1.1.40 -nd-41 10.1.1.41 -nd-42 10.1.1.42 -nd-60 10.1.1.43 -nd-60 10.1.1.44 diff --git a/scripts/update-limits.sh b/scripts/update-limits.sh old mode 100755 new mode 100644 diff --git a/setup-evt.sh b/setup-evt.sh index 6cea252..1320456 100755 --- a/setup-evt.sh +++ b/setup-evt.sh @@ -1,8 +1,11 @@ #!/bin/bash -sudo apt install -y python3.8 python3.8-dev python3.8-venv python3-pip git +sudo apt install -y software-properties-common +sudo add-apt-repository ppa:deadsnakes/ppa +sudo apt update +sudo apt install -y python3.9 python3.9-dev python3.9-venv python3-pip git #sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.8 38 deactivate 2>/dev/null -python3 -m venv venv && \ +python3.9 -m venv venv && \ source venv/bin/activate && \ chmod 775 ./evt && \ pip3 install gitpython simplejson diff --git a/testbed/template-config.json b/testbed/template-config.json deleted file mode 100644 index 852b3b7..0000000 --- a/testbed/template-config.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "CFx": { - "Model": "Default", - "InjectFaults": {"Topology": 3}, - "Overlays": [ - "C0CA0" - ], - "NodeId": "c0ca###ffffffffffffffffffc0ca###" - }, - "Logger": {}, - "Signal": { - "Overlays": { - "C0CA0": { - "HostAddress": "", - "AuthenticationMethod": "PASSWORD", - "Port": "5222", - "Username": "", - "Password": "", - "_AuthenticationMethod": "x509", - "_Port": "5223", - "_CertDirectory": "/var/evio/cert", - "_CertFile": "evpn.crt", - "_KeyFile": "evpn.key" - } - } - }, - "Topology": { - "Dependencies": [ - "Logger", - "TincanInterface", - "LinkManager", - "OverlayVisualizer" - ], - "Overlays": { - "C0CA0": {} - } - }, - "LinkManager": { - "Dependencies": [ - "Logger", - "TincanInterface", - "Signal", - "OverlayVisualizer" - ], - "Stun": [ - "stun.l.google.com:19302", - "stun1.l.google.com:19302" - ], - "Overlays": { - "C0CA0": { - "IgnoredNetInterfaces": ["flannel.1", "cni0", "docker0", "ovs-system"] - } - } - }, - "BridgeController": { - "BoundedFlood": { - "Overlays": { - "C0CA0": {} - } - }, - "Overlays": { - "C0CA0": { - "NetDevice": { - "AppBridge": { - "NetworkAddress": "10.1.1.0/24" - } - } - } - } - }, - "UsageReport": { - "Enabled": true, - "TimerInterval": 3600, - "WebService": "https://qdscz6pg37.execute-api.us-west-2.amazonaws.com/default/EvioUsageReport" - }, - "OverlayVisualizer": { - "Enabled": true, - "Dependencies": [ - "Logger" - ], - "TimerInterval": 30, - "WebServiceAddress": "", - "NodeName": "nd-###", - "GeoCoordinate": "0,0" - } -} \ No newline at end of file diff --git a/testbed/testbed.py b/testbed/testbed.py deleted file mode 100755 index 952bdd5..0000000 --- a/testbed/testbed.py +++ /dev/null @@ -1,555 +0,0 @@ -#!/usr/bin/env python3 -# pylint: disable=missing-docstring -try: - import simplejson as json -except ImportError: - import json -import sys -import os -import subprocess -import random -from distutils import spawn -import pickle -import argparse -import shutil -import time -from abc import ABCMeta, abstractmethod -import ipaddress -PACKAGE_PARENT = '..' -SCRIPT_DIR = os.path.dirname(os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__)))) -sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, PACKAGE_PARENT))) - -from versioning.tool_config import MAJOR_VER -from versioning.tool_config import MINOR_VER -from versioning.tool_config import REVISION_VER -from versioning.tool_config import CONTROL_VER -from versioning.tool_config import OFFICIAL - -class Testbed(): - __metaclass__ = ABCMeta - - LAUNCH_WAIT = 60 - BATCH_SZ = 8 - VIRT = NotImplemented - APT = spawn.find_executable("apt-get") - CONTAINER = NotImplemented - BF_VIRT_IMG = "edgevpnio/evio-node:21.9.0" - - def __init__(self, exp_dir=None): - parser = argparse.ArgumentParser( - description="Configures and runs EdgeVPN Testbed", - fromfile_prefix_chars='@') - parser.add_argument("--clean", action="store_true", default=False, dest="clean", - help="Removes all generated files and directories") - - parser.add_argument("--configure", action="store", dest="configure", type=int, - help="Generates the config files and directories") - - parser.add_argument("--slice", action="store", dest="slice", default=[], nargs=2, - help="Specifies the portion of the range to use. Given in format slice=#,#") - - parser.add_argument("-r", "--run", action="store_true", default=False, dest="run", - help="Runs the currently configured testbed") - - parser.add_argument("-e", "--end", action="store_true", default=False, dest="end", - help="End the currently running testbed") - - parser.add_argument("-i", "--info", action="store_true", default=False, dest="info", - help="Displays the current testbed configuration") - - parser.add_argument("--ping", action="store", dest="ping", - help="Ping the specified address from each container") - - parser.add_argument("--arp", action="store", dest="arp", - help="arPing the specified address from each container") - - parser.add_argument("--evio", action="store", dest="evio", - help="Perform the specified service action: stop/start/restart") - - parser.add_argument("--churn", action="store", dest="churn", - help="Restarts the specified amount of nodes in the overlay," - "one every interval") - - parser.add_argument("-test", action="store_true", default=False, dest="test", - help="Performs latency and bandwidth test between random pairs of " - "nodes. Ex test=") - - parser.add_argument("-v", "--verbose", action="store_true", default=False, dest="verbose", - help="Displays extended testbed activity") - - self.args = parser.parse_args() - self.exp_dir = exp_dir - if not self.exp_dir: - self.exp_dir = os.path.abspath(".") - self.bld_num_file = "/var/tmp/evio_build_number" - self.load_build_ver_info() - Testbed.BF_VIRT_IMG = "edgevpnio/evio-node:{0}.{1}.{2}".format( - MAJOR_VER, MINOR_VER, REVISION_VER) - if not OFFICIAL: - Testbed.BF_VIRT_IMG += ".{0}-{1}".format(self._bld_num, "dev") - self.template_file = "{0}/template-config.json".format(self.exp_dir) - self.config_dir = "{0}/config".format(self.exp_dir) - self.log_dir = "{0}/log".format(self.exp_dir) - self.data_dir = "{0}/data".format(self.exp_dir) - self.cert_dir = "{0}/cert".format(self.exp_dir) - self.config_file_base = "{0}/config-".format(self.config_dir) - self.seq_file = "{0}/startup.list".format(self.exp_dir) - self.range_file = "{0}/range_file".format(self.exp_dir) - - self.range_end = 2 - self.range_start = 1 - if os.path.isfile("range_file"): - with open(self.range_file, "rb") as rng_fle: - self.range_end = pickle.load(rng_fle) + 1 - - self.seq_list : list - self.load_seq_list() - - self.slice_end = len(self.seq_list) - self.slice_start = int(0) - if self.args.slice: - self.slice_end = int(self.args.slice[1]) - self.slice_start = int(self.args.slice[0]) - - self.total_inst = self.slice_end - self.slice_start - - @classmethod - def runshell(cls, cmd, resultfile=None): - """ Run a shell command. if fails, raise an exception. """ - if cmd[0] is None: - raise ValueError("No executable specified to run") - if resultfile is None: - resp = subprocess.run(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - else: - with open(resultfile, "w") as outfile: - resp = subprocess.run(cmd, stdout=outfile) - return resp - - @property - @abstractmethod - def gen_config(self, range_start, range_end): - pass - - @property - @abstractmethod - def start_instance(self, instance): - pass - - @property - @abstractmethod - def end(self): - pass - - def load_build_ver_info(self): - with open(self.bld_num_file, "r") as bn_fle: - self._bld_num = int(bn_fle.read()) - - def clean_config(self): - if os.path.isdir(self.config_dir): - shutil.rmtree(self.config_dir) - if self.args.verbose: - print("Removed dir {}".format(self.config_dir)) - if os.path.isfile(self.seq_file): - os.remove(self.seq_file) - if self.args.verbose: - print("Removed file {}".format(self.seq_file)) - if os.path.isfile(self.range_file): - os.remove(self.range_file) - if self.args.verbose: - print("Removed file {}".format(self.range_file)) - - def make_clean(self): - self.clean_config() - if os.path.isdir(self.log_dir): - shutil.rmtree(self.log_dir) - if self.args.verbose: - print("Removed dir {}".format(self.log_dir)) - - def configure(self): - self.range_end = self.args.configure + 1 - with open(self.range_file, "wb") as rng_fle: - pickle.dump(self.args.configure, rng_fle) - rng_fle.flush() - self.gen_config(self.range_start, self.range_end) - self.gen_seq_list(self.range_start, self.range_end) - self.total_inst = self.range_end - self.range_start - self.save_seq_list() - - def gen_seq_list(self, start, end): - self.seq_list = list(range(start, end)) - random.shuffle(self.seq_list) - - def save_seq_list(self): - with open(self.seq_file, "wb") as seq_fle: - pickle.dump(self.seq_list, seq_fle) - seq_fle.flush() - if self.args.verbose: - print("Instance sequence saved with {0} entries\n{1}" - .format(self.total_inst, self.seq_list)) - - def load_seq_list(self): - if os.path.isfile(self.seq_file): - with open(self.seq_file, "rb") as seq_fle: - self.seq_list = pickle.load(seq_fle) - if self.args.verbose: - print("Sequence list loaded from existing file - {0} entries\n{1}". - format(len(self.seq_list), self.seq_list)) - else: - self.gen_seq_list(self.range_start, self.range_end) - - def start_slice(self, num, wait): - cnt = 0 - sequence = self.seq_list[self.slice_start:self.slice_end] - for inst in sequence: - self.start_instance(inst) - cnt += 1 - if cnt % num == 0 and cnt < len(sequence): - # if self.args.verbose: - print("{0}/{1} container(s) instantiated".format(cnt, len(sequence))) - time.sleep(wait) - print("{0} container(s) instantiated".format(cnt)) - - def run(self): - self.start_slice(Testbed.BATCH_SZ, Testbed.LAUNCH_WAIT) - - def display_current_config(self): - print("----Testbed Configuration----") - print("Major:{0}, Minor:{1}, Revision:{2}, Build:{3}, Control:{4}, Official:{5}" - .format(MAJOR_VER, MINOR_VER, REVISION_VER, self._bld_num, CONTROL_VER, OFFICIAL)) - print("{0} instances, Range [{1}-{2}), Slice [{3}-{4})".format(self.total_inst, - self.range_start, - self.range_end, - self.slice_start, - self.slice_end)) - print("Config dir {0}".format(self.config_dir)) - print("Config base filename {0}".format(self.config_file_base)) - print("Log dir {0}".format(self.log_dir)) - print("Contianer image {0}".format(Testbed.BF_VIRT_IMG)) - print("Sequence List: {}".format(self.seq_list)) - - # def setup_system(self): - # setup_cmds = [["./setup-system.sh"]] - # for cmd_list in setup_cmds: - # if self.args.verbose: - # print(cmd_list) - # resp = Testbed.runshell(cmd_list) - # print(resp.stdout.decode("utf-8") if resp.returncode == 0 else - # resp.stderr.decode("utf-8")) - - @abstractmethod - def run_container_cmd(self, cmd_line, instance_num): - pass - - def churn(self, param): - params = param.rsplit(",", 2) - iters = int(params[0]) - inval = int(params[1]) - self._churn(iters, inval) - - def _churn(self, churn_count=0, interval=30): - if churn_count == 0: - churn_count = self.total_inst - cnt = 0 - restarted_nds = set() - while cnt < churn_count: - inst = random.choice(range(self.slice_start, self.slice_end)) - print("Stopping node", inst) - self.run_container_cmd(["systemctl", "stop", "evio"], inst) - if self.args.verbose: - print("Waiting", interval, "seconds") - time.sleep(interval) - print("Resuming node", inst) - self.run_container_cmd(["systemctl", "start", "evio"], inst) - restarted_nds.add(inst) - cnt += 1 - if self.args.verbose: - print("Waiting", interval, "seconds") - time.sleep(interval) - if self.args.verbose: - print("{0} nodes restarted\n{1}".format(cnt, str(restarted_nds))) - - def run_test(self): - # test = None - # if self.args.test == "lui": - # test = TestLinkUtilization() - # test.create_input_files() - # if self.args.test == "lur": - # test = TestLinkUtilization() - # test.create_result_report() - print("Test case not implemented") - - -class DockerTestbed(Testbed): - VIRT = spawn.find_executable("docker") - CONTAINER = "evio-dkr{0}" - - def __init__(self, exp_dir=None): - super().__init__(exp_dir=exp_dir) - self.network_name = "dkrnet" - - def create_network(self): - # netid=docker network ls | grep dkrnet | awk 'BEGIN { FS=" "} {print $2}' - # docker network create dkrnet - pass - - def gen_config(self, range_start, range_end): - with open(self.template_file) as cfg_tmpl: - template = json.load(cfg_tmpl) - olid = template["CFx"].get("Overlays", None) - olid = olid[0] - node_id = template["CFx"].get( - "NodeId", "a000###feb6040628e5fb7e70b04f###") - node_name = template["OverlayVisualizer"].get("NodeName", "dkr###") - netwk = template["BridgeController"]["Overlays"][olid]["NetDevice"]["AppBridge"].get( - "NetworkAddress", "10.10.1.0/24") - netwk = ipaddress.IPv4Network(netwk) - for val in range(range_start, range_end): - rng_str = "{0:03}".format(val) - cfg_file = "{0}{1}.json".format(self.config_file_base, rng_str) - node_id = "{0}{1}{2}{1}{3}".format( - node_id[:4], rng_str, node_id[7:29], node_id[32:]) - node_name = "{0}{1}".format(node_name[:3], rng_str) - node_ip = str(netwk[val]) - template["CFx"]["NodeId"] = node_id - template["OverlayVisualizer"]["NodeName"] = node_name - template["BridgeController"]["Overlays"][olid]["NetDevice"]["AppBridge"]["IP4"] = node_ip - template["BridgeController"]["Overlays"][olid]["NetDevice"]["AppBridge"]["PrefixLen"] = netwk.prefixlen - os.makedirs(self.config_dir, exist_ok=True) - with open(cfg_file, "w") as cfg_fle: - json.dump(template, cfg_fle, indent=2) - cfg_fle.flush() - if self.args.verbose: - print("{0} config file(s) generated".format(range_end-range_start)) - - def start_instance(self, instance): - instance = "{0:03}".format(instance) - container = DockerTestbed.CONTAINER.format(instance) - log_dir = "{0}/dkr{1}".format(self.log_dir, instance) - os.makedirs(log_dir, exist_ok=True) - - cfg_file = "{0}{1}.json".format(self.config_file_base, instance) - if not os.path.isfile(cfg_file): - self.gen_config(instance, instance+1) - - mount_cfg = "{0}:/etc/opt/evio/config.json".format(cfg_file) - mount_log = "{0}/:/var/log/evio/".format(log_dir) - #mount_data = "{0}/:/var/evio/".format(self.data_dir) - mount_cert = "{0}/:/var/evio/cert/".format(self.cert_dir) - args = ["--rm", "--privileged"] - opts = "-d" - img = Testbed.BF_VIRT_IMG - cmd = "/sbin/init" - cmd_list = [DockerTestbed.VIRT, "run", opts, "-v", mount_cfg, "-v", mount_log, "-v", mount_cert, - args[0], args[1], "--name", container, "--network", self.network_name, - img, cmd] - if self.args.verbose: - print(cmd_list) - resp = Testbed.runshell(cmd_list) - print(resp.stdout.decode("utf-8") if resp.returncode == - 0 else resp.stderr.decode("utf-8")) - - def run_container_cmd(self, cmd_line, instance_num): - cmd_list = [DockerTestbed.VIRT, "exec", "-it"] - inst = "{0:03}".format(instance_num) - container = DockerTestbed.CONTAINER.format(inst) - cmd_list.append(container) - cmd_list += cmd_line - resp = Testbed.runshell(cmd_list) - if self.args.verbose: - print(cmd_list) - print(resp.stdout.decode("utf-8")) - - - def run_cmd_on_slice(self, cmd_line, delay=0): - report = dict(fail_count=0, fail_node=[]) - for inst in self.seq_list[self.slice_start:self.slice_end]: - cmd_list = [DockerTestbed.VIRT, "exec", "-it"] - inst = "{0:03}".format(inst) - container = DockerTestbed.CONTAINER.format(inst) - cmd_list.append(container) - cmd_list += cmd_line - resp = Testbed.runshell(cmd_list) - if self.args.verbose: - print(cmd_list) - print(resp.stdout.decode("utf-8")) - if resp.returncode != 0: - report["fail_count"] += 1 - report["fail_node"].append("node-{0}".format(inst)) - if delay > 0: - time.sleep(delay) - rpt_msg = "{0}: {1}/{2} failed\n{3}".format(cmd_line, report["fail_count"], - self.slice_end - self.slice_start, - report["fail_node"]) - print(rpt_msg) - - def pull_image(self): - cmd_list = [DockerTestbed.VIRT, "pull", Testbed.BF_VIRT_IMG] - resp = Testbed.runshell(cmd_list) - if self.args.verbose: - print(resp) - - def stop_range(self): - cnt = 0 - cmd_list = [DockerTestbed.VIRT, "kill"] - sequence = self.seq_list[self.slice_start:self.slice_end] - for inst in sequence: - cnt += 1 - inst = "{0:03}".format(inst) - container = DockerTestbed.CONTAINER.format(inst) - cmd_list.append(container) - if self.args.verbose: - print(cmd_list) - resp = Testbed.runshell(cmd_list) - print(resp.stdout.decode("utf-8") if resp.returncode == 0 else - resp.stderr.decode("utf-8")) - print("{0} Docker container(s) terminated".format(cnt)) - - def end(self): - self.run_cmd_on_slice(["systemctl", "stop", "evio"]) - self.stop_range() - - def run_ping(self, target_address): - report = dict(fail_count=0, fail_node=[]) - for inst in range(self.range_start, self.range_end): - cmd_list = [DockerTestbed.VIRT, "exec", "-it"] - inst = "{0:03}".format(inst) - container = DockerTestbed.CONTAINER.format(inst) - cmd_list.append(container) - cmd_list += ["ping", "-c1"] - cmd_list.append(target_address) - resp = Testbed.runshell(cmd_list) - if self.args.verbose: - print(cmd_list) - print("ping ", target_address, "\n", - resp.stdout.decode("utf-8")) - if resp.returncode != 0: - report["fail_count"] += 1 - report["fail_node"].append("node-{0}".format(inst)) - rpt_msg = "ping {0}: {1}/{2} failed\n{3}".format(target_address, report["fail_count"], - self.range_end - self.range_start, - report["fail_node"]) - print(rpt_msg) - - def run_arp(self, target_address): - for inst in range(self.range_start, self.range_end): - cmd_list = [DockerTestbed.VIRT, "exec", "-it"] - inst = "{0:03}".format(inst) - container = DockerTestbed.CONTAINER.format(inst) - cmd_list.append(container) - cmd_list += ["arping", "-C1"] - cmd_list.append(target_address) - if self.args.verbose: - print(cmd_list) - resp = Testbed.runshell(cmd_list) - print(resp.stdout.decode("utf-8") if resp.returncode == 0 else - resp.stderr.decode("utf-8")) - - def run_svc_ctl(self, svc_ctl): - if svc_ctl == "stop": - self.run_cmd_on_slice(["systemctl", "stop", "evio"]) - elif svc_ctl == "start": - self.run_cmd_on_slice(["systemctl", "start", "evio"], 10) - elif svc_ctl == "restart": - self.run_cmd_on_slice(["systemctl", "restart", "evio"], 1) - else: - print("Invalid service control specified, only accepts start/stop/restart") - - def eval_join_interval(self, inst, target_address, outfile): - cmd_list = [DockerTestbed.VIRT, "exec", "-it"] - container = DockerTestbed.CONTAINER.format(inst) - cmd_list.append(container) - cmd_list += ["ping", "-c40", "-D", "-A"] - cmd_list.append(target_address) - if self.args.verbose: - print(cmd_list) - Testbed.runshell(cmd_list, outfile) - - - def stop_evio_svc(self, container): - cmd_list = [DockerTestbed.VIRT, "exec", "-it"] - cmd_list.append(container) - cmd_list += ["systemctl", "stop", "evio"] - if self.args.verbose: - print(cmd_list) - Testbed.runshell(cmd_list) - - def start_evio_svc(self, container): - cmd_list = [DockerTestbed.VIRT, "exec", "-it"] - cmd_list.append(container) - cmd_list += ["systemctl", "start", "evio"] - if self.args.verbose: - print(cmd_list) - Testbed.runshell(cmd_list) - - def run_test(self): - delay = 30 - netwk = ipaddress.IPv4Network("10.1.1.0/24") - #for inst in self.seq_list[self.slice_start:self.slice_end]: - inst=53 - instance = "{0:03}".format(inst) - container = DockerTestbed.CONTAINER.format(instance) - print(f"Stopping node {container} for {delay} seconds") - self.stop_evio_svc(container) - time.sleep(delay) - print(f"Starting node {container}") - self.start_evio_svc(container) - pingresult = "{0}/dkr{1}/ping-result.log".format(self.log_dir, instance) - self.eval_join_interval(instance, str(netwk[inst+1]), pingresult) - - - -def main(): # pylint: disable=too-many-return-statements - exp = DockerTestbed() - - if exp.args.run and exp.args.end: - print("Error! Both run and end were specified.") - return - - if exp.args.info: - exp.display_current_config() - return - - if exp.args.clean: - exp.make_clean() - return - - if exp.range_end - exp.range_start <= 0: - print("Invalid range, please fix RANGE_START={0} RANGE_END={1}". - format(exp.range_start, exp.range_end)) - return - - if exp.args.configure: - exp.configure() - - if exp.args.run: - exp.run() - return - - if exp.args.end: - exp.end() - return - - if exp.args.ping: - exp.run_ping(exp.args.ping) - return - - if exp.args.arp: - exp.run_arp(exp.args.arp) - return - - if exp.args.evio: - exp.run_svc_ctl(exp.args.evio) - return - - if exp.args.churn: - exp.churn(exp.args.churn) - return - - if exp.args.test: - exp.run_test() - return - - -if __name__ == "__main__": - main() diff --git a/versioning/Versioning.py b/versioning/Versioning.py deleted file mode 100644 index 2aad3b1..0000000 --- a/versioning/Versioning.py +++ /dev/null @@ -1,146 +0,0 @@ -import argparse -import os -import time -import subprocess -import fileinput -import sys -import shutil -from tool_config import CONTROL_VER as control -from tool_config import OFFICIAL as official -from tool_config import MAJOR_VER as major -from tool_config import MINOR_VER as minor -from tool_config import REVISION_VER as revision - -class Versioning(): - LICENSE = "{1}{0} EdgeVPNio\n{0} Copyright 2020, University of Florida\n{0}\n" \ - "{0} Permission is hereby granted, free of charge, to any person obtaining a copy\n" \ - "{0} of this software and associated documentation files (the \"Software\"), to deal\n" \ - "{0} in the Software without restriction, including without limitation the rights\n" \ - "{0} to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n" \ - "{0} copies of the Software, and to permit persons to whom the Software is\n" \ - "{0} furnished to do so, subject to the following conditions:\n" \ - "{0}\n" \ - "{0} The above copyright notice and this permission notice shall be included in\n" \ - "{0} all copies or substantial portions of the Software.\n" \ - "{0}\n" \ - "{0} THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n" \ - "{0} IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n" \ - "{0} FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n" \ - "{0} AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n" \ - "{0} LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n" \ - "{0} OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n" \ - "{0} THE SOFTWARE.\n" \ - "{2}" - tincan_version_template = LICENSE.format("*", "/*\n", "*/\n") + \ - "#ifndef TINCAN_VERSION_H_\n" \ - "#define TINCAN_VERSION_H_\n" \ - "namespace tincan\n" \ - "{{\n" \ - " static const uint16_t kTincanVerMjr = {0};\n" \ - " static const uint16_t kTincanVerMnr = {1};\n" \ - " static const uint16_t kTincanVerRev = {2};\n" \ - " static const uint16_t kTincanVerBld = {3};\n" \ - " static const uint8_t kTincanControlVer = {4};\n" \ - "}} // namespace tincan\n" \ - "#endif // TINCAN_VERSION_H_\n" - - controller_version_template = LICENSE.format("#", "", "\n") + \ - "EVIO_VER_MJR = {0}\n" \ - "EVIO_VER_MNR = {1}\n" \ - "EVIO_VER_REV = {2}\n" \ - "EVIO_VER_BLD = {3}\n" \ - "EVIO_VER_CTL = {4}\n" - - build_num_filename = "/var/tmp/evio_build_number" - - def __init__(self): - parser = argparse.ArgumentParser(description="Generates and manages version info and files for Evio build") - parser.add_argument("--workspace_root", action="store", dest="wrksproot", - help="Absolute pathname to the workspace directory.") - parser.add_argument("--version", action="store_true", default=False, dest="version", - help="Prints the current version to stdout.") - parser.add_argument("--build_num", action="store_true", default=False, dest="build_num", - help="Prints the current build number to stdout.") - parser.add_argument("--next_build_num", action="store_true", default=False, dest="next_build_num", - help="Increments and prints the build number to stdout.") - parser.add_argument("-q", action="store_true", default=False, dest="quiet", - help="Nothing is printed stdout.") - parser.add_argument("-verbose", action="store_true", default=False, dest="verbose", - help="Extra info printed to stdout.") - parser.add_argument("--gen_version_files", action="store_true", default=False, dest="gen_version_files", - help="Generates tincan_version.h and Version.py with appropriate values.") - - self.args = parser.parse_args() - if not self.args.wrksproot: - self.args.wrksproot = os.getcwd() - self._tincan_version_fqn = os.path.join(self.args.wrksproot, "EdgeVPNio/evio/tincan/trunk/include/tincan_version.h") - self._controller_version_fqn = os.path.join(self.args.wrksproot, "EdgeVPNio/evio/controller/framework/Version.py") - self._version = "{0}.{1}.{2}".format(major, minor, revision) - self._build_num = 0 - self._load_build_number() - if not official: - self._version += ".{}-dev".format(self._build_num) - - def generate_tincan_ver_header(self): - tincan_version_str = self.tincan_version_template.format(major, minor, revision, self._build_num, control) - if self.args.verbose: - print("Generating ", self._tincan_version_fqn) - with open(self._tincan_version_fqn, 'w') as tvfl: - tvfl.write(tincan_version_str) - - def generate_controller_version_file(self): - controller_version_str = self.controller_version_template.format(major, minor, revision, self._build_num, control) - if self.args.verbose: - print("Generating ", self._controller_version_fqn) - with open(self._controller_version_fqn, 'w') as cvfl: - cvfl.write(controller_version_str) - - @property - def build_number(self): - return self._build_num - - @property - def version_string(self): - return self._version - - @property - def next_build_number(self): - self._next_build_number() - return self._build_num - - def _load_build_number(self): - try: - with open(self.build_num_filename, 'r') as bnfl: - self._build_num = int(bnfl.readline()) - except FileNotFoundError: - self._build_num = 0 - self._next_build_number() - - def _next_build_number(self): - self._build_num = self._build_num + 1 if self._build_num < 65536 else 1 - with open(self.build_num_filename, 'w') as bnfl: - bnfl.write(str(self._build_num)) - -def main(): # pylint: disable=too-many-return-statements - app = Versioning() - - if app.args.version: - if not app.args.quiet: - print(app.version_string) - return - if app.args.build_num: - if not app.args.quiet: - print(app.build_number) - return - if app.args.next_build_num: - build_number = app.next_build_number - if not app.args.quiet: - print(build_number) - return - if app.args.gen_version_files: - app.generate_controller_version_file() - app.generate_tincan_ver_header() - return - -if __name__ == '__main__': - main() diff --git a/versioning/tool_config.py b/versioning/tool_config.py index e259721..4de24ae 100644 --- a/versioning/tool_config.py +++ b/versioning/tool_config.py @@ -1,4 +1,3 @@ - # MAJOR_VER = 20 # MINOR_VER = 11 # REVISION_VER = 0 @@ -11,8 +10,26 @@ # CONTROL_VER = 6 # OFFICIAL = True -MAJOR_VER = 21 -MINOR_VER = 6 -REVISION_VER = 0 +# MAJOR_VER = 21 +# MINOR_VER = 9 +# REVISION_VER = 0 +# CONTROL_VER = 6 +# OFFICIAL = True + +# MAJOR_VER = 23 +# MINOR_VER = 3 +# REVISION_VER = 1 +# CONTROL_VER = 6 +# OFFICIAL = True + +# MAJOR_VER = 23 +# MINOR_VER = 3 +# REVISION_VER = 2 +# CONTROL_VER = 6 +# OFFICIAL = True + +MAJOR_VER = 23 +MINOR_VER = 3 +REVISION_VER = 3 CONTROL_VER = 6 OFFICIAL = False diff --git a/versioning/versioning.py b/versioning/versioning.py new file mode 100644 index 0000000..0d9061a --- /dev/null +++ b/versioning/versioning.py @@ -0,0 +1,202 @@ +import argparse +import fileinput +import os +import shutil +import subprocess +import sys +import time + +from tool_config import CONTROL_VER as control +from tool_config import MAJOR_VER as major +from tool_config import MINOR_VER as minor +from tool_config import OFFICIAL as official +from tool_config import REVISION_VER as revision + + +class Versioning: + LICENSE = ( + "{1}{0} EdgeVPNio\n{0} Copyright 2020, University of Florida\n{0}\n" + "{0} Permission is hereby granted, free of charge, to any person obtaining a copy\n" + '{0} of this software and associated documentation files (the "Software"), to deal\n' + "{0} in the Software without restriction, including without limitation the rights\n" + "{0} to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n" + "{0} copies of the Software, and to permit persons to whom the Software is\n" + "{0} furnished to do so, subject to the following conditions:\n" + "{0}\n" + "{0} The above copyright notice and this permission notice shall be included in\n" + "{0} all copies or substantial portions of the Software.\n" + "{0}\n" + '{0} THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n' + "{0} IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n" + "{0} FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n" + "{0} AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n" + "{0} LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n" + "{0} OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n" + "{0} THE SOFTWARE.\n" + "{2}" + ) + tincan_version_template = ( + LICENSE.format("*", "/*\n", "*/\n") + "#ifndef TINCAN_VERSION_H_\n" + "#define TINCAN_VERSION_H_\n" + "namespace tincan\n" + "{{\n" + " static const uint16_t kTincanVerMjr = {0};\n" + " static const uint16_t kTincanVerMnr = {1};\n" + " static const uint16_t kTincanVerRev = {2};\n" + " static const uint16_t kTincanVerBld = {3};\n" + " static const uint8_t kTincanControlVer = {4};\n" + "}} // namespace tincan\n" + "#endif // TINCAN_VERSION_H_\n" + ) + + controller_version_template = ( + LICENSE.format("#", "", "\n") + "EVIO_VER_MJR = {0}\n" + "EVIO_VER_MNR = {1}\n" + "EVIO_VER_REV = {2}\n" + "EVIO_VER_BLD = {3}\n" + "EVIO_VER_CTL = {4}\n" + ) + + build_num_filename = "/var/tmp/evio_build_number" + + def __init__(self): + parser = argparse.ArgumentParser( + description="Generates and manages version info and files for Evio build" + ) + parser.add_argument( + "--workspace_root", + action="store", + dest="wrksproot", + help="Absolute pathname to the workspace directory.", + ) + parser.add_argument( + "--version", + action="store_true", + default=False, + dest="version", + help="Prints the current version to stdout.", + ) + parser.add_argument( + "--build_num", + action="store_true", + default=False, + dest="build_num", + help="Prints the current build number to stdout.", + ) + parser.add_argument( + "--next_build_num", + action="store_true", + default=False, + dest="next_build_num", + help="Increments and prints the build number to stdout.", + ) + parser.add_argument( + "-q", + action="store_true", + default=False, + dest="quiet", + help="Nothing is printed stdout.", + ) + parser.add_argument( + "-verbose", + action="store_true", + default=False, + dest="verbose", + help="Extra info printed to stdout.", + ) + parser.add_argument( + "--gen_version_files", + action="store_true", + default=False, + dest="gen_version_files", + help="Generates tincan_version.h and Version.py with appropriate values.", + ) + + self.args = parser.parse_args() + if not self.args.wrksproot: + self.args.wrksproot = os.getcwd() + self._tincan_path = os.path.join( + self.args.wrksproot, "EdgeVPNio/tincan/include/" + ) + self._tincan_version_fqn = os.path.join(self._tincan_path, "tincan_version.h") + self._controller_path = os.path.join( + self.args.wrksproot, "EdgeVPNio/evio/evio/broker/" + ) + self._controller_version_fqn = os.path.join(self._controller_path, "version.py") + self._version = "{0}.{1}.{2}".format(major, minor, revision) + self._build_num = 0 + self._load_build_number() + if not official: + self._version += ".{}-dev".format(self._build_num) + + def generate_tincan_ver_header(self): + tincan_version_str = self.tincan_version_template.format( + major, minor, revision, self._build_num, control + ) + if self.args.verbose: + print("Generating ", self._tincan_version_fqn) + os.makedirs(self._tincan_path, exist_ok=True) + with open(self._tincan_version_fqn, "w") as tvfl: + tvfl.write(tincan_version_str) + + def generate_controller_version_file(self): + controller_version_str = self.controller_version_template.format( + major, minor, revision, self._build_num, control + ) + if self.args.verbose: + print("Generating ", self._controller_version_fqn) + os.makedirs(self._controller_path, exist_ok=True) + with open(self._controller_version_fqn, "w") as cvfl: + cvfl.write(controller_version_str) + + @property + def build_number(self): + return self._build_num + + @property + def version_string(self): + return self._version + + @property + def next_build_number(self): + self._next_build_number() + return self._build_num + + def _load_build_number(self): + try: + with open(self.build_num_filename, "r") as bnfl: + self._build_num = int(bnfl.readline()) + except FileNotFoundError: + self._build_num = 0 + self._next_build_number() + + def _next_build_number(self): + self._build_num = self._build_num + 1 if self._build_num < 65536 else 1 + with open(self.build_num_filename, "w") as bnfl: + bnfl.write(str(self._build_num)) + + +def main(): # pylint: disable=too-many-return-statements + app = Versioning() + + if app.args.version: + if not app.args.quiet: + print(app.version_string) + return + if app.args.build_num: + if not app.args.quiet: + print(app.build_number) + return + if app.args.next_build_num: + build_number = app.next_build_number + if not app.args.quiet: + print(build_number) + return + if app.args.gen_version_files: + app.generate_controller_version_file() + app.generate_tincan_ver_header() + return + + +if __name__ == "__main__": + main()