add initial SFTP benchmark collection scripts

.github/workflows/sftp-benchmark.yml

@@ -0,0 +1,123 @@
+name: Benchmark Test
+
+on:
+  push:
+    branches: [ '*' ]
+  pull_request:
+    branches: [ '*' ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build_wolfssl:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest ]
+        wolfssl: [ master ]
+    name: Build wolfssl
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 4
+    steps:
+      - name: Checking cache for wolfssl
+        uses: actions/cache@v4
+        id: cache-wolfssl
+        with:
+          path: build-dir/
+          key: wolfssh-sshd-wolfssl-${{ matrix.wolfssl }}-${{ matrix.os }}
+          lookup-only: true
+
+      - name: Checkout, build, and install wolfssl
+        if: steps.cache-wolfssl.outputs.cache-hit != 'true'
+        uses: wolfSSL/actions-build-autotools-project@v1
+        with:
+          repository: wolfssl/wolfssl
+          ref: ${{ matrix.wolfssl }}
+          path: wolfssl
+          configure: --enable-all
+          check: false
+          install: true
+
+  build_wolfssh:
+    needs:
+      - build_wolfssl
+      - create_matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest ]
+        wolfssl: ${{ fromJson(needs.create_matrix.outputs['versions']) }}
+    name: Build and test wolfsshd
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    steps:
+      - name: Checking cache for wolfssl
+        uses: actions/cache@v4
+        with:
+          path: build-dir/
+          key: wolfssh-sshd-wolfssl-${{ matrix.wolfssl }}-${{ matrix.os }}
+          fail-on-cache-miss: true
+
+      - uses: actions/checkout@v4
+        with:
+          path: wolfssh/
+
+      - name: autogen
+        working-directory: ./wolfssh/
+        run: ./autogen.sh
+
+      - name: configure
+        working-directory: ./wolfssh/
+        run : |
+          ./configure --enable-all LDFLAGS="-L${{ github.workspace }}/build-dir/lib" CPPFLAGS="-I${{ github.workspace }}/build-dir/include -DWOLFSSH_NO_FPKI -DWOLFSSH_NO_SFTP_TIMEOUT -DWOLFSSH_MAX_SFTP_RW=4000000 -DMAX_PATH_SZ=120"
+
+      - name: make
+        working-directory: ./wolfssh/
+        run: make
+
+      - name: Get Saved OpenSSH Upload Results
+        uses: actions/downlad-artifact@v4
+        with:
+          path: ./wolfssh/
+          artifact_id: 'openssh-upload'
+
+
+      - name: Put test key in authorized keys file
+        run: |
+          touch ~/.ssh/authorized_keys_test
+          cat ./keys/hansel-*.pub > authorized_keys_test
+          chmod 600 ./keys/hansel-key-*.pem
+          sudo systemctl restart sshd
+          sudo service sshd restart
+
+      - name: Run SFTP client benchmark
+        working-directory: ./wolfssh/
+        run: |
+          ./scripts/get-sftp-benchmark.sh
+
+      - name: Store Upload Speed PNG
+        uses: actions/upload-artifact@v4
+        with:
+          name: upload-results.png
+          path: wolfssh/upload-results.png
+
+      - name: Store Download Speed PNG
+        uses: actions/upload-artifact@v4
+        with:
+          name: download-results.png
+          path: wolfssh/download-results.png
+
+      - name: Comment on PR about performance 
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          PR_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")
+          gh pr comment $PR_NUMBER --body "Attached is the performance results" \
+            --attach download-results.png \
+            --attach upload-results.png
+
+      artifacts:
+        'openssh-upload': { 'path': 'wolfssh/openssh-average-upload.csv' }
+        'openssh-download': { 'path': 'wolfssh/openssh-average-download.csv' }

examples/sftpclient/sftpclient.c

@@ -132,6 +132,8 @@ static void err_msg(const char* s)
 #else
     #include <sys/time.h>
 
+    double current_time_ms(int);
+
     /* return number of seconds*/
     word32 current_time(int reset)
     {
@@ -142,6 +144,18 @@ static void err_msg(const char* s)
         gettimeofday(&tv, 0);
         return (word32)tv.tv_sec;
     }
+
+    /* return number of micro seconds */
+    double current_time_ms(int reset)
+    {
+        struct timeval tv;
+
+        (void)reset;
+
+        gettimeofday(&tv, 0);
+        return (word64)(tv.tv_sec*1000000) + tv.tv_usec;
+    }
+
 #endif /* USE_WINDOWS_API */
 #endif /* !WOLFSSH_NO_TIMESTAMP */
 
@@ -1106,6 +1120,12 @@ static int doAutopilot(int cmd, char* local, char* remote)
     char fullpath[128] = ".";
     WS_SFTPNAME* name  = NULL;
     byte remoteAbsPath = 0;
+#if !defined(WOLFSSH_NO_TIMESTAMP) && !defined(USE_WINDOWS_API)
+    double currentTime;
+    double longBytes = 0;
+    FILE* f;
+#endif
+
 
     /* check if is absolute path before making it one */
     if (remote != NULL && WSTRLEN(remote) > 2 && remote[1] == ':' &&
@@ -1137,6 +1157,18 @@ static int doAutopilot(int cmd, char* local, char* remote)
             remote);
     }
 
+#if !defined(WOLFSSH_NO_TIMESTAMP) && !defined(USE_WINDOWS_API)
+    ret = WFOPEN(NULL, &f, fullpath, "rb");
+    if (ret != 0 || f == WBADFILE) return WS_BAD_FILE_E;
+    if (WFSEEK(NULL, f, 0, WSEEK_END) != 0) {
+        WFCLOSE(NULL, f);
+        return WS_BAD_FILE_E;
+    }
+    longBytes = (word32)WFTELL(NULL, f);
+    WREWIND(NULL, f);
+    currentTime = current_time_ms(0);
+#endif
+
     do {
         if (cmd == AUTOPILOT_PUT) {
             ret = wolfSSH_SFTP_Put(ssh, local, fullpath, 0, NULL);
@@ -1158,6 +1190,15 @@ static int doAutopilot(int cmd, char* local, char* remote)
                     fullpath, local);
         }
     }
+#if !defined(WOLFSSH_NO_TIMESTAMP) && !defined(USE_WINDOWS_API)
+    else {
+        currentTime = current_time_ms(0) - currentTime;
+        double result;
+        result = (double)longBytes / 1000000;
+        result = result / ((double)currentTime / 1000000);
+        printf("Transfered %s at %.2fMB/s\n", fullpath, result);
+    }
+#endif
 
     wolfSSH_SFTPNAME_list_free(name);
     return ret;

scripts/get-sftp-benchmark.sh

@@ -0,0 +1,173 @@
+#!/bin/bash
+
+KEY="keys/hansel-key-ecc.pem"
+TEST_FILE="/home/jak/Documents/wolfssh-fork/test"
+FILE_SIZES=("5000" "10000" "50000" "100000" "150000" "200000" "250000" "300000" "350000" "400000" "500000" "1000000")
+TRANSFER_MBS=""
+NUMBER_RUNS=10
+LOG_FILE="$PWD/log.csv"
+COMPARE_TO=""
+AVERAGE_FILE=""
+
+if [ -z $1 ]; then
+    echo "Assuming default server port of 22 (pass port number as first"
+    echo "argument if wanting to connect to a different port)"
+    PORT=22
+else
+    PORT="$1"
+fi
+
+do_openssh_put_test() {
+    cp $TEST_FILE $TEST_FILE-out
+    sftp_command="sftp -P$PORT -i $KEY jak@127.0.0.1"
+    output_file="sftp_log.txt"
+
+    # Start the script command to capture the sftp session
+script -qc "$sftp_command << EOF
+    put $TEST_FILE $TEST_FILE-out
+    bye
+EOF" /dev/null 2>&1 | tee $output_file | while read line; do
+    if [[ "$line" == *'MB/s'* ]]; then
+        #TRANSFER_MBS=$(echo "$line" | awk '{print $(NF-2)}' | sed 's/MB\/s//')
+        TRANSFER_MBS="$(echo "$line" | awk '{print $(NF-2)}' | sed 's/MB\/s//')"
+        printf " $TRANSFER_MBS" >> $LOG_FILE
+    fi
+    done
+}
+
+do_openssh_get_test() {
+    cp $TEST_FILE $TEST_FILE-out
+    sftp_command="sftp -P $PORT -i $KEY jak@127.0.0.1"
+    output_file="sftp_log.txt"
+
+    # Start the script command to capture the sftp session
+script -qc "$sftp_command << EOF
+    get $TEST_FILE $TEST_FILE-out
+    bye
+EOF" /dev/null 2>&1 | tee $output_file | while read line; do
+    if [[ "$line" == *'MB/s'* ]]; then
+        #TRANSFER_MBS=$(echo "$line" | awk '{print $(NF-2)}' | sed 's/MB\/s//')
+        TRANSFER_MBS="$(echo "$line" | awk '{print $(NF-2)}' | sed 's/MB\/s//')"
+        printf " $TRANSFER_MBS" >> $LOG_FILE
+    fi
+    done
+}
+
+do_wolfssh_put_test() {
+    cp $TEST_FILE $TEST_FILE-out
+    RESULT=$(./examples/sftpclient/wolfsftp -g -l $TEST_FILE -r $TEST_FILE-out -i $PWD/keys/hansel-key-ecc.der -j $PWD/keys/hansel-key-ecc.pub -u jak -p $PORT)
+    TRANSFER_MBS="$(echo "$RESULT" | awk '{print $(NF-0)}' | sed 's/MB\/s//')"
+    printf " $TRANSFER_MBS" >> $LOG_FILE
+}
+
+do_wolfssh_get_test() {
+    cp $TEST_FILE $TEST_FILE-out
+    RESULT=$(./examples/sftpclient/wolfsftp -G -l $TEST_FILE-out -r $TEST_FILE -i $PWD/keys/hansel-key-ecc.der -j $PWD/keys/hansel-key-ecc.pub -u jak -p $PORT)
+    TRANSFER_MBS="$(echo "$RESULT" | awk '{print $(NF-0)}' | sed 's/MB\/s//')"
+    printf " $TRANSFER_MBS" >> $LOG_FILE
+}
+
+# Create a log with averages
+do_create_average() {
+    awk -F', ' '{sum[$1]+=$2; count[$1]++} END {for (i in sum) print i, sum[i]/count[i]}' "$LOG_FILE" | sort -n > "$AVERAGE_FILE"
+    sed -i 's/ /, /' $AVERAGE_FILE
+}
+
+
+do_create_plot() {
+    gnuplot -e "set title '$TITLE';set ylabel 'MB/s';set xlabel 'File Size in Bytes';set grid; set format x \"%2.1t{/Symbol \264}10^{%L}\"; set term png;set output '$OUTPUT_FILE';plot '$LOG_FILE' using 1:2, '$AVERAGE_FILE' with lines lc rgb 'red' lw 2, '$COMPARE_TO' with lines lc rgb 'gold' lw 2"
+}
+
+echo "Starting tests"
+echo "Getting the average over $NUMBER_RUNS runs"
+
+# create openssh average if not found
+AVERAGE_FILE="$PWD/openssh-average-upload.csv"
+if [ ! -f "$AVERAGE_FILE" ]; then
+    echo "Collecting openssh average upload"
+    rm -f $LOG_FILE && touch $LOG_FILE
+    for run in $(seq 1 $NUMBER_RUNS); do
+        printf "Run $run: "
+        for i in "${FILE_SIZES[@]}"; do
+            tail -c "$i" /dev/urandom > "$TEST_FILE"
+            printf "$i," >> $LOG_FILE
+            do_openssh_put_test
+            printf "\n" >> $LOG_FILE
+        done
+        printf "done\n"
+    done
+
+    do_create_average
+    echo ""
+fi
+
+# create wolfssh average upload
+echo "Collecting wolfssh average upload"
+rm -f $LOG_FILE && touch $LOG_FILE
+for run in $(seq 1 $NUMBER_RUNS); do
+    printf "Run $run: "
+    for i in "${FILE_SIZES[@]}"; do
+        tail -c "$i" /dev/urandom > "$TEST_FILE"
+        printf "$i," >> $LOG_FILE
+        do_wolfssh_put_test
+        printf "\n" >> $LOG_FILE
+    done
+    printf "done\n"
+done
+
+# compile and plot the results of average upload
+AVERAGE_FILE="$PWD/wolfssh-average-upload.csv"
+do_create_average
+
+TITLE="SFTP Client Upload Speeds [$NUMBER_RUNS runs]"
+COMPARE_TO="$PWD/openssh-average-upload.csv"
+AVERAGE_FILE="$PWD/wolfssh-average-upload.csv"
+OUTPUT_FILE="$PWD/upload-results.png"
+do_create_plot
+
+# create openssh average download if not found
+AVERAGE_FILE="$PWD/openssh-average-download.csv"
+if [ ! -f "$AVERAGE_FILE" ]; then
+    echo "Collecting openssh average download"
+    rm -f $LOG_FILE && touch $LOG_FILE
+    for run in $(seq 1 $NUMBER_RUNS); do
+        printf "Run $run: "
+        for i in "${FILE_SIZES[@]}"; do
+            tail -c "$i" /dev/urandom > "$TEST_FILE"
+            printf "$i," >> $LOG_FILE
+            do_openssh_get_test
+            printf "\n" >> $LOG_FILE
+        done
+        printf "done\n"
+    done
+
+    do_create_average
+    echo ""
+fi
+
+# create wolfssh average download
+echo "Collecting wolfssh average download"
+rm -f $LOG_FILE && touch $LOG_FILE
+for run in $(seq 1 $NUMBER_RUNS); do
+    printf "Run $run: "
+    for i in "${FILE_SIZES[@]}"; do
+        tail -c "$i" /dev/urandom > "$TEST_FILE"
+        printf "$i," >> $LOG_FILE
+        do_wolfssh_get_test
+        printf "\n" >> $LOG_FILE
+    done
+    printf "done\n"
+done
+
+# compile and plot the results of average download speeds
+AVERAGE_FILE="$PWD/wolfssh-average-download.csv"
+do_create_average
+
+TITLE="SFTP Client Download Speeds [$NUMBER_RUNS runs]"
+COMPARE_TO="$PWD/openssh-average-download.csv"
+AVERAGE_FILE="$PWD/wolfssh-average-download.csv"
+OUTPUT_FILE="$PWD/download-results.png"
+do_create_plot
+
+rm -rf $TEST_FILE
+rm -rf $TEST_FILE-out