From 7f18dc6087abb114b736450d29c7f32bc3050ac7 Mon Sep 17 00:00:00 2001 From: Jack Tjaden Date: Fri, 11 Apr 2025 16:39:13 -0600 Subject: [PATCH] Added signature benchmarks, refactored tests solving provider conflicts --- examples/provider/CryptoBenchmark.java | 384 +++++++++++++++++++++---- 1 file changed, 332 insertions(+), 52 deletions(-) diff --git a/examples/provider/CryptoBenchmark.java b/examples/provider/CryptoBenchmark.java index cc83f45..0e126ef 100644 --- a/examples/provider/CryptoBenchmark.java +++ b/examples/provider/CryptoBenchmark.java @@ -21,6 +21,8 @@ import javax.crypto.spec.PBEKeySpec; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.security.NoSuchProviderException; +import java.security.Signature; +import sun.security.ec.SunEC; import java.util.*; import com.wolfssl.provider.jce.WolfCryptProvider; @@ -103,10 +105,95 @@ public class CryptoBenchmark { } } + /* List of signature algorithms supported by wolfJCE - must match exactly what's in WolfCryptProvider */ + private static final String[] SIGNATURE_ALGORITHMS = { + /* RSA algorithms */ + "MD5withRSA", + "SHA1withRSA", + "SHA224withRSA", + "SHA256withRSA", + "SHA384withRSA", + "SHA512withRSA", + "SHA3-224withRSA", + "SHA3-256withRSA", + "SHA3-384withRSA", + "SHA3-512withRSA", + + /* ECDSA algorithms */ + "SHA1withECDSA", + "SHA224withECDSA", + "SHA256withECDSA", + "SHA384withECDSA", + "SHA512withECDSA", + "SHA3-224withECDSA", + "SHA3-256withECDSA", + "SHA3-384withECDSA", + "SHA3-512withECDSA" + }; + + /* List of signature algorithms supported by SUN providers - based on screenshot */ + private static final String[] SUN_SIGNATURE_ALGORITHMS = { + /* RSA algorithms */ + "MD2withRSA", + "MD5withRSA", + "SHA1withRSA", + "SHA224withRSA", + "SHA256withRSA", + "SHA384withRSA", + "SHA512withRSA", + + /* ECDSA algorithms */ + "SHA1withECDSA", + "SHA224withECDSA", + "SHA256withECDSA", + "SHA384withECDSA", + "SHA512withECDSA", + }; + private static void printProviderInfo(Provider provider) { System.out.printf("%s version: %.1f%n", provider.getName(), provider.getVersion()); } + private static void setupProvidersForTest(Provider testProvider) { + /* Remove only our test providers */ + Security.removeProvider("wolfJCE"); + Security.removeProvider("BC"); + + /* Add test provider at priority 1 */ + Security.insertProviderAt(testProvider, 1); + + /* For SunJCE tests, ensure SunEC is available */ + if (testProvider.getName().equals("SunJCE") && Security.getProvider("SunEC") == null) { + try { + Provider sunEC = new sun.security.ec.SunEC(); + Security.addProvider(sunEC); + } catch (Exception e) { + System.out.println("Warning: SunEC provider not available: " + e.getMessage()); + } + } + } + + private static void setupDigestProvider(String testProviderName) { + /* For digest operations, we need special handling */ + if (testProviderName.equals("wolfJCE") || testProviderName.equals("BC")) { + // wolfJCE and BC can handle their own digests + return; + } else { + /* For SunJCE, we need SUN provider for MessageDigest */ + if (Security.getProvider("SUN") == null) { + try { + Provider sunProvider = Security.getProvider("SUN"); + if (sunProvider == null) { + /* SUN provider should be built-in, but let's be safe */ + System.out.println("SUN provider not found for MessageDigest operations"); + } + } catch (Exception e) { + System.out.println("Failed to set up SUN provider: " + e.getMessage()); + } + } + } + } + private static void printDeltaTable() { /* Variables for table generation */ Map> groupedResults; @@ -118,10 +205,10 @@ public class CryptoBenchmark { double deltaPercent; System.out.println("\nPerformance Delta (compared to wolfJCE)"); - System.out.println("--------------------------------------------------------------------------------"); - System.out.println("| Operation | Provider | Delta | Delta |"); - System.out.println("| | | Value* | (%) |"); - System.out.println("|------------------------------------------|--------------|----------|----------|"); + System.out.println("------------------------------------------------------------------------------------"); + System.out.println("| Operation | Provider | Delta | Delta |"); + System.out.println("| | | Value* | (%) |"); + System.out.println("|----------------------------------------------|--------------|----------|----------|"); /* Group results by operation */ groupedResults = new HashMap<>(); @@ -174,7 +261,7 @@ public class CryptoBenchmark { /* Ensure unique operation-provider combination */ String uniqueKey = operation + "|" + displayProvider; if (!groupedResults.containsKey(uniqueKey)) { - System.out.printf("| %-40s | %-12s | %+8.2f | %+8.1f |%n", + System.out.printf("| %-44s | %-12s | %+8.2f | %+8.1f |%n", operation.replace("RSA", "RSA/ECB/PKCS1Padding RSA"), displayProvider, deltaValue, @@ -186,7 +273,7 @@ public class CryptoBenchmark { } } } - System.out.println("--------------------------------------------------------------------------------"); + System.out.println("------------------------------------------------------------------------------------"); System.out.println("* Delta Value: MiB/s for symmetric ciphers, operations/second for RSA and ECC"); } @@ -422,6 +509,7 @@ public class CryptoBenchmark { /* Initialize key generator */ if (providerName.equals("SunJCE")) { keyGen = KeyPairGenerator.getInstance("EC", "SunEC"); + keyGen.initialize(new ECGenParameterSpec(curveName)); providerName = "SunEC"; } else { keyGen = KeyPairGenerator.getInstance("EC", providerName); @@ -650,6 +738,150 @@ public class CryptoBenchmark { results.add(new BenchmarkResult(providerName, algorithm, throughput)); } + /* Run signature benchmarks */ + private static void runSignatureBenchmark(String algorithm, String providerName) throws Exception { + KeyPairGenerator keyGen; + Signature signature; + byte[] testData; + int ops = 0; + long startTime; + double elapsedTime; + KeyPair keyPair; + + /* Generate small test data */ + testData = generateTestData(SMALL_MESSAGE_SIZE); + + /* Determine the correct provider and key type based on algorithm */ + String keyGenProvider = providerName; + String signatureProvider = providerName; + String keyType; + + if (algorithm.contains("withRSA")) { + keyType = "RSA"; + if (providerName.equals("SunJCE")) { + keyGenProvider = "SunRsaSign"; + signatureProvider = "SunRsaSign"; + } + } else if (algorithm.contains("withECDSA")) { + keyType = "EC"; + if (providerName.equals("SunJCE")) { + Provider sunECProvider = Security.getProvider("SunEC"); + if (sunECProvider == null) { + try { + Provider sunEC = new sun.security.ec.SunEC(); + Security.addProvider(sunEC); + } catch (Exception e) { + throw new Exception("SunEC provider not available: " + e.getMessage()); + } + } + keyGenProvider = "SunEC"; + signatureProvider = "SunEC"; + + } + } + else if (algorithm.contains("withDSA")) { + keyType = "DSA"; + if (providerName.equals("SunJCE")) { + keyGenProvider = "SUN"; + signatureProvider = "SUN"; + } + } else { + throw new IllegalArgumentException("Unsupported signature algorithm: " + algorithm); + } + + try { + /* Initialize key generator and signature with correct providers */ + if (keyType.equals("EC") && keyGenProvider.equals("SunEC")) { + keyGen = KeyPairGenerator.getInstance("EC", "SunEC"); + keyGen.initialize(new ECGenParameterSpec("secp256r1")); + } else { + keyGen = KeyPairGenerator.getInstance(keyType, keyGenProvider); + + /* Initialize key generator based on type */ + if (keyType.equals("RSA")) { + keyGen.initialize(2048); + } else if (keyType.equals("EC")) { + keyGen.initialize(new ECGenParameterSpec("secp256r1")); + } else if (keyType.equals("DSA")) { + keyGen.initialize(1024); + } + } + signature = Signature.getInstance(algorithm, signatureProvider); + + /* Initialize key generator based on type */ + if (keyType.equals("RSA")) { + keyGen.initialize(2048); + } else if (keyType.equals("EC")) { + if (keyGenProvider.equals("SunEC")) { + keyGen.initialize(new ECGenParameterSpec("secp256r1")); + } else { + keyGen.initialize(new ECGenParameterSpec("secp256r1")); + } + } else if (keyType.equals("DSA")) { + keyGen.initialize(1024); + } + + /* Generate key pair */ + keyPair = keyGen.generateKeyPair(); + + /* Test that signing works before benchmarking */ + signature.initSign(keyPair.getPrivate()); + signature.update(testData); + byte[] sig = signature.sign(); + + /* Warm up phase */ + for (int i = 0; i < WARMUP_ITERATIONS; i++) { + signature.initSign(keyPair.getPrivate()); + signature.update(testData); + signature.sign(); + + signature.initVerify(keyPair.getPublic()); + signature.update(testData); + signature.verify(sig); + } + + /* Benchmark signing */ + ops = 0; + startTime = System.nanoTime(); + elapsedTime = 0; + + do { + signature.initSign(keyPair.getPrivate()); + signature.update(testData); + signature.sign(); + ops++; + elapsedTime = (System.nanoTime() - startTime) / 1_000_000_000.0; + } while (elapsedTime < TEST_MIN_TIME_SECONDS); + + double signOpsPerSec = ops / elapsedTime; + System.out.printf(" %-40s %8d ops took %.3f sec, %8.3f ops/sec%n", + algorithm + " sign (" + signatureProvider + ")", ops, elapsedTime, signOpsPerSec); + results.add(new BenchmarkResult(signatureProvider, algorithm + " sign", signOpsPerSec)); + + /* Benchmark verification */ + ops = 0; + startTime = System.nanoTime(); + elapsedTime = 0; + + do { + signature.initVerify(keyPair.getPublic()); + signature.update(testData); + signature.verify(sig); + ops++; + elapsedTime = (System.nanoTime() - startTime) / 1_000_000_000.0; + } while (elapsedTime < TEST_MIN_TIME_SECONDS); + + double verifyOpsPerSec = ops / elapsedTime; + System.out.printf(" %-40s %8d ops took %.3f sec, %8.3f ops/sec%n", + algorithm + " verify (" + signatureProvider + ")", ops, elapsedTime, verifyOpsPerSec); + results.add(new BenchmarkResult(signatureProvider, algorithm + " verify", verifyOpsPerSec)); + + } catch (Exception e) { + System.out.printf(" %-40s Not supported: %s%n", + algorithm + " (" + signatureProvider + ")", e.getMessage()); + } + } + public static void main(String[] args) { try { /* Check if Bouncy Castle is available */ @@ -690,9 +922,9 @@ public class CryptoBenchmark { System.out.println(" Symmetric Cipher Benchmark"); System.out.println("-----------------------------------------------------------------------------\n"); - /* Run symmetric benchmarks */ + /* Run symmetric benchmarks with clean provider setup */ for (int i = 0; i < providers.length; i++) { - Security.insertProviderAt(providers[i], 1); + setupProvidersForTest(providers[i]); runEncDecBenchmark("AES", "CBC", "NoPadding", providerNames[i]); runEncDecBenchmark("AES", "CBC", "PKCS5Padding", providerNames[i]); @@ -703,20 +935,25 @@ public class CryptoBenchmark { } } - /* Run RSA benchmarks */ + /* Run RSA benchmarks with clean provider setup */ System.out.println("\n-----------------------------------------------------------------------------"); System.out.println("RSA Benchmark Results"); System.out.println("-----------------------------------------------------------------------------"); for (Provider provider : providers) { - Security.insertProviderAt(provider, 1); + setupProvidersForTest(provider); System.out.println("\n" + (provider.getName().equals("SunJCE") ? "SunJCE / SunRsaSign" : provider.getName()) + ":"); for (int keySize : RSA_KEY_SIZES) { - runRSABenchmark(provider.getName(), keySize); + try { + runRSABenchmark(provider.getName(), keySize); + } catch (Exception e) { + System.out.printf("Failed to benchmark RSA %d with provider %s: %s%n", + keySize, provider.getName(), e.getMessage()); + } } - Security.removeProvider(provider.getName()); } + /* Run ECC benchmarks with clean provider setup */ System.out.println("\n-----------------------------------------------------------------------------"); System.out.println("ECC Benchmark Results"); System.out.println("-----------------------------------------------------------------------------"); @@ -725,8 +962,8 @@ public class CryptoBenchmark { if (provider instanceof WolfCryptProvider && !FeatureDetect.EccKeyGenEnabled()) { continue; } - Security.insertProviderAt(provider, 1); - System.out.println("\n" + (provider.getName().equals("SunJCE") ? "SunJCE / SunEC" : provider.getName()) + ":"); + setupProvidersForTest(provider); + System.out.println("\n" + (provider.getName().equals("SunJCE") ? "SunJCE / SunEC" : provider.getName()) + ":"); for (String curve : ECC_CURVES) { try { runECCBenchmark(provider.getName(), curve); @@ -737,30 +974,57 @@ public class CryptoBenchmark { } } + /* Run HMAC benchmarks with clean provider setup */ System.out.println("\n-----------------------------------------------------------------------------"); System.out.println("HMAC Benchmark Results"); System.out.println("-----------------------------------------------------------------------------"); for (int i = 0; i < providers.length; i++) { - Security.insertProviderAt(providers[i], 1); + setupProvidersForTest(providers[i]); if (FeatureDetect.HmacMd5Enabled()) { - runHmacBenchmark("HmacMD5", providerNames[i]); + try { + runHmacBenchmark("HmacMD5", providerNames[i]); + } catch (Exception e) { + System.out.printf("Failed to benchmark HmacMD5 with provider %s: %s%n", + providerNames[i], e.getMessage()); + } } if (FeatureDetect.HmacShaEnabled()) { - runHmacBenchmark("HmacSHA1", providerNames[i]); + try { + runHmacBenchmark("HmacSHA1", providerNames[i]); + } catch (Exception e) { + System.out.printf("Failed to benchmark HmacSHA1 with provider %s: %s%n", + providerNames[i], e.getMessage()); + } } if (FeatureDetect.HmacSha256Enabled()) { - runHmacBenchmark("HmacSHA256", providerNames[i]); + try { + runHmacBenchmark("HmacSHA256", providerNames[i]); + } catch (Exception e) { + System.out.printf("Failed to benchmark HmacSHA256 with provider %s: %s%n", + providerNames[i], e.getMessage()); + } } if (FeatureDetect.HmacSha384Enabled()) { - runHmacBenchmark("HmacSHA384", providerNames[i]); + try { + runHmacBenchmark("HmacSHA384", providerNames[i]); + } catch (Exception e) { + System.out.printf("Failed to benchmark HmacSHA384 with provider %s: %s%n", + providerNames[i], e.getMessage()); + } } if (FeatureDetect.HmacSha512Enabled()) { - runHmacBenchmark("HmacSHA512", providerNames[i]); + try { + runHmacBenchmark("HmacSHA512", providerNames[i]); + } catch (Exception e) { + System.out.printf("Failed to benchmark HmacSHA512 with provider %s: %s%n", + providerNames[i], e.getMessage()); + } } } + /* Run DH benchmarks with clean provider setup */ System.out.println("\n-----------------------------------------------------------------------------"); System.out.println("DH Benchmark Results"); System.out.println("-----------------------------------------------------------------------------"); @@ -769,7 +1033,7 @@ public class CryptoBenchmark { if (provider instanceof WolfCryptProvider && !FeatureDetect.DhEnabled()) { continue; } - Security.insertProviderAt(provider, 1); + setupProvidersForTest(provider); System.out.println("\n" + provider.getName() + ":"); for (int keySize : DH_KEY_SIZES) { try { @@ -781,11 +1045,11 @@ public class CryptoBenchmark { } } + /* Run PBKDF2 benchmarks with clean provider setup */ System.out.println("\n-----------------------------------------------------------------------------"); System.out.println("PBKDF2 Benchmark Results"); System.out.println("-----------------------------------------------------------------------------"); - /* List of PBKDF2 algorithms to test */ String[] pbkdf2Algorithms = { "PBKDF2WithHmacSHA1", "PBKDF2WithHmacSHA224", @@ -798,54 +1062,43 @@ public class CryptoBenchmark { "PBKDF2WithHmacSHA3-512" }; - for (String providerName : providerNames) { - System.out.println("\n" + providerName + ":"); + for (int i = 0; i < providers.length; i++) { + setupProvidersForTest(providers[i]); + System.out.println("\n" + providerNames[i] + ":"); for (String algorithm : pbkdf2Algorithms) { try { /* Skip SHA3 algorithms for SunJCE */ - if (providerName.equals("SunJCE") && algorithm.contains("SHA3")) { + if (providerNames[i].equals("SunJCE") && algorithm.contains("SHA3")) { continue; } - runPBKDF2Benchmark(algorithm, providerName); + runPBKDF2Benchmark(algorithm, providerNames[i]); } catch (Exception e) { - /* Print but continue with other algorithms */ System.out.printf(" %-40s Error: %s%n", - algorithm + " (" + providerName + ")", e.getMessage()); + algorithm + " (" + providerNames[i] + ")", e.getMessage()); } } } + /* Run MessageDigest benchmarks with clean provider setup */ System.out.println("\n-----------------------------------------------------------------------------"); System.out.println("MessageDigest Benchmark Results"); System.out.println("-----------------------------------------------------------------------------"); for (int i = 0; i < providers.length; i++) { - Security.insertProviderAt(providers[i], 1); + setupProvidersForTest(providers[i]); String providerName = providerNames[i]; String digestProviderName = providerName; - if (!providerName.equals("wolfJCE")) { - if (providerName.equals("BC")) { - digestProviderName = "BC"; - } else { - try { - Provider sunProvider = Security.getProvider("SUN"); - if (sunProvider != null) { - Security.insertProviderAt(sunProvider, 1); - digestProviderName = "SUN"; - } else { - System.out.println("SUN provider not available, using " + providerName + " for MessageDigest"); - } - } catch (Exception e) { - System.out.println("Failed to set up SUN provider for " + providerName + ": " + e.getMessage()); - System.out.println("Using " + providerName + " for MessageDigest instead"); - } - } + // Handle special case for digest providers + if (!providerName.equals("wolfJCE") && !providerName.equals("BC")) { + digestProviderName = "SUN"; // Use SUN for standard digest algorithms } + setupDigestProvider(providerName); System.out.println("\n" + digestProviderName + ":"); + try { if (FeatureDetect.Md5Enabled() && isAlgorithmSupported("MD5", digestProviderName)) { runMessageDigestBenchmark("MD5", digestProviderName); @@ -879,12 +1132,39 @@ public class CryptoBenchmark { } } catch (Exception e) { System.out.println("Failed to benchmark MessageDigest with provider " + digestProviderName + ": " + e.getMessage()); - } finally { - Security.removeProvider(providers[i].getName()); - if (!providerName.equals("wolfJCE") && !providerName.equals("BC")) { - Provider sunProvider = Security.getProvider("SUN"); - if (sunProvider != null) { - Security.removeProvider(sunProvider.getName()); + } + } + + /* Run Signature benchmarks with clean provider setup */ + System.out.println("\n-----------------------------------------------------------------------------"); + System.out.println("Signature Benchmark Results"); + System.out.println("-----------------------------------------------------------------------------"); + + for (Provider provider : providers) { + setupProvidersForTest(provider); + String providerName = provider.getName(); + + System.out.println("\n" + providerName + ":"); + + if (providerName.equals("wolfJCE") || providerName.equals("BC")) { + /* Test wolfJCE/BC signature algorithms */ + for (String algorithm : SIGNATURE_ALGORITHMS) { + try { + runSignatureBenchmark(algorithm, providerName); + } catch (Exception e) { + System.out.printf(" %-40s Error: %s%n", + algorithm + " (" + providerName + ")", e.getMessage()); + } + } + } else if (providerName.equals("SunJCE")) { + /* For SunJCE, we'll test with the Sun providers */ + System.out.println("(Using SunRsaSign for RSA, SunEC for ECDSA)"); + for (String algorithm : SUN_SIGNATURE_ALGORITHMS) { + try { + runSignatureBenchmark(algorithm, "SunJCE"); + } catch (Exception e) { + System.out.printf(" %-40s Error: %s%n", + algorithm + " (SunJCE)", e.getMessage()); } } }