Skip to content

Fix critical cache-test bugs and improve similarity computation efficiency #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
atomiczsec merged 1 commit into from
Jun 11, 2026
Merged

Fix critical cache-test bugs and improve similarity computation efficiency #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions src/main/java/burp/BurpExtender.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,14 +336,13 @@ public void run() {
}
}

// Only run medium/low priority tests if no high-severity vulnerabilities found yet
// (hasHighSeverityVuln already checked above, but check again after medium tests)
boolean stillNoHighSeverityVuln = hashTraversalVulnerable ||
// Only run low-priority tests if no high/medium severity vulnerabilities found
boolean hasSignificantVuln = hashTraversalVulnerable ||
!successfulSelfRefExploits.isEmpty() ||
!successfulReverseTraversals.isEmpty() ||
!successfulHeaderAttacks.isEmpty();

if (!stillNoHighSeverityVuln) {
if (!hasSignificantVuln) {
updateStatus("Testing path normalization...");
// Prioritize common cacheable paths and templates
List<String> priorityPaths = Arrays.asList("/robots.txt", "/favicon.ico", "/", "/index.html");
Expand Down
197 changes: 135 additions & 62 deletions src/main/java/burp/RequestSender.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,14 @@
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Map;
import java.util.HashMap;
import java.util.Arrays;
import java.util.concurrent.ConcurrentHashMap;
import java.util.LinkedHashMap;

import java.util.Set;
import java.util.HashSet;
import java.util.concurrent.TimeUnit;
Expand All @@ -40,8 +40,9 @@ class RequestSender {
private static final Charset DEFAULT_CHARSET = StandardCharsets.ISO_8859_1;
private static final Pattern CHARSET_PATTERN = Pattern.compile("charset=([^;\\s]+)", Pattern.CASE_INSENSITIVE);
private static final Pattern SCRIPT_TAG_PATTERN = Pattern.compile("<script[^>]*>.*?</script>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
private static final int MAX_BODY_SIZE_FOR_COMPARISON = 200_000; // 200KB cap for similarity
private static final JaroWinklerSimilarity JARO_WINKLER = new JaroWinklerSimilarity();
private static final LevenshteinDistance LEVENSHTEIN = new LevenshteinDistance();
private static final LevenshteinDistance LEVENSHTEIN = new LevenshteinDistance(LEVENSHTEIN_THRESHOLD);

// High-performance Caffeine cache with TTL - configured via PerformanceConfig
private static final Cache<String, Map<String, Object>> RESPONSE_CACHE = Caffeine.newBuilder()
Expand All @@ -50,8 +51,8 @@ class RequestSender {
.build();

// Rate limiting and circuit breaker state per host
private static final Map<String, AtomicInteger> REQUEST_COUNTS = new ConcurrentHashMap<>();
private static final Map<String, AtomicLong> RATE_LIMIT_TIMESTAMPS = new ConcurrentHashMap<>();
private static final ConcurrentHashMap<String, AtomicInteger> REQUEST_COUNTS = new ConcurrentHashMap<>();
private static final ConcurrentHashMap<String, AtomicLong> RATE_LIMIT_TIMESTAMPS = new ConcurrentHashMap<>();
private static final Map<String, AtomicLong> LAST_FAILURE_TIME = new ConcurrentHashMap<>();
private static final Map<String, AtomicInteger> FAILURE_COUNTS = new ConcurrentHashMap<>();
// Rate limit configured via PerformanceConfig
Expand Down Expand Up @@ -470,6 +471,74 @@ protected static Map<String, Object> retrieveResponseDetails(IHttpService servic
return retrieveResponseDetails(service, request, 0);
}

/**
* Retrieves response details bypassing the local Caffeine cache.
* Required for two-phase cache tests where the second request must actually
* hit the remote server to observe X-Cache transitions (miss → hit).
*/
protected static Map<String, Object> retrieveResponseDetailsNoCache(IHttpService service, byte[] request) {
return retrieveResponseDetailsNoCache(service, request, 0);
}

private static Map<String, Object> retrieveResponseDetailsNoCache(IHttpService service, byte[] request, int retryCount) {
String hostKey = service.getHost();

for (int attempt = retryCount; attempt < MAX_RETRIES; attempt++) {
try {
if (isCircuitOpen(hostKey)) {
BurpExtender.logDebug("Circuit breaker open for " + hostKey);
return null;
}

if (!checkRateLimit(hostKey)) {
waitForRateLimit(hostKey);
}

long delay = calculateAdaptiveDelay(hostKey);
sleepRespectingInterrupts(delay);

long startTime = System.currentTimeMillis();
IHttpRequestResponse response = BurpExtender.getCallbacks().makeHttpRequest(service, request);
long responseTime = System.currentTimeMillis() - startTime;

if (response == null || response.getResponse() == null) {
recordFailure(hostKey);
if (attempt < MAX_RETRIES - 1) {
sleepRespectingInterrupts(calculateRetryDelay(attempt));
}
continue;
}

IResponseInfo responseInfo = BurpExtender.getHelpers().analyzeResponse(response.getResponse());
Map<String, Object> details = new HashMap<>();
details.put("statusCode", (int) responseInfo.getStatusCode());
details.put("headers", responseInfo.getHeaders());
details.put("responseTime", responseTime);

byte[] responseBody = java.util.Arrays.copyOfRange(
response.getResponse(),
responseInfo.getBodyOffset(),
response.getResponse().length);
details.put("body", responseBody);

if (responseInfo.getStatusCode() >= 200 && responseInfo.getStatusCode() < 500) {
recordSuccess(hostKey, responseTime);
} else {
recordFailure(hostKey);
}

return details;
} catch (Exception e) {
recordFailure(hostKey);
if (attempt < MAX_RETRIES - 1) {
sleepRespectingInterrupts(calculateRetryDelay(attempt));
}
}
}

return null;
}

private static Map<String, Object> retrieveResponseDetails(IHttpService service, byte[] request, int retryCount) {
String hostKey = service.getHost();
String cacheKey = buildServiceCacheKey(service, request);
Expand Down Expand Up @@ -580,32 +649,30 @@ private static boolean isCircuitOpen(String hostKey) {

private static boolean checkRateLimit(String hostKey) {
AtomicInteger count = REQUEST_COUNTS.computeIfAbsent(hostKey, k -> new AtomicInteger(0));
AtomicLong lastTime = RATE_LIMIT_TIMESTAMPS.computeIfAbsent(hostKey, k -> new AtomicLong(System.currentTimeMillis()));
AtomicLong windowStart = RATE_LIMIT_TIMESTAMPS.computeIfAbsent(hostKey, k -> new AtomicLong(System.currentTimeMillis()));

long currentTime = System.currentTimeMillis();
long timeDiff = currentTime - lastTime.get();
long windowStartTime = windowStart.get();

if (timeDiff >= 1000) {
count.set(0);
if (currentTime - windowStartTime >= 1000) {
// Atomically reset the window: CAS ensures only one thread resets
if (windowStart.compareAndSet(windowStartTime, currentTime)) {
count.set(0);
}
}

if (count.get() >= getMaxRequestsPerSecond()) {
// Atomically check-and-increment
int current = count.get();
if (current >= getMaxRequestsPerSecond()) {
return false;
}

// incrementAndGet may briefly exceed limit under contention, but that's acceptable
count.incrementAndGet();
lastTime.set(currentTime);
return true;
}

private static long calculateAdaptiveDelay(String hostKey) {
// Start with base delay, adjust based on response times
AtomicLong lastTime = RATE_LIMIT_TIMESTAMPS.get(hostKey);
if (lastTime == null) {
return 50; // Base delay
}
// Could be enhanced to track average response times
return 50;
return PerformanceConfig.getDelayMs();
}

private static int calculateRetryDelay(int retryCount) {
Expand All @@ -623,8 +690,15 @@ private static void recordFailure(String hostKey) {
LAST_FAILURE_TIME.computeIfAbsent(hostKey, k -> new AtomicLong(System.currentTimeMillis())).set(System.currentTimeMillis());
}

private static final int MAX_RATE_LIMIT_WAIT_MS = 5000;

private static void waitForRateLimit(String hostKey) {
long deadline = System.currentTimeMillis() + MAX_RATE_LIMIT_WAIT_MS;
while (!checkRateLimit(hostKey)) {
if (System.currentTimeMillis() >= deadline) {
BurpExtender.logDebug("Rate limit wait exceeded for " + hostKey + ", proceeding");
break;
}
sleepRespectingInterrupts(100);
}
}
Expand Down Expand Up @@ -708,17 +782,36 @@ private static Map<String, Object> testSimilar(String firstString, String second
String cleanedFirst = cleanResponseBody(firstString);
String cleanedSecond = cleanResponseBody(secondString);

double jaroDist = JARO_WINKLER.apply(cleanedFirst, cleanedSecond);
int levenDist = LEVENSHTEIN.apply(cleanedFirst, cleanedSecond);
// Early termination: if body sizes differ drastically, they're not similar
if (cleanedFirst.isEmpty() && cleanedSecond.isEmpty()) {
results.put("similar", true);
results.put("jaro", 1.0);
results.put("levenshtein", 0);
return results;
}
int maxLen = Math.max(cleanedFirst.length(), cleanedSecond.length());
int sizeDiff = Math.abs(cleanedFirst.length() - cleanedSecond.length());
if (maxLen > 0 && sizeDiff > maxLen / 2) {
results.put("similar", false);
results.put("jaro", 0.0);
results.put("levenshtein", sizeDiff);
return results;
}

// Cap body size to avoid O(n*m) explosion on large responses
String cappedFirst = cleanedFirst.length() > MAX_BODY_SIZE_FOR_COMPARISON
? cleanedFirst.substring(0, MAX_BODY_SIZE_FOR_COMPARISON) : cleanedFirst;
String cappedSecond = cleanedSecond.length() > MAX_BODY_SIZE_FOR_COMPARISON
? cleanedSecond.substring(0, MAX_BODY_SIZE_FOR_COMPARISON) : cleanedSecond;

double jaroDist = JARO_WINKLER.apply(cappedFirst, cappedSecond);
// LevenshteinDistance with threshold returns -1 if distance exceeds the threshold
int levenDist = LEVENSHTEIN.apply(cappedFirst, cappedSecond);

// Fixed similarity logic: Both metrics must indicate similarity for a positive match
// JaroWinklerSimilarity returns 0-1 (higher is better)
// LevenshteinDistance returns edit distance (lower is better)
boolean jaroSimilar = jaroDist >= JARO_THRESHOLD;
boolean levenSimilar = levenDist <= LEVENSHTEIN_THRESHOLD;

// Require BOTH metrics to indicate similarity to reduce false positives
// This prevents cases where very different content has coincidentally low edit distance
// -1 means exceeded threshold → not similar
boolean levenSimilar = levenDist >= 0 && levenDist <= LEVENSHTEIN_THRESHOLD;

boolean similar = jaroSimilar && levenSimilar;

results.put("similar", similar);
Expand All @@ -727,10 +820,9 @@ private static Map<String, Object> testSimilar(String firstString, String second
return results;
}

// Helper method to generate a random alphanumeric string
protected static String generateRandomString(int length) {
String chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
Random random = new Random();
ThreadLocalRandom random = ThreadLocalRandom.current();
StringBuilder sb = new StringBuilder(length);
for (int i = 0; i < length; i++) {
sb.append(chars.charAt(random.nextInt(chars.length())));
Expand Down Expand Up @@ -775,9 +867,8 @@ protected static boolean testRelativeNormalizationExploit(IHttpRequestResponse m
return false;
}

try { Thread.sleep(PerformanceConfig.getDelayMs()); } catch (InterruptedException ignored) {}
byte[] requestBytes2 = requestBytes1;
Map<String, Object> details2 = retrieveResponseDetails(message.getHttpService(), requestBytes2);
sleepRespectingInterrupts(PerformanceConfig.getDelayMs());
Map<String, Object> details2 = retrieveResponseDetailsNoCache(message.getHttpService(), requestBytes1);
if (details2 == null) {
return false;
}
Expand Down Expand Up @@ -885,9 +976,8 @@ protected static boolean testPrefixNormalizationExploit(IHttpRequestResponse mes

// --- Second Request ---
// Introduce a small delay - sometimes caches need a moment
try { Thread.sleep(PerformanceConfig.getDelayMs()); } catch (InterruptedException ignored) {}
byte[] requestBytes2 = requestBytes1; // Re-use
Map<String, Object> details2 = retrieveResponseDetails(message.getHttpService(), requestBytes2);
sleepRespectingInterrupts(PerformanceConfig.getDelayMs());
Map<String, Object> details2 = retrieveResponseDetailsNoCache(message.getHttpService(), requestBytes1);
if (details2 == null) return false;
int statusCode2 = (int) details2.get("statusCode");
@SuppressWarnings("unchecked") List<String> headers2 = (List<String>) details2.get("headers");
Expand Down Expand Up @@ -943,8 +1033,8 @@ protected static boolean testHashPathTraversal(IHttpRequestResponse message, Str
}

if (statusCode1 == 200) {
byte[] requestBytes2 = requestBytes1;
Map<String, Object> details2 = retrieveResponseDetails(message.getHttpService(), requestBytes2);
sleepRespectingInterrupts(PerformanceConfig.getDelayMs());
Map<String, Object> details2 = retrieveResponseDetailsNoCache(message.getHttpService(), requestBytes1);

if (details2 != null) {
@SuppressWarnings("unchecked")
Expand Down Expand Up @@ -1026,9 +1116,8 @@ protected static boolean testSelfReferentialNormalization(IHttpRequestResponse m
return false;
}

try { Thread.sleep(PerformanceConfig.getDelayMs()); } catch (InterruptedException ignored) {}
byte[] requestBytes2 = requestBytes1;
Map<String, Object> details2 = retrieveResponseDetails(message.getHttpService(), requestBytes2);
sleepRespectingInterrupts(PerformanceConfig.getDelayMs());
Map<String, Object> details2 = retrieveResponseDetailsNoCache(message.getHttpService(), requestBytes1);
if (details2 == null) {
return false;
}
Expand Down Expand Up @@ -1142,9 +1231,8 @@ protected static boolean testReverseTraversal(IHttpRequestResponse message, Stri
return false;
}

try { Thread.sleep(PerformanceConfig.getDelayMs()); } catch (InterruptedException ignored) {}
byte[] requestBytes2 = requestBytes1;
Map<String, Object> details2 = retrieveResponseDetails(message.getHttpService(), requestBytes2);
sleepRespectingInterrupts(PerformanceConfig.getDelayMs());
Map<String, Object> details2 = retrieveResponseDetailsNoCache(message.getHttpService(), requestBytes1);
if (details2 == null) {
return false;
}
Expand Down Expand Up @@ -1365,7 +1453,7 @@ protected static boolean testUnicodeNormalization(IHttpRequestResponse message,
* Returns confidence level: "High", "Medium", or "Low".
*/
protected static String multiRoundVerification(IHttpRequestResponse message, byte[] testRequest, int rounds) {
if (rounds < 2) rounds = 3; // Minimum 3 rounds
if (rounds < 3) rounds = 3;

byte[] originalAuthRequest = buildHttpRequest(message, null, null, true);
Map<String, Object> originalDetails = retrieveResponseDetails(message.getHttpService(), originalAuthRequest);
Expand All @@ -1376,9 +1464,9 @@ protected static String multiRoundVerification(IHttpRequestResponse message, byt
int similarResponses = 0;

for (int i = 0; i < rounds; i++) {
try { Thread.sleep(PerformanceConfig.getDelayMs()); } catch (InterruptedException ignored) {}
sleepRespectingInterrupts(PerformanceConfig.getDelayMs());

Map<String, Object> details = retrieveResponseDetails(message.getHttpService(), testRequest);
Map<String, Object> details = retrieveResponseDetailsNoCache(message.getHttpService(), testRequest);
if (details == null) continue;

@SuppressWarnings("unchecked")
Expand Down Expand Up @@ -1406,22 +1494,7 @@ protected static String multiRoundVerification(IHttpRequestResponse message, byt
}
}

/**
* Enhanced similarity test with early termination for large responses.
*/
protected static Map<String, Object> testSimilarOptimized(String firstString, String secondString) {
// Early termination for very different sizes
int sizeDiff = Math.abs(firstString.length() - secondString.length());
if (sizeDiff > firstString.length() * 0.5) {
Map<String, Object> results = new HashMap<>();
results.put("similar", false);
results.put("jaro", 0.0);
results.put("levenshtein", Integer.MAX_VALUE);
return results;
}

return testSimilar(firstString, secondString);
}

/**
* Builds a request by cloning the original message and replacing the query string.
Expand Down
Loading