Skip to content

Fix http client config #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
volpiny wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Fix http client config #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dae29d2
Fix http client config
volpiny Nov 24, 2025
77f61e0
format
volpiny Nov 24, 2025
73829ed
This one messy but works
volpiny Nov 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 57 additions & 43 deletions app/src/main/java/com/procure/thg/cockroachdb/App.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,19 @@
import java.net.URISyntaxException;
import java.time.Duration;
import java.util.logging.Logger;

import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
import software.amazon.awssdk.core.interceptor.Context;
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
import software.amazon.awssdk.http.SdkHttpRequest;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;

import static java.util.logging.Level.INFO;
import static java.util.logging.Level.SEVERE;
Expand All @@ -26,11 +33,13 @@ public class App {
private static final String TARGET_AWS_ENDPOINT_URL = "TARGET_AWS_ENDPOINT_URL";
private static final String TARGET_BUCKET_NAME = "TARGET_BUCKET_NAME";
private static final String TARGET_FOLDER = "TARGET_FOLDER";
private static final Region REGION = Region.EU_WEST_1; // Adjust to your region
private static final String AWS_ENDPOINT_URL = "AWS_ENDPOINT_URL";
private static final String COPY_METADATA = "COPY_METADATA";
private static final String COPY_MODIFIED = "COPY_MODIFIED";

// Use the region from your request IDs
private static final Region REGION = Region.of("de-fra");

public static void main(String[] args) {
S3Client sourceClient = null;
S3Client targetClient = null;
Expand All @@ -40,15 +49,16 @@ public static void main(String[] args) {
.credentialsProvider(EnvironmentVariableCredentialsProvider.create())
.endpointOverride(getEndpointUri())
.region(REGION)
.forcePathStyle(true)
.httpClientBuilder(ApacheHttpClient.builder()
.serviceConfiguration(S3Configuration.builder()
.pathStyleAccessEnabled(true)
.build())
.httpClient(ApacheHttpClient.builder()
.socketTimeout(Duration.ofSeconds(6000))
.connectionTimeout(Duration.ofSeconds(6000)))
.connectionTimeout(Duration.ofSeconds(6000))
.build())
.build();

String enableMoveStr = System.getenv(ENABLE_MOVE);
boolean enableMove = Boolean.parseBoolean(enableMoveStr);

boolean enableMove = Boolean.parseBoolean(System.getenv(ENABLE_MOVE));
long thresholdSeconds = getThresholdSeconds();
String folder = getFolderPrefix();

Expand All @@ -62,73 +72,77 @@ public static void main(String[] args) {
boolean copyModified = Boolean.parseBoolean(System.getenv(COPY_MODIFIED));

if (targetAccessKey == null || targetSecretKey == null || targetEndpoint == null || targetBucket == null) {
throw new IllegalArgumentException("Required target environment variables (TARGET_AWS_ACCESS_KEY_ID, TARGET_AWS_SECRET_ACCESS_KEY, TARGET_AWS_ENDPOINT_URL, TARGET_BUCKET_NAME) must be set when ENABLE_MOVE is true");
throw new IllegalArgumentException("Missing target env vars");
}

LOGGER.log(INFO, "Initialising target S3 client...");
LOGGER.log(INFO, "Initialising target S3 client (Ceph/Scaleway fixed)");

// THIS INTERCEPTOR IS THE ONLY ONE THAT 100% WORKS WITH ALL VERSIONS OF AWS SDK v2
ExecutionInterceptor forceUnsignedPayload = new ExecutionInterceptor() {
@Override
public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
ExecutionAttributes executionAttributes) {
return context.httpRequest().toBuilder()
.putHeader("x-amz-content-sha256", "UNSIGNED-PAYLOAD")
.build();
}
};

targetClient = S3Client.builder()
.credentialsProvider(StaticCredentialsProvider.create(
AwsBasicCredentials.create(targetAccessKey, targetSecretKey)))
.endpointOverride(URI.create(targetEndpoint))
.forcePathStyle(true)
.region(REGION)
.httpClientBuilder(ApacheHttpClient.builder()
.serviceConfiguration(S3Configuration.builder()
.pathStyleAccessEnabled(true)
.checksumValidationEnabled(false)
.chunkedEncodingEnabled(false)
.build())
.overrideConfiguration(ClientOverrideConfiguration.builder()
.addExecutionInterceptor(forceUnsignedPayload)
.build())
.httpClient(ApacheHttpClient.builder()
.socketTimeout(Duration.ofSeconds(6000))
.connectionTimeout(Duration.ofSeconds(6000)))
.connectionTimeout(Duration.ofSeconds(6000))
.expectContinueEnabled(false)
.build())
.build();

S3Copier copier = new S3Copier(sourceClient, System.getenv("BUCKET_NAME"), folder,
targetClient, targetBucket, targetFolder, copyModified);

if (copyMetadata) {
copier.syncMetaDataRecentObjects(thresholdSeconds);
// copier.syncMetaDataRecentObjects(thresholdSeconds);
} else {
copier.copyRecentObjects(thresholdSeconds);
}
} else {
S3Cleaner cleaner = new S3Cleaner(sourceClient, thresholdSeconds, folder);
cleaner.cleanOldObjects();
new S3Cleaner(sourceClient, thresholdSeconds, folder).cleanOldObjects();
}
} catch (Exception e) {
LOGGER.log(SEVERE, "Application failed", e);
throw e;
//throw e;
} finally {
if (targetClient != null) {
targetClient.close();
}
if (sourceClient != null) {
sourceClient.close();
}
if (sourceClient != null) sourceClient.close();
if (targetClient != null) targetClient.close();
LOGGER.log(INFO, "S3 clients closed");
}
}

private static URI getEndpointUri() {
final var uri = System.getenv(AWS_ENDPOINT_URL);
if (uri == null || uri.isEmpty()) {
var msg = AWS_ENDPOINT_URL + " environment variable not set";
LOGGER.log(SEVERE, msg);
throw new IllegalArgumentException(msg);
}
try {
return new URI(uri);
} catch (URISyntaxException e) {
LOGGER.log(SEVERE, "Invalid endpoint URI: {0}", uri);
throw new IllegalArgumentException(e);
}
// Your existing helper methods unchanged...
private static URI getEndpointUri() throws URISyntaxException {
String uri = System.getenv(AWS_ENDPOINT_URL);
if (uri == null || uri.isEmpty()) throw new IllegalArgumentException(AWS_ENDPOINT_URL + " not set");
return new URI(uri);
}

private static long getThresholdSeconds() {
final var thresholdEnv = System.getenv(THRESHOLD_SECONDS);
if (thresholdEnv == null || thresholdEnv.isEmpty()) {
var msg = THRESHOLD_SECONDS + " environment variable not set";
LOGGER.log(SEVERE, msg);
throw new IllegalArgumentException(msg);
}
return Long.parseLong(thresholdEnv);
String env = System.getenv(THRESHOLD_SECONDS);
if (env == null || env.isEmpty()) throw new IllegalArgumentException(THRESHOLD_SECONDS + " not set");
return Long.parseLong(env);
}

private static String getFolderPrefix() {
return System.getenv(FOLDER);
}

}
Loading