Skip to content

Add S3 backend to cloudsync #18815

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
russell-parks wants to merge 12 commits into
base: master
Choose a base branch
from
Open

Add S3 backend to cloudsync #18815

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
c7bf80f
S3 Implementation
russell-parks May 5, 2025
64dc265
Standards cleanup
russell-parks Mar 1, 2026
39af5ec
Move to task-based non-blocking pattern
russell-parks Mar 1, 2026
737ba8f
Build fixes
russell-parks Mar 1, 2026
a1457a1
Add multipart uploads for S3
russell-parks Mar 9, 2026
28f0d1c
Remove comments [skip ci]
russell-parks Mar 10, 2026
39153fc
Enable windows builds
russell-parks Mar 10, 2026
05dea59
Removing untested branches
russell-parks Mar 10, 2026
666fd49
Fix code review comments
russell-parks Mar 11, 2026
0b2493f
Changed url parsing logic
russell-parks Mar 12, 2026
9495430
Fix b2 for path and virtual host styles
russell-parks Mar 13, 2026
c289bd1
Move net_http logging to http_t
russell-parks Mar 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion configuration.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1617,12 +1617,19 @@ static struct config_array_setting *populate_settings_array(
SETTING_ARRAY("cheevos_leaderboards_enable", settings->arrays.cheevos_leaderboards_enable, true, "", true); /* deprecated */
#endif

#ifdef HAVE_NETWORKING
Copy link
Copy Markdown
Collaborator

@warmenhoven warmenhoven Mar 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why move this down a line?

Copy link
Copy Markdown
Contributor Author

@russell-parks russell-parks Mar 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was having a problem when I was finding the right place for some of the config and #ifdefs and had forced myself into a faux problem with netplay_mitm_server. Looks like I didn't revert that pushdown, I'll move it back up.

SETTING_ARRAY("netplay_mitm_server", settings->arrays.netplay_mitm_server, false, NULL, true);
#ifdef HAVE_NETWORKING
#ifdef HAVE_CLOUDSYNC
SETTING_ARRAY("webdav_url", settings->arrays.webdav_url, false, NULL, true);
SETTING_ARRAY("webdav_username", settings->arrays.webdav_username, false, NULL, true);
SETTING_ARRAY("webdav_password", settings->arrays.webdav_password, false, NULL, true);
SETTING_ARRAY("google_drive_refresh_token", settings->arrays.google_drive_refresh_token, false, NULL, true);
#ifdef HAVE_S3
SETTING_ARRAY("s3_url", settings->arrays.s3_url, false, NULL, true);
SETTING_ARRAY("access_key_id", settings->arrays.access_key_id, false, NULL, true);
SETTING_ARRAY("secret_access_key", settings->arrays.secret_access_key, false, NULL, true);
#endif
#endif
SETTING_ARRAY("youtube_stream_key", settings->arrays.youtube_stream_key, true, NULL, true);
SETTING_ARRAY("twitch_stream_key", settings->arrays.twitch_stream_key, true, NULL, true);
SETTING_ARRAY("facebook_stream_key", settings->arrays.facebook_stream_key, true, NULL, true);
Expand Down
9 changes: 9 additions & 0 deletions configuration.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -541,10 +541,19 @@ typedef struct settings
char audio_device[NAME_MAX_LENGTH];
char camera_device[NAME_MAX_LENGTH];
char netplay_mitm_server[NAME_MAX_LENGTH];
#ifdef HAVE_NETWORKING
#ifdef HAVE_CLOUDSYNC
char webdav_url[NAME_MAX_LENGTH];
char webdav_username[NAME_MAX_LENGTH];
char webdav_password[NAME_MAX_LENGTH];
char google_drive_refresh_token[2048];
#ifdef HAVE_S3
char s3_url[NAME_MAX_LENGTH];
char access_key_id[128];
char secret_access_key[186]; /* TODO/RESEARCH - check size, ex https://github.com/winscp/winscp/pull/15/files */
#endif
#endif
#endif

char crt_switch_timings[NAME_MAX_LENGTH];
char input_reserved_devices[MAX_USERS][NAME_MAX_LENGTH];
Expand Down
3 changes: 3 additions & 0 deletions griffin/griffin.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1721,6 +1721,9 @@ CLOUD SYNC
#ifdef HAVE_SMBCLIENT
#include "../network/cloud_sync/smb.c"
#endif
#ifdef HAVE_S3
#include "../network/cloud_sync/s3.c"
#endif
#endif

/*============================================================
Expand Down
12 changes: 12 additions & 0 deletions intl/msg_hash_lbl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3550,6 +3550,18 @@ MSG_HASH(
MENU_ENUM_LABEL_CLOUD_SYNC_PASSWORD,
"cloud_sync_password"
)
MSG_HASH(
MENU_ENUM_LABEL_CLOUD_SYNC_ACCESS_KEY_ID,
"cloud_sync_access_key_id"
)
MSG_HASH(
MENU_ENUM_LABEL_CLOUD_SYNC_SECRET_ACCESS_KEY,
"cloud_sync_secret_access_key"
)
MSG_HASH(
MENU_ENUM_LABEL_CLOUD_SYNC_S3_URL,
"cloud_sync_s3_url"
)
MSG_HASH(
MENU_ENUM_LABEL_SCAN_DIRECTORY,
"scan_directory"
Expand Down
24 changes: 24 additions & 0 deletions intl/msg_hash_us.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1158,6 +1158,30 @@ MSG_HASH(
MENU_ENUM_SUBLABEL_CLOUD_SYNC_PASSWORD,
"Your password for your cloud storage account."
)
MSG_HASH(
MENU_ENUM_LABEL_VALUE_CLOUD_SYNC_ACCESS_KEY_ID,
"Access Key ID"
)
MSG_HASH(
MENU_ENUM_SUBLABEL_CLOUD_SYNC_ACCESS_KEY_ID,
"Your access key ID for your cloud storage account."
)
MSG_HASH(
MENU_ENUM_LABEL_VALUE_CLOUD_SYNC_SECRET_ACCESS_KEY,
"Secret Access Key"
)
MSG_HASH(
MENU_ENUM_SUBLABEL_CLOUD_SYNC_SECRET_ACCESS_KEY,
"Your secret access key for your cloud storage account."
)
MSG_HASH(
MENU_ENUM_LABEL_VALUE_CLOUD_SYNC_S3_URL,
"S3 URL"
)
MSG_HASH(
MENU_ENUM_SUBLABEL_CLOUD_SYNC_S3_URL,
"Your S3 endpoint URL for cloud storage."
)
MSG_HASH(
MENU_ENUM_LABEL_VALUE_LOGGING_SETTINGS,
"Logging"
Expand Down
1 change: 1 addition & 0 deletions libretro-common/include/net/net_http.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ bool net_http_error(struct http_t *state);
* If the status is not 20x and accept_error is false, it returns NULL.
**/
struct string_list *net_http_headers(struct http_t *state);
struct string_list *net_http_headers_ex(struct http_t *state, bool accept_error);

/**
* net_http_data:
Expand Down
87 changes: 81 additions & 6 deletions libretro-common/net/net_http.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <errno.h>

#include <net/net_http.h>
#include <net/net_compat.h>
Expand Down Expand Up @@ -133,6 +134,54 @@ struct http_connection_t
bool ssl;
};

static void net_http_log_transport_state(
const struct http_t *state, const char *stage, ssize_t io_len)
{
#if defined(DEBUG)
const char *method = "GET";
const char *domain = "<null>";
const char *path = "<null>";
int port = 0;
int fd = -1;
int connected = 0;

if (state)
{
method = state->request.method ? state->request.method : "GET";
domain = state->request.domain ? state->request.domain : "<null>";
path = state->request.path ? state->request.path : "<null>";
port = state->request.port;

if (state->conn)
{
fd = state->conn->fd;
connected = state->conn->connected ? 1 : 0;
}
}

fprintf(stderr,
Copy link
Copy Markdown
Collaborator

@warmenhoven warmenhoven Mar 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel like this is better done as additional state on http_t that can then be logged by the caller.

Copy link
Copy Markdown
Contributor Author

@russell-parks russell-parks Mar 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added this while troubleshooting the -1 http status. I could definitely move it to http_t or even remove it if you prefer, thoughts?

Copy link
Copy Markdown
Collaborator

@warmenhoven warmenhoven Mar 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like the idea of being able to introspect what's going on inside of net_http but verbosity/log isn't available and printf/fprintf is not a good answer. So I'd be more in favor of moving it so callers can do the logging. If that ends up being too big or too big for this pr I'd prefer it's just removed entirely (and possibly added in a new pr). Up to you if you want to do that as part of this pr, or just remove it for this one.

Copy link
Copy Markdown
Contributor Author

@russell-parks russell-parks Mar 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved this to http_t. Lmk how it looks.

"[net_http] %s: method=%s host=%s port=%d path=/%s ssl=%d fd=%d connected=%d request_sent=%d err=%d io_len=%ld errno=%d (%s)\n",
stage ? stage : "unknown",
method,
domain,
port,
path,
state ? (state->ssl ? 1 : 0) : 0,
fd,
connected,
state ? (state->request_sent ? 1 : 0) : 0,
state ? (state->err ? 1 : 0) : 0,
(long)io_len,
errno,
strerror(errno));
fflush(stderr);
#else
(void)state;
(void)stage;
(void)io_len;
#endif
}

struct dns_cache_entry
{
char *domain;
Expand Down Expand Up @@ -920,13 +969,16 @@ static bool net_http_new_socket(struct http_t *state)
int fd;
if (!entry->addr)
{
net_http_log_transport_state(state, "dns_lookup_failed", -1);
UNLOCK_DNS_CACHE();
return false;
}
addr = entry->addr;
fd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
if (fd >= 0)
state->conn = net_http_conn_pool_add(state->request.domain, state->request.port, fd, state->ssl);
else
net_http_log_transport_state(state, "socket_create_failed", -1);
/* still waiting on thread */
UNLOCK_DNS_CACHE();
return (fd >= 0);
Expand Down Expand Up @@ -960,6 +1012,11 @@ static bool net_http_connect(struct http_t *state)
struct conn_pool_entry *conn = state->conn;
struct dns_cache_entry *dns_entry = net_http_dns_cache_find(state->request.domain, state->request.port);
/* we just used/added this in _new_socket above, if it's not there it's a big bug */
if (!dns_entry || !dns_entry->addr || !conn)
Comment thread
russell-parks marked this conversation as resolved.
Outdated
{
net_http_log_transport_state(state, "connect_missing_dns_or_conn", -1);
return false;
}
addr = dns_entry->addr;

#ifndef HAVE_SSL
Expand All @@ -968,13 +1025,11 @@ static bool net_http_connect(struct http_t *state)
#else
if (state->ssl)
{
if (!conn)
return false;

for (next_addr = addr; conn->fd >= 0; conn->fd = socket_next((void**)&next_addr))
{
if (!(conn->ssl_ctx = ssl_socket_init(conn->fd, state->request.domain)))
{
net_http_log_transport_state(state, "ssl_init_failed", -1);
socket_close(conn->fd);
break;
}
Expand All @@ -992,6 +1047,7 @@ static bool net_http_connect(struct http_t *state)

if (ssl_socket_connect(conn->ssl_ctx, next_addr, timeout, true) < 0)
{
net_http_log_transport_state(state, "ssl_connect_failed", -1);
ssl_socket_close(conn->ssl_ctx);
ssl_socket_free(conn->ssl_ctx);
conn->ssl_ctx = NULL;
Expand Down Expand Up @@ -1019,6 +1075,7 @@ static bool net_http_connect(struct http_t *state)
return true;
}

net_http_log_transport_state(state, "socket_connect_failed", -1);
socket_close(conn->fd);
}
conn->fd = -1; /* already closed */
Expand All @@ -1039,14 +1096,20 @@ static void net_http_send_str(
{
if (!ssl_socket_send_all_blocking(
state->conn->ssl_ctx, text, text_size, true))
{
state->err = true;
net_http_log_transport_state(state, "ssl_send_failed", -1);
}
}
else
#endif
{
if (!socket_send_all_blocking(
state->conn->fd, text, text_size, true))
{
state->err = true;
net_http_log_transport_state(state, "socket_send_failed", -1);
}
}
}

Expand Down Expand Up @@ -1097,9 +1160,12 @@ static bool net_http_send_request(struct http_t *state)
size_t _len, len;
char *len_str = NULL;

if (!request->postdata && !string_is_equal(request->method, "PUT"))
if (!request->postdata &&
!string_is_equal(request->method, "PUT") &&
request->contentlength > 0)
{
state->err = true;
net_http_log_transport_state(state, "post_without_payload", -1);
return true;
}

Expand Down Expand Up @@ -1460,6 +1526,7 @@ bool net_http_update(struct http_t *state, size_t* progress, size_t* total)
{
if (_len < 0 || state->err)
{
net_http_log_transport_state(state, "receive_header_failed", _len);
net_http_conn_pool_remove(state->conn);
state->conn = NULL;
state->err = true;
Expand All @@ -1474,6 +1541,7 @@ bool net_http_update(struct http_t *state, size_t* progress, size_t* total)
{
if (!net_http_receive_body(state, _len))
{
net_http_log_transport_state(state, "receive_body_failed", _len);
net_http_conn_pool_remove(state->conn);
state->conn = NULL;
state->err = true;
Expand Down Expand Up @@ -1548,13 +1616,20 @@ int net_http_status(struct http_t *state)
* caller of net_http_new; it is not freed by net_http_delete().
* If the status is not 20x and accept_err is false, it returns NULL.
**/
struct string_list *net_http_headers(struct http_t *state)
struct string_list *net_http_headers_ex(struct http_t *state, bool accept_err)
{
if (!state || !state->err)
if (!state)
return NULL;
if (!accept_err && !state->err)
return NULL;
return state->response.headers;
}

struct string_list *net_http_headers(struct http_t *state)
{
return net_http_headers_ex(state, false);
}

/**
* net_http_data:
*
Expand Down
12 changes: 12 additions & 0 deletions menu/cbs/menu_cbs_sublabel.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,8 +270,11 @@ DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_cloud_sync_sync_thumbs, MENU_
DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_cloud_sync_sync_system, MENU_ENUM_SUBLABEL_CLOUD_SYNC_SYNC_SYSTEM)
DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_cloud_sync_driver, MENU_ENUM_SUBLABEL_CLOUD_SYNC_DRIVER)
DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_cloud_sync_url, MENU_ENUM_SUBLABEL_CLOUD_SYNC_URL)
DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_cloud_sync_s3_url, MENU_ENUM_SUBLABEL_CLOUD_SYNC_S3_URL)
DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_cloud_sync_username, MENU_ENUM_SUBLABEL_CLOUD_SYNC_USERNAME)
DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_cloud_sync_password, MENU_ENUM_SUBLABEL_CLOUD_SYNC_PASSWORD)
DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_cloud_sync_access_key_id, MENU_ENUM_SUBLABEL_CLOUD_SYNC_ACCESS_KEY_ID)
DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_cloud_sync_secret_access_key, MENU_ENUM_SUBLABEL_CLOUD_SYNC_SECRET_ACCESS_KEY)
DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_logging_settings_list, MENU_ENUM_SUBLABEL_LOGGING_SETTINGS)
DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_user_interface_settings_list, MENU_ENUM_SUBLABEL_USER_INTERFACE_SETTINGS)
DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_ai_service_settings_list, MENU_ENUM_SUBLABEL_AI_SERVICE_SETTINGS)
Expand Down Expand Up @@ -5254,12 +5257,21 @@ int menu_cbs_init_bind_sublabel(menu_file_list_cbs_t *cbs,
case MENU_ENUM_LABEL_CLOUD_SYNC_URL:
BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_cloud_sync_url);
break;
case MENU_ENUM_LABEL_CLOUD_SYNC_S3_URL:
BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_cloud_sync_s3_url);
break;
case MENU_ENUM_LABEL_CLOUD_SYNC_USERNAME:
BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_cloud_sync_username);
break;
case MENU_ENUM_LABEL_CLOUD_SYNC_PASSWORD:
BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_cloud_sync_password);
break;
case MENU_ENUM_LABEL_CLOUD_SYNC_ACCESS_KEY_ID:
BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_cloud_sync_access_key_id);
break;
case MENU_ENUM_LABEL_CLOUD_SYNC_SECRET_ACCESS_KEY:
BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_cloud_sync_secret_access_key);
break;
case MENU_ENUM_LABEL_LOGGING_SETTINGS:
BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_logging_settings_list);
break;
Expand Down
Loading
Loading