diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..b73e7437 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.direnv +.env +.git +.npmrc +coverage +dist +node_modules +oclif.manifest.json +tsconfig.tsbuildinfo diff --git a/docker/README.md b/docker/README.md index 0ee96db1..57ae5278 100644 --- a/docker/README.md +++ b/docker/README.md @@ -27,6 +27,7 @@ services: WEBDAV_USERNAME: "" # (Optional) Custom username for WebDAV authentication WEBDAV_PASSWORD: "" # (Optional) Custom password for WebDAV authentication WEBDAV_DELETE_FILES_PERMANENTLY: "" # (Optional) Delete files permanently. Set to 'true' to enable + WEBDAV_HYPER_BACKUP_MODE: "" # (Optional) Enable Synology Hyper Backup compatibility and performance shortcuts. Set to 'true' to enable ports: - "127.0.0.1:3005:3005" # Map container port to host. Change if WEBDAV_PORT is customized ``` @@ -54,6 +55,7 @@ docker run -d \ -e WEBDAV_USERNAME="" \ -e WEBDAV_PASSWORD="" \ -e WEBDAV_DELETE_FILES_PERMANENTLY="" \ + -e WEBDAV_HYPER_BACKUP_MODE="" \ -p 127.0.0.1:3005:3005 \ internxt/webdav:latest ``` @@ -94,6 +96,7 @@ You can also run the `internxt/webdav` image directly on popular NAS devices lik | `WEBDAV_USERNAME` | ❌ No | Username for custom WebDAV authentication. Required if `WEBDAV_CUSTOM_AUTH` is enabled. | | `WEBDAV_PASSWORD` | ❌ No | Password for custom WebDAV authentication. Required if `WEBDAV_CUSTOM_AUTH` is enabled. | | `WEBDAV_DELETE_FILES_PERMANENTLY` | ❌ No | Delete files permanently instead of moving them to trash. Set to `true`to enable. | +| `WEBDAV_HYPER_BACKUP_MODE` | ❌ No | Enable Synology Hyper Backup compatibility and performance shortcuts. Set to `true` to enable. | --- diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 4c81ecf3..c83eaf9a 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -57,10 +57,13 @@ if [ "$deleteFilesPermanently" = "true" ] || [ "$deleteFilesPermanently" = "1" ] WEBDAV_ARGS="$WEBDAV_ARGS -d" fi -internxt webdav-config $WEBDAV_ARGS +hyperBackupMode=$(echo "$WEBDAV_HYPER_BACKUP_MODE" | tr '[:upper:]' '[:lower:]') +if [ "$hyperBackupMode" = "true" ] || [ "$hyperBackupMode" = "1" ] || [ "$hyperBackupMode" = "yes" ] || [ "$hyperBackupMode" = "y" ]; then + echo "Enabling WebDAV Hyper Backup mode" + WEBDAV_ARGS="$WEBDAV_ARGS --hyperBackupMode" +fi -internxt webdav enable +internxt webdav-config $WEBDAV_ARGS mkdir -p /root/.internxt-cli/logs -touch /root/.internxt-cli/logs/internxt-webdav-combined.log -tail -f /root/.internxt-cli/logs/internxt-webdav-combined.log +exec node /app/dist/webdav/index.js diff --git a/docker/health_check.sh b/docker/health_check.sh index 91a6507e..071fdb0b 100644 --- a/docker/health_check.sh +++ b/docker/health_check.sh @@ -6,9 +6,14 @@ set -e WHOAMI_OUTPUT=$(internxt whoami --json 2>/dev/null || true) WHOAMI_EMAIL=$(echo "$WHOAMI_OUTPUT" | jq -r '.login.user.email // empty') -# Check WebDAV status -STATUS_OUTPUT=$(internxt webdav status --json 2>/dev/null || true) -WEBDAV_STATUS=$(echo "$STATUS_OUTPUT" | jq -r '.message | split(" ") | last // empty') +PORT="${WEBDAV_PORT:-3005}" + +# Check the WebDAV listener. A TCP connection is enough here; protocol may be HTTP or HTTPS. +if node -e "const net = require('node:net'); const socket = net.createConnection({ host: '127.0.0.1', port: Number(process.env.WEBDAV_PORT || 3005) }, () => { socket.destroy(); process.exit(0); }); socket.setTimeout(2000); socket.on('timeout', () => { socket.destroy(); process.exit(1); }); socket.on('error', () => process.exit(1));"; then + WEBDAV_STATUS="online" +else + WEBDAV_STATUS="offline" +fi if [ "$WHOAMI_EMAIL" = "$INXT_USER" ] && [ "$WEBDAV_STATUS" = "online" ]; then echo "Healthcheck passed. User: $INXT_USER, WebDAV status: $WEBDAV_STATUS" diff --git a/src/commands/webdav-config.ts b/src/commands/webdav-config.ts index 3d13f893..89de24f8 100644 --- a/src/commands/webdav-config.ts +++ b/src/commands/webdav-config.ts @@ -74,13 +74,30 @@ export default class WebDAVConfig extends Command { default: undefined, allowNo: true, }), + hyperBackupMode: Flags.boolean({ + description: 'Enables Synology Hyper Backup compatibility and performance shortcuts.', + required: false, + default: undefined, + allowNo: true, + }), }; static readonly enableJsonFlag = true; public run = async () => { const { flags } = await this.parse(WebDAVConfig); - const { host, port, https, http, timeout, createFullPath, customAuth, username, password, deleteFilesPermanently } = - flags; + const { + host, + port, + https, + http, + timeout, + createFullPath, + customAuth, + username, + password, + deleteFilesPermanently, + hyperBackupMode, + } = flags; const nonInteractive = flags['non-interactive']; const webdavConfig = await ConfigService.instance.readWebdavConfig(); @@ -129,6 +146,9 @@ export default class WebDAVConfig extends Command { if (deleteFilesPermanently !== undefined) { webdavConfig['deleteFilesPermanently'] = deleteFilesPermanently; } + if (hyperBackupMode !== undefined) { + webdavConfig['hyperBackupMode'] = hyperBackupMode; + } await ConfigService.instance.saveWebdavConfig(webdavConfig); diff --git a/src/constants/configs.ts b/src/constants/configs.ts index 9ee38c6d..eac7c0cf 100644 --- a/src/constants/configs.ts +++ b/src/constants/configs.ts @@ -14,3 +14,4 @@ export const WEBDAV_DEFAULT_TIMEOUT = 0; export const WEBDAV_DEFAULT_CREATE_FULL_PATH = true; export const WEBDAV_DEFAULT_CUSTOM_AUTH = false; export const WEBDAV_DEFAULT_DELETE_FILES_PERMANENTLY = false; +export const WEBDAV_DEFAULT_HYPER_BACKUP_MODE = false; diff --git a/src/services/config.service.ts b/src/services/config.service.ts index bc826c93..ac272f5d 100644 --- a/src/services/config.service.ts +++ b/src/services/config.service.ts @@ -16,6 +16,7 @@ import { WEBDAV_DEFAULT_CUSTOM_AUTH, WEBDAV_SSL_CERTS_DIR, WEBDAV_DEFAULT_DELETE_FILES_PERMANENTLY, + WEBDAV_DEFAULT_HYPER_BACKUP_MODE, } from '../constants/configs'; import { CacheService } from './cache.service'; @@ -112,6 +113,7 @@ export class ConfigService { username: configs?.username ?? '', password: configs?.password ?? '', deleteFilesPermanently: configs?.deleteFilesPermanently ?? WEBDAV_DEFAULT_DELETE_FILES_PERMANENTLY, + hyperBackupMode: configs?.hyperBackupMode ?? WEBDAV_DEFAULT_HYPER_BACKUP_MODE, }; CacheService.instance.set(CacheService.WEBDAV_CONFIG_CACHE_KEY, webdavConfig); return webdavConfig; @@ -126,6 +128,7 @@ export class ConfigService { username: '', password: '', deleteFilesPermanently: WEBDAV_DEFAULT_DELETE_FILES_PERMANENTLY, + hyperBackupMode: WEBDAV_DEFAULT_HYPER_BACKUP_MODE, }; } }; diff --git a/src/services/drive/drive-file.service.ts b/src/services/drive/drive-file.service.ts index 3d33aed8..4f411da3 100644 --- a/src/services/drive/drive-file.service.ts +++ b/src/services/drive/drive-file.service.ts @@ -31,6 +31,31 @@ export class DriveFileService { return driveFileItem; }; + public replaceFile = async (uuid: string, payload: StorageTypes.FileEntryByUuid): Promise => { + const storageClient = SdkManager.instance.getStorage(); + const driveFile = await storageClient.replaceFile(uuid, { + fileId: payload.fileId ?? '', + size: payload.size, + }); + + const driveFileItem: DriveFileItem = { + itemType: 'file', + name: driveFile.plainName ?? driveFile.name, + uuid: driveFile.uuid, + size: driveFile.size, + bucket: driveFile.bucket, + createdAt: new Date(driveFile.createdAt), + updatedAt: new Date(driveFile.updatedAt), + fileId: driveFile.fileId ?? null, + type: driveFile.type ?? null, + status: driveFile.status as DriveFileItem['status'], + folderUuid: driveFile.folderUuid, + creationTime: new Date(driveFile.creationTime ?? driveFile.createdAt), + modificationTime: new Date(driveFile.modificationTime ?? driveFile.updatedAt), + }; + return driveFileItem; + }; + private readonly createDriveFileEntry = async ( payload: StorageTypes.FileEntryByUuid, ): Promise => { diff --git a/src/services/webdav/webdav-cache.service.ts b/src/services/webdav/webdav-cache.service.ts new file mode 100644 index 00000000..18418e18 --- /dev/null +++ b/src/services/webdav/webdav-cache.service.ts @@ -0,0 +1,85 @@ +import path from 'node:path'; +import { DriveFileItem, DriveFolderItem } from '../../types/drive.types'; + +type CacheEntry = { + value: T; + expiresAt: number; +}; + +export type WebDavFolderContent = { + folders: DriveFolderItem[]; + files: DriveFileItem[]; +}; + +export class WebDavCacheService { + public static readonly instance = new WebDavCacheService(); + + private static readonly TTL_MS = 10 * 60 * 1000; + + private readonly foldersByPath = new Map>(); + private readonly filesByPath = new Map>(); + private readonly folderContentByPath = new Map>(); + + private readonly normalizeFolderPath = (folderPath: string): string => { + let normalizedPath = folderPath || '/'; + if (!normalizedPath.startsWith('/')) normalizedPath = `/${normalizedPath}`; + if (!normalizedPath.endsWith('/')) normalizedPath = `${normalizedPath}/`; + return normalizedPath; + }; + + private readonly normalizeFilePath = (filePath: string): string => { + const normalizedPath = filePath.startsWith('/') ? filePath : `/${filePath}`; + return normalizedPath.length > 1 ? normalizedPath.replace(/\/$/, '') : normalizedPath; + }; + + private readonly entry = (value: T): CacheEntry => ({ + value, + expiresAt: Date.now() + WebDavCacheService.TTL_MS, + }); + + private readonly getFresh = (map: Map>, key: string): T | undefined => { + const cached = map.get(key); + if (!cached) return; + if (cached.expiresAt < Date.now()) { + map.delete(key); + return; + } + return cached.value; + }; + + getFolder = (folderPath: string) => this.getFresh(this.foldersByPath, this.normalizeFolderPath(folderPath)); + + setFolder = (folderPath: string, folder: DriveFolderItem) => { + this.foldersByPath.set(this.normalizeFolderPath(folderPath), this.entry(folder)); + }; + + getFile = (filePath: string) => this.getFresh(this.filesByPath, this.normalizeFilePath(filePath)); + + setFile = (filePath: string, file: DriveFileItem) => { + this.filesByPath.set(this.normalizeFilePath(filePath), this.entry(file)); + }; + + getFolderContent = (folderPath: string) => + this.getFresh(this.folderContentByPath, this.normalizeFolderPath(folderPath)); + + setFolderContent = (folderPath: string, folderContent: WebDavFolderContent) => { + this.folderContentByPath.set(this.normalizeFolderPath(folderPath), this.entry(folderContent)); + }; + + invalidateResource = (resourcePath: string) => { + const filePath = this.normalizeFilePath(resourcePath); + const folderPath = this.normalizeFolderPath(resourcePath); + const parentFolderPath = this.normalizeFolderPath(path.posix.dirname(filePath)); + + this.filesByPath.delete(filePath); + this.foldersByPath.delete(folderPath); + this.folderContentByPath.delete(folderPath); + this.folderContentByPath.delete(parentFolderPath); + }; + + clear = () => { + this.foldersByPath.clear(); + this.filesByPath.clear(); + this.folderContentByPath.clear(); + }; +} diff --git a/src/services/webdav/webdav-fast-path.service.ts b/src/services/webdav/webdav-fast-path.service.ts new file mode 100644 index 00000000..ae7ed948 --- /dev/null +++ b/src/services/webdav/webdav-fast-path.service.ts @@ -0,0 +1,143 @@ +import { FileStatus } from '@internxt/sdk/dist/drive/storage/types'; +import { AuthService } from '../auth.service'; +import { ConfigService } from '../config.service'; +import { DriveFolderService } from '../drive/drive-folder.service'; +import { SdkManager } from '../sdk-manager.service'; +import { DriveUtils } from '../../utils/drive.utils'; +import { WebDavUtils } from '../../utils/webdav.utils'; +import { DriveFileItem, DriveFolderItem, DriveItem } from '../../types/drive.types'; +import { WebDavRequestedResource } from '../../types/webdav.types'; +import { WebDavCacheService, WebDavFolderContent } from './webdav-cache.service'; +import { MissingCredentialsError } from '../../types/command.types'; + +export class WebDavFastPathService { + public static readonly instance = new WebDavFastPathService(); + + private readonly folderContentFolderToItem = (folder: { + uuid: string; + plainName?: string; + name?: string; + bucket?: string; + parentUuid: string | null; + createdAt: string; + updatedAt: string; + creationTime?: string; + modificationTime?: string; + deleted?: boolean; + removed?: boolean; + }): DriveFolderItem => ({ + itemType: 'folder', + uuid: folder.uuid, + name: folder.plainName ?? folder.name ?? '', + bucket: folder.bucket ?? null, + parentUuid: folder.parentUuid, + status: folder.deleted || folder.removed ? FileStatus.TRASHED : FileStatus.EXISTS, + createdAt: new Date(folder.createdAt), + updatedAt: new Date(folder.updatedAt), + creationTime: new Date(folder.creationTime ?? folder.createdAt), + modificationTime: new Date(folder.modificationTime ?? folder.updatedAt), + }); + + isEnabled = async (): Promise => { + const { hyperBackupMode } = await ConfigService.instance.readWebdavConfig(); + if (!hyperBackupMode) return false; + + try { + const currentWorkspace = await AuthService.instance.getCurrentWorkspace(); + return !currentWorkspace; + } catch (error) { + if (error instanceof MissingCredentialsError) { + return true; + } + throw error; + } + }; + + getFolderFromPath = async (folderPath: string): Promise => { + if (!(await this.isEnabled())) return WebDavUtils.getDriveFolderFromResource(folderPath); + + const normalizedPath = WebDavUtils.normalizeFolderPath(folderPath); + const cached = WebDavCacheService.instance.getFolder(normalizedPath); + if (cached) return cached; + + try { + const storageClient = SdkManager.instance.getStorage(); + const folderMeta = await storageClient.getFolderByPath(normalizedPath); + const folder = DriveUtils.driveFolderMetaToItem(folderMeta); + if (folder.status !== FileStatus.EXISTS) return; + + WebDavCacheService.instance.setFolder(normalizedPath, folder); + return folder; + } catch { + return; + } + }; + + getFileFromPath = async (filePath: string): Promise => { + if (!(await this.isEnabled())) return WebDavUtils.getDriveFileFromResource(filePath); + + const cached = WebDavCacheService.instance.getFile(filePath); + if (cached) return cached; + + try { + const storageClient = SdkManager.instance.getStorage(); + const fileMeta = await storageClient.getFileByPath(filePath); + const file = DriveUtils.driveFileMetaToItem(fileMeta); + if (file.status !== FileStatus.EXISTS) return; + + WebDavCacheService.instance.setFile(filePath, file); + return file; + } catch { + return; + } + }; + + getItemFromResource = async (resource: WebDavRequestedResource): Promise => { + if (!(await this.isEnabled())) return WebDavUtils.getDriveItemFromResource(resource); + + if (resource.url.endsWith('/')) { + return this.getFolderFromPath(resource.url); + } + + return (await this.getFileFromPath(resource.url)) ?? (await this.getFolderFromPath(resource.url)); + }; + + getFolderContent = async (folderPath: string, folderUuid: string): Promise => { + if (!(await this.isEnabled())) { + const folderContent = await DriveFolderService.instance.getFolderContent(folderUuid); + return { + folders: folderContent.folders.map(this.folderContentFolderToItem), + files: folderContent.files.map((file) => DriveUtils.driveFileMetaToItem(file)), + }; + } + + const normalizedPath = WebDavUtils.normalizeFolderPath(folderPath); + const cached = WebDavCacheService.instance.getFolderContent(normalizedPath); + if (cached) return cached; + + const folderContent = await DriveFolderService.instance.getFolderContent(folderUuid); + const mappedFolderContent = { + folders: folderContent.folders.map(this.folderContentFolderToItem), + files: folderContent.files.map((file) => DriveUtils.driveFileMetaToItem(file)), + }; + + WebDavCacheService.instance.setFolderContent(normalizedPath, mappedFolderContent); + return mappedFolderContent; + }; + + registerCreatedFile = (filePath: string, file: DriveFileItem) => { + WebDavCacheService.instance.setFile(filePath, file); + WebDavCacheService.instance.invalidateResource(filePath); + WebDavCacheService.instance.setFile(filePath, file); + }; + + registerCreatedFolder = (folderPath: string, folder: DriveFolderItem) => { + WebDavCacheService.instance.setFolder(folderPath, folder); + WebDavCacheService.instance.invalidateResource(folderPath); + WebDavCacheService.instance.setFolder(folderPath, folder); + }; + + invalidateResource = (resourcePath: string) => { + WebDavCacheService.instance.invalidateResource(resourcePath); + }; +} diff --git a/src/services/webdav/webdav-folder.service.ts b/src/services/webdav/webdav-folder.service.ts index b49ed15d..4bad47bd 100644 --- a/src/services/webdav/webdav-folder.service.ts +++ b/src/services/webdav/webdav-folder.service.ts @@ -6,6 +6,7 @@ import { WebDavUtils } from '../../utils/webdav.utils'; import { AsyncUtils } from '../../utils/async.utils'; import { AuthService } from '../../services/auth.service'; import { DriveUtils } from '../../utils/drive.utils'; +import { WebDavFastPathService } from './webdav-fast-path.service'; export class WebDavFolderService { public static readonly instance: WebDavFolderService = new WebDavFolderService(); @@ -61,10 +62,16 @@ export class WebDavFolderService { const newPath = WebDavUtils.joinURL(accumulatedPath, currentFolderName); const folderPath = WebDavUtils.normalizeFolderPath(newPath); + const hyperBackupMode = await WebDavFastPathService.instance.isEnabled(); const folder = - (await this.getDriveFolderItemFromPath(folderPath)) ?? + (hyperBackupMode + ? await WebDavFastPathService.instance.getFolderFromPath(folderPath) + : await this.getDriveFolderItemFromPath(folderPath)) ?? (await this.createFolder({ folderName: currentFolderName, parentFolderUuid })); + if (hyperBackupMode) { + WebDavFastPathService.instance.registerCreatedFolder(folderPath, folder); + } if (rest.length === 0) { return folder; diff --git a/src/types/command.types.ts b/src/types/command.types.ts index 259b47fd..ef5db07b 100644 --- a/src/types/command.types.ts +++ b/src/types/command.types.ts @@ -56,6 +56,7 @@ export interface WebdavConfig { username: string; password: string; deleteFilesPermanently: boolean; + hyperBackupMode: boolean; } export class NotValidEmailError extends Error { diff --git a/src/webdav/handlers/DELETE.handler.ts b/src/webdav/handlers/DELETE.handler.ts index bbdba635..2c6785cc 100644 --- a/src/webdav/handlers/DELETE.handler.ts +++ b/src/webdav/handlers/DELETE.handler.ts @@ -4,13 +4,14 @@ import { WebDavMethodHandler } from '../../types/webdav.types'; import { WebDavUtils } from '../../utils/webdav.utils'; import { webdavLogger } from '../../utils/logger.utils'; import { NotFoundError } from '../../utils/errors.utils'; +import { WebDavFastPathService } from '../../services/webdav/webdav-fast-path.service'; export class DELETERequestHandler implements WebDavMethodHandler { handle = async (req: Request, res: Response) => { const resource = await WebDavUtils.getRequestedResource(req.url); webdavLogger.info(`[DELETE] Request received for item at ${resource.url}`); - const driveItem = await WebDavUtils.getDriveItemFromResource(resource); + const driveItem = await WebDavFastPathService.instance.getItemFromResource(resource); if (!driveItem) { throw new NotFoundError(`Resource not found on Internxt Drive at ${resource.url}`); @@ -18,6 +19,7 @@ export class DELETERequestHandler implements WebDavMethodHandler { await WebDavUtils.deleteOrTrashItem(driveItem); await DriveItemRepository.instance.delete([driveItem.uuid]); + WebDavFastPathService.instance.invalidateResource(resource.url); res.status(204).send(); }; diff --git a/src/webdav/handlers/GET.handler.ts b/src/webdav/handlers/GET.handler.ts index fddd7063..2221a46e 100644 --- a/src/webdav/handlers/GET.handler.ts +++ b/src/webdav/handlers/GET.handler.ts @@ -7,13 +7,16 @@ import { webdavLogger } from '../../utils/logger.utils'; import { NetworkUtils } from '../../utils/network.utils'; import { NotValidFileIdError } from '../../types/command.types'; import { CLIUtils } from '../../utils/cli.utils'; +import { WebDavFastPathService } from '../../services/webdav/webdav-fast-path.service'; export class GETRequestHandler implements WebDavMethodHandler { handle = async (req: Request, res: Response) => { const resource = await WebDavUtils.getRequestedResource(req.url); webdavLogger.info(`[GET] Request received item at ${resource.url}`); - const driveFile = await WebDavUtils.getDriveFileFromResource(resource.url); + const metadataLookupTimer = CLIUtils.timer(); + const driveFile = await WebDavFastPathService.instance.getFileFromPath(resource.url); + const metadataLookupTime = metadataLookupTimer.stop(); if (!driveFile) { throw new NotFoundError( @@ -21,10 +24,11 @@ export class GETRequestHandler implements WebDavMethodHandler { ); } - webdavLogger.info(`[GET] [${driveFile.uuid}] Found Drive File`); + webdavLogger.info(`[GET] [${driveFile.uuid}] Found Drive File in ${CLIUtils.formatDuration(metadataLookupTime)}`); + const authTimer = CLIUtils.timer(); const { user } = await AuthService.instance.getAuthDetails(); - webdavLogger.info(`[GET] [${driveFile.uuid}] Network ready for download`); + webdavLogger.info(`[GET] [${driveFile.uuid}] Auth details ready in ${CLIUtils.formatDuration(authTimer.stop())}`); res.header('Content-Type', 'application/octet-stream'); @@ -56,8 +60,13 @@ export class GETRequestHandler implements WebDavMethodHandler { throw new NotValidFileIdError(); } + const networkTimer = CLIUtils.timer(); const { networkFacade, bucket, mnemonic } = await CLIUtils.prepareNetwork(user); + webdavLogger.info( + `[GET] [${driveFile.uuid}] Network ready for download in ${CLIUtils.formatDuration(networkTimer.stop())}`, + ); + const prepareDownloadTimer = CLIUtils.timer(); const [executeDownload] = await networkFacade.downloadToStream( bucket, mnemonic, @@ -66,7 +75,11 @@ export class GETRequestHandler implements WebDavMethodHandler { writable, rangeOptions, ); - webdavLogger.info(`[GET] [${driveFile.uuid}] Download prepared, executing...`); + webdavLogger.info( + `[GET] [${driveFile.uuid}] Download prepared in ${CLIUtils.formatDuration( + prepareDownloadTimer.stop(), + )}, executing...`, + ); /** * If the client doesn't receive a 200 status code, the download can be aborted. @@ -75,8 +88,13 @@ export class GETRequestHandler implements WebDavMethodHandler { */ res.status(200); + const executeDownloadTimer = CLIUtils.timer(); await executeDownload; - webdavLogger.info(`[GET] [${driveFile.uuid}] ✅ Download ready, replying to client`); + webdavLogger.info( + `[GET] [${driveFile.uuid}] ✅ Download ready in ${CLIUtils.formatDuration( + executeDownloadTimer.stop(), + )}, replying to client`, + ); } else { webdavLogger.info(`[GET] [${driveFile.uuid}] File is empty, replying to client with no content`); res.header('Content-length', '0'); diff --git a/src/webdav/handlers/HEAD.handler.ts b/src/webdav/handlers/HEAD.handler.ts index 2f78a73b..d787c104 100644 --- a/src/webdav/handlers/HEAD.handler.ts +++ b/src/webdav/handlers/HEAD.handler.ts @@ -4,6 +4,7 @@ import { WebDavUtils } from '../../utils/webdav.utils'; import { webdavLogger } from '../../utils/logger.utils'; import { NetworkUtils } from '../../utils/network.utils'; import { NotFoundError } from '../../utils/errors.utils'; +import { WebDavFastPathService } from '../../services/webdav/webdav-fast-path.service'; export class HEADRequestHandler implements WebDavMethodHandler { handle = async (req: Request, res: Response) => { @@ -11,7 +12,7 @@ export class HEADRequestHandler implements WebDavMethodHandler { webdavLogger.info(`[HEAD] Request received for item at ${resource.url}`); - const driveItem = await WebDavUtils.getDriveItemFromResource(resource); + const driveItem = await WebDavFastPathService.instance.getItemFromResource(resource); if (!driveItem) { throw new NotFoundError(`Resource not found on Internxt Drive at ${resource.url}`); diff --git a/src/webdav/handlers/PROPFIND.handler.ts b/src/webdav/handlers/PROPFIND.handler.ts index 3802fae9..1bec9562 100644 --- a/src/webdav/handlers/PROPFIND.handler.ts +++ b/src/webdav/handlers/PROPFIND.handler.ts @@ -3,7 +3,6 @@ import { XMLUtils } from '../../utils/xml.utils'; import { DriveFileItem, DriveFolderItem } from '../../types/drive.types'; import { DriveItemBD } from '../../services/database/drive-item/drive-item.domain'; import { DriveItemRepository } from '../../services/database/drive-item/drive-item.repository'; -import { DriveFolderService } from '../../services/drive/drive-folder.service'; import { FormatUtils } from '../../utils/format.utils'; import { Request, Response } from 'express'; import { randomUUID } from 'node:crypto'; @@ -11,13 +10,16 @@ import mime from 'mime-types'; import { WebDavUtils } from '../../utils/webdav.utils'; import { webdavLogger } from '../../utils/logger.utils'; import { UsageService } from '../../services/usage.service'; +import { WebDavFastPathService } from '../../services/webdav/webdav-fast-path.service'; export class PROPFINDRequestHandler implements WebDavMethodHandler { + private static readonly SYNOLOGY_LOCK_KEEPALIVE_STALE_MS = 10 * 60 * 1000; + handle = async (req: Request, res: Response) => { const resource = await WebDavUtils.getRequestedResource(req.url); webdavLogger.info(`[PROPFIND] Request received for item at ${resource.url}`); - const driveItem = await WebDavUtils.getDriveItemFromResource(resource); + const driveItem = await WebDavFastPathService.instance.getItemFromResource(resource); if (!driveItem) { res.status(404).send(); @@ -90,25 +92,26 @@ export class PROPFINDRequestHandler implements WebDavMethodHandler { }; private readonly getFolderChildsXMLNode = async (relativePath: string, folderUuid: string) => { - const folderContent = await DriveFolderService.instance.getFolderContent(folderUuid); + const folderContent = await WebDavFastPathService.instance.getFolderContent(relativePath, folderUuid); + const files = await this.filterStaleSynologyLockKeepAliveFiles(relativePath, folderContent.files); const xmlNodes: object[] = []; const cachedItems: DriveItemBD[] = []; for (const folder of folderContent.folders) { - const folderRelativePath = WebDavUtils.joinURL(relativePath, folder.plainName, '/'); + const folderRelativePath = WebDavUtils.joinURL(relativePath, folder.name, '/'); xmlNodes.push( this.driveFolderItemToXMLNode( { itemType: 'folder', - name: folder.plainName, + name: folder.name, bucket: folder.bucket, - status: folder.deleted || folder.removed ? 'TRASHED' : 'EXISTS', - createdAt: new Date(folder.createdAt), - updatedAt: new Date(folder.updatedAt), - creationTime: new Date(folder.creationTime), - modificationTime: new Date(folder.modificationTime), + status: folder.status, + createdAt: folder.createdAt, + updatedAt: folder.updatedAt, + creationTime: folder.creationTime, + modificationTime: folder.modificationTime, uuid: folder.uuid, parentUuid: folder.parentUuid, }, @@ -127,17 +130,14 @@ export class PROPFINDRequestHandler implements WebDavMethodHandler { ); } - for (const file of folderContent.files) { - const fileRelativePath = WebDavUtils.joinURL( - relativePath, - file.type ? `${file.plainName}.${file.type}` : file.plainName, - ); + for (const file of files) { + const fileRelativePath = WebDavUtils.joinURL(relativePath, this.getFileDisplayName(file)); xmlNodes.push( this.driveFileItemToXMLNode( { itemType: 'file', - name: file.plainName, + name: file.name, bucket: file.bucket, fileId: file.fileId, uuid: file.uuid, @@ -145,10 +145,10 @@ export class PROPFINDRequestHandler implements WebDavMethodHandler { status: file.status, folderUuid: file.folderUuid, size: Number(file.size), - creationTime: new Date(file.creationTime), - modificationTime: new Date(file.modificationTime), - createdAt: new Date(file.createdAt), - updatedAt: new Date(file.updatedAt), + creationTime: file.creationTime, + modificationTime: file.modificationTime, + createdAt: file.createdAt, + updatedAt: file.updatedAt, }, fileRelativePath, ), @@ -159,7 +159,7 @@ export class PROPFINDRequestHandler implements WebDavMethodHandler { uuid: file.uuid, path: fileRelativePath, type: 'file', - createdAt: new Date(file.createdAt), + createdAt: file.createdAt, updatedAt: new Date(), }), ); @@ -172,6 +172,64 @@ export class PROPFINDRequestHandler implements WebDavMethodHandler { return xmlNodes; }; + private readonly filterStaleSynologyLockKeepAliveFiles = async ( + relativePath: string, + files: DriveFileItem[], + ): Promise => { + if (!(await WebDavFastPathService.instance.isEnabled())) return files; + if (!/\/Control\/lock\/?$/.test(relativePath)) return files; + + const lockFiles = files.filter((file) => this.isSynologyLockKeepAliveFile(file)); + if (lockFiles.length === 0) return files; + + const newestLock = lockFiles.reduce((newest, file) => + this.getFileModifiedTime(file) > this.getFileModifiedTime(newest) ? file : newest, + ); + const now = Date.now(); + const staleLocks = new Set( + lockFiles + .filter((file) => { + const isOlderDuplicate = file.uuid !== newestLock.uuid; + const isExpired = + now - this.getFileModifiedTime(file) > PROPFINDRequestHandler.SYNOLOGY_LOCK_KEEPALIVE_STALE_MS; + return isOlderDuplicate || isExpired; + }) + .map((file) => file.uuid), + ); + + if (staleLocks.size === 0) return files; + + for (const file of files) { + if (!staleLocks.has(file.uuid)) continue; + + const lockPath = WebDavUtils.joinURL(relativePath, this.getFileDisplayName(file)); + webdavLogger.warn(`[PROPFIND] Hiding and deleting stale Synology lock keepalive file at ${lockPath}`); + void WebDavUtils.deleteOrTrashItem(file) + .then(() => WebDavFastPathService.instance.invalidateResource(lockPath)) + .catch((error) => + webdavLogger.warn( + `[PROPFIND] Failed to delete stale Synology lock keepalive file at ${lockPath}: ${ + error instanceof Error ? error.message : String(error) + }`, + ), + ); + } + + return files.filter((file) => !staleLocks.has(file.uuid)); + }; + + private readonly isSynologyLockKeepAliveFile = (file: DriveFileItem): boolean => { + return file.size === 0 && this.getFileDisplayName(file).startsWith('lock_keep_alive.@writer_version_'); + }; + + private readonly getFileDisplayName = (file: DriveFileItem): string => { + return file.type ? `${file.name}.${file.type}` : file.name; + }; + + private readonly getFileModifiedTime = (file: DriveFileItem): number => { + return (file.modificationTime ?? file.updatedAt ?? file.createdAt).getTime(); + }; + private readonly driveFolderRootStatsToXMLNode = async ( driveFolderItem: DriveFolderItem, relativePath: string, diff --git a/src/webdav/handlers/PUT.handler.ts b/src/webdav/handlers/PUT.handler.ts index c8899801..5dd58b25 100644 --- a/src/webdav/handlers/PUT.handler.ts +++ b/src/webdav/handlers/PUT.handler.ts @@ -12,8 +12,19 @@ import { WebDavFolderService } from '../../services/webdav/webdav-folder.service import { ThumbnailService } from '../../services/thumbnail.service'; import { FormatUtils } from '../../utils/format.utils'; import { UploadUtils } from '../../utils/upload.utils'; +import { WebDavFastPathService } from '../../services/webdav/webdav-fast-path.service'; export class PUTRequestHandler implements WebDavMethodHandler { + private readonly isHyperBackupZeroBytePlaceholder = ( + resource: Awaited>, + contentLength: number, + hyperBackupMode: boolean, + ): boolean => { + if (!hyperBackupMode || contentLength !== 0) return false; + + return /\/Pool\/.+\/[^/]+\.(bucket|index)\.\d+$/.test(resource.url); + }; + handle = async (req: Request, res: Response) => { let contentLength = Number(req.headers['content-length']); if (!contentLength || Number.isNaN(contentLength) || contentLength <= 0) { @@ -23,6 +34,16 @@ export class PUTRequestHandler implements WebDavMethodHandler { await UploadUtils.checkUploadSizeLimits(contentLength); const resource = await WebDavUtils.getRequestedResource(req.url); + const hyperBackupMode = await WebDavFastPathService.instance.isEnabled(); + + if (this.isHyperBackupZeroBytePlaceholder(resource, contentLength, hyperBackupMode)) { + webdavLogger.info( + `[PUT] Hyper Backup zero-byte placeholder acknowledged without Internxt upload: ${resource.url}`, + ); + res.status(201).send(); + return; + } + webdavLogger.info(`[PUT] Request received for file at ${resource.url}`); webdavLogger.info( `[PUT] Uploading '${resource.name}' (${FormatUtils.humanFileSize(contentLength)}) to '${resource.parentPath}'`, @@ -35,14 +56,16 @@ export class PUTRequestHandler implements WebDavMethodHandler { }; const parentDriveFolderItem = - (await WebDavFolderService.instance.getDriveFolderItemFromPath(resource.parentPath)) ?? + (hyperBackupMode + ? await WebDavFastPathService.instance.getFolderFromPath(resource.parentPath) + : await WebDavFolderService.instance.getDriveFolderItemFromPath(resource.parentPath)) ?? (await WebDavFolderService.instance.createParentPathOrThrow(resource.parentPath)); let isReplacement = false; // If the file already exists, the WebDAV specification states that 'PUT /…/file' should replace it. // http://www.webdav.org/specs/rfc4918.html#put-resources - const driveFileItem = await WebDavUtils.getDriveItemFromResource(resource); + const driveFileItem = await WebDavFastPathService.instance.getItemFromResource(resource); if (driveFileItem && driveFileItem.status === 'EXISTS') { if (driveFileItem.itemType === 'folder') { webdavLogger.info('[PUT] ❌ A folder exists on the cloud with the same name.'); @@ -52,11 +75,13 @@ export class PUTRequestHandler implements WebDavMethodHandler { webdavLogger.info( `[PUT] File '${resource.name}' already exists in '${resource.path.dir}', it will be replaced...`, ); - try { - await WebDavUtils.deleteOrTrashItem(driveFileItem); - await DriveItemRepository.instance.delete([driveFileItem.uuid]); - } catch { - //noop + if (!hyperBackupMode) { + try { + await WebDavUtils.deleteOrTrashItem(driveFileItem); + await DriveItemRepository.instance.delete([driveFileItem.uuid]); + } catch { + // noop + } } } @@ -104,7 +129,7 @@ export class PUTRequestHandler implements WebDavMethodHandler { } const driveTimer = CLIUtils.timer(); - const file = await DriveFileService.instance.createFile({ + const filePayload = { plainName: resource.path.name, type: fileType, size: contentLength, @@ -112,8 +137,31 @@ export class PUTRequestHandler implements WebDavMethodHandler { fileId, bucket, encryptVersion: EncryptionVersion.Aes03, - }); + }; + + let file; + if (hyperBackupMode && driveFileItem?.itemType === 'file' && contentLength > 0) { + try { + file = await DriveFileService.instance.replaceFile(driveFileItem.uuid, filePayload); + } catch (error) { + webdavLogger.warn( + `[PUT] Fast replace failed for '${resource.url}', falling back to delete and create: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + await WebDavUtils.deleteOrTrashItem(driveFileItem); + await DriveItemRepository.instance.delete([driveFileItem.uuid]); + file = await DriveFileService.instance.createFile(filePayload); + } + } else { + if (hyperBackupMode && driveFileItem?.itemType === 'file') { + await WebDavUtils.deleteOrTrashItem(driveFileItem); + await DriveItemRepository.instance.delete([driveFileItem.uuid]); + } + file = await DriveFileService.instance.createFile(filePayload); + } timings.driveUpload = driveTimer.stop(); + WebDavFastPathService.instance.registerCreatedFile(resource.url, file); await DriveItemRepository.instance.createOrUpdate([ { diff --git a/test/fixtures/webdav.fixture.ts b/test/fixtures/webdav.fixture.ts index 9d4c99e8..384f262b 100644 --- a/test/fixtures/webdav.fixture.ts +++ b/test/fixtures/webdav.fixture.ts @@ -9,7 +9,11 @@ import { UserFixture } from './auth.fixture'; import { NetworkFacade } from '../../src/services/network/network-facade.service'; import { NetworkOptions } from '../../src/types/network.types'; import { WebdavConfig } from '../../src/types/command.types'; -import { WEBDAV_DEFAULT_CUSTOM_AUTH, WEBDAV_DEFAULT_DELETE_FILES_PERMANENTLY } from '../../src/constants/configs'; +import { + WEBDAV_DEFAULT_CUSTOM_AUTH, + WEBDAV_DEFAULT_DELETE_FILES_PERMANENTLY, + WEBDAV_DEFAULT_HYPER_BACKUP_MODE, +} from '../../src/constants/configs'; import { randomInt } from 'node:crypto'; export const createWebDavRequestFixture = (request: T): T & Request => { @@ -107,6 +111,7 @@ export const getWebdavConfigMock = (attributes?: Partial): WebdavC username: '', password: '', deleteFilesPermanently: WEBDAV_DEFAULT_DELETE_FILES_PERMANENTLY, + hyperBackupMode: WEBDAV_DEFAULT_HYPER_BACKUP_MODE, }; return { ...config, ...attributes }; }; diff --git a/test/services/config.service.test.ts b/test/services/config.service.test.ts index b95e389d..b91ac1e7 100644 --- a/test/services/config.service.test.ts +++ b/test/services/config.service.test.ts @@ -17,6 +17,7 @@ import { WEBDAV_DEFAULT_CUSTOM_AUTH, WEBDAV_SSL_CERTS_DIR, WEBDAV_DEFAULT_DELETE_FILES_PERMANENTLY, + WEBDAV_DEFAULT_HYPER_BACKUP_MODE, } from '../../src/constants/configs'; import { getWebdavConfigMock } from '../fixtures/webdav.fixture'; import { CacheService } from '../../src/services/cache.service'; @@ -34,6 +35,7 @@ describe('Config service', () => { username: '', password: '', deleteFilesPermanently: WEBDAV_DEFAULT_DELETE_FILES_PERMANENTLY, + hyperBackupMode: WEBDAV_DEFAULT_HYPER_BACKUP_MODE, }; beforeEach(() => {