From 27d728ec4825d072f0aac39e430c195f8ff43e5d Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Sun, 21 Jun 2026 19:48:05 +0300 Subject: [PATCH 1/2] feat(storage): rewrite GCS provider to IStorageProvider parity Replace the deprecated Google Cloud Storage provider with a full implementation aligned to AWS S3 and Azure, including UBLA-safe IAM public access, v4 signed URLs, ADC/inline JSON auth, and fixes for deleteContainer, deleteFolder, exists, getSignedUrl, and folder markers. --- modules/storage/src/config/config.ts | 4 + .../storage/src/interfaces/StorageConfig.ts | 1 + modules/storage/src/providers/google/index.ts | 296 ++++++++++++------ 3 files changed, 198 insertions(+), 103 deletions(-) diff --git a/modules/storage/src/config/config.ts b/modules/storage/src/config/config.ts index 141eb6877..835771c85 100644 --- a/modules/storage/src/config/config.ts +++ b/modules/storage/src/config/config.ts @@ -26,6 +26,10 @@ export default { format: 'String', default: '', }, + serviceAccountKeyJson: { + format: 'String', + default: '', + }, }, azure: { connectionString: { format: 'String', default: '' }, diff --git a/modules/storage/src/interfaces/StorageConfig.ts b/modules/storage/src/interfaces/StorageConfig.ts index 4300e2cdf..d57820b1d 100644 --- a/modules/storage/src/interfaces/StorageConfig.ts +++ b/modules/storage/src/interfaces/StorageConfig.ts @@ -5,6 +5,7 @@ export interface StorageConfig { */ google: { serviceAccountKeyPath: string; + serviceAccountKeyJson: string; }; aws: { accessKeyId: string; diff --git a/modules/storage/src/providers/google/index.ts b/modules/storage/src/providers/google/index.ts index 5435621fc..ad0b9af5b 100644 --- a/modules/storage/src/providers/google/index.ts +++ b/modules/storage/src/providers/google/index.ts @@ -1,34 +1,76 @@ import { IStorageProvider, StorageConfig, UrlOptions } from '../../interfaces/index.js'; -import { Storage } from '@google-cloud/storage'; +import { Bucket, Storage } from '@google-cloud/storage'; import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; import { SIGNED_URL_EXPIRY_DATE } from '../../constants/expiry.js'; import { constructDispositionHeader } from '../../utils/index.js'; +import fs from 'fs'; + +type GoogleServiceAccountKey = { + project_id?: string; + client_email?: string; + private_key?: string; +}; + +const OBJECT_VIEWER_ROLE = 'roles/storage.objectViewer'; +const FOLDER_MARKER_SUFFIX = '.keep.txt'; -/** - * WARNING: DO NOT USE THIS, IT NEEDS A REWRITE - * @Deprecated - */ export class GoogleCloudStorage implements IStorageProvider { - _storage: Storage; - _activeBucket: string = ''; + private readonly _storage: Storage; + private _activeBucket: string = ''; constructor(options: StorageConfig) { - this._storage = new Storage({ - keyFilename: options.google.serviceAccountKeyPath, - }); + this._storage = createGoogleStorageClient(options); + } + + container(name: string): IStorageProvider { + this._activeBucket = name; + return this; } - deleteContainer(name: string): Promise { - throw new Error('Method not implemented.'); + async store( + fileName: string, + data: Buffer | string, + _isPublic: boolean = false, + ): Promise { + await this.bucket().file(fileName).save(data); + return true; } - deleteFolder(name: string): Promise { - throw new Error('Method not implemented.'); + async get(fileName: string, downloadPath?: string): Promise { + const [contents] = await this.bucket().file(fileName).download(); + if (downloadPath) { + fs.writeFileSync(downloadPath, new Uint8Array(contents)); + } + return contents; + } + + async createFolder(name: string): Promise { + const exists = await this.folderExists(name); + if (exists) return true; + + await this.bucket() + .file(this.folderMarkerKey(name)) + .save(Buffer.from('DO NOT DELETE')); + ConduitGrpcSdk.Metrics?.increment('folders_total'); + return true; + } + + async folderExists(name: string): Promise { + const [bucketExists] = await this.bucket().exists(); + if (!bucketExists) return false; + + const [markerExists] = await this.bucket().file(this.folderMarkerKey(name)).exists(); + return markerExists; } async createContainer(name: string, isPublic?: boolean): Promise { - // Creates the new bucket - await this._storage.createBucket(name); + await this._storage.createBucket(name, { + iamConfiguration: { + uniformBucketLevelAccess: { + enabled: true, + }, + }, + }); this._activeBucket = name; if (isPublic) { await this.setContainerPublicAccess(name, true); @@ -42,136 +84,184 @@ export class GoogleCloudStorage implements IStorageProvider { isPublic: boolean, ): Promise { const bucket = this._storage.bucket(name); + await ensureUniformBucketLevelAccess(bucket); + + const [policy] = await bucket.iam.getPolicy({ requestedPolicyVersion: 3 }); + if (isPublic) { - // Make all objects in the bucket publicly readable - await bucket.makePublic(); + const viewerBinding = policy.bindings.find( + binding => binding.role === OBJECT_VIEWER_ROLE, + ); + if (viewerBinding) { + if (!viewerBinding.members.includes('allUsers')) { + viewerBinding.members.push('allUsers'); + } + } else { + policy.bindings.push({ + role: OBJECT_VIEWER_ROLE, + members: ['allUsers'], + }); + } } else { - // Remove public access - await bucket.makePrivate({ includeFiles: true }); + policy.bindings = policy.bindings + .map(binding => { + if (binding.role !== OBJECT_VIEWER_ROLE) return binding; + return { + ...binding, + members: binding.members.filter(member => member !== 'allUsers'), + }; + }) + .filter(binding => binding.members.length > 0); } - return true; - } - container(name: string): IStorageProvider { - this._activeBucket = name; - return this; + await bucket.iam.setPolicy(policy); + return true; } async containerExists(name: string): Promise { - const exists = await this._storage.bucket(name).exists(); - return exists[0]; + const [exists] = await this._storage.bucket(name).exists(); + return exists; } - /** - * Used to create a new folder - * @param name For the folder - */ - async createFolder(name: string): Promise { - const bucket = await this._storage.bucket(this._activeBucket); - let exists = await bucket.exists(); - if (!exists[0]) { - await bucket.create(); - } - - exists = await bucket.file(name + '/keep.txt').exists(); - if (exists[0]) { - return true; - } - await bucket.file(name + '/keep.txt').save(Buffer.from('DO NOT DELETE')); - ConduitGrpcSdk.Metrics?.increment('folders_total'); + async deleteContainer(name: string): Promise { + const bucket = this._storage.bucket(name); + await bucket.deleteFiles({ force: true }); + await bucket.delete(); + ConduitGrpcSdk.Metrics?.decrement('containers_total'); return true; } - async folderExists(name: string): Promise { - const bucket = await this._storage.bucket(this._activeBucket); - let exists = await bucket.exists(); - if (!exists[0]) { - return false; - } + async deleteFolder(name: string): Promise { + const exists = await this.folderExists(name); + if (!exists) return false; - exists = await bucket.file(name + '/keep.txt').exists(); + ConduitGrpcSdk.Logger.log('Getting files list...'); + const files = await this.listFiles(name); - return exists[0]; + ConduitGrpcSdk.Logger.log('Deleting files...'); + let deleted = 0; + for (const file of files) { + deleted++; + await file.delete({ ignoreNotFound: true }); + ConduitGrpcSdk.Logger.log(file.name); + } + ConduitGrpcSdk.Logger.log(`${deleted} files deleted.`); + ConduitGrpcSdk.Metrics?.decrement('folders_total'); + return true; } async delete(fileName: string): Promise { - await this._storage.bucket(this._activeBucket).file(fileName).delete(); + await this.bucket().file(fileName).delete({ ignoreNotFound: true }); return true; } async exists(fileName: string): Promise { - await this._storage.bucket(this._activeBucket).file(fileName).exists(); - return true; - } - - async get(fileName: string, downloadPath?: string): Promise { - let promise; - if (downloadPath) { - promise = this._storage.bucket(this._activeBucket).file(fileName).download({ - destination: downloadPath, - }); - } else { - promise = this._storage.bucket(this._activeBucket).file(fileName).download(); - } + const [bucketExists] = await this.bucket().exists(); + if (!bucketExists) return false; - return promise.then((r: any) => { - if (r.data && r.data[0]) { - return r.data[0]; - } - return r; - }); + const [fileExists] = await this.bucket().file(fileName).exists(); + return fileExists; } - async getSignedUrl(fileName: string, options?: UrlOptions): Promise { - this._storage - .bucket(this._activeBucket) + async getSignedUrl(fileName: string, options?: UrlOptions): Promise { + const [url] = await this.bucket() .file(fileName) .getSignedUrl({ + version: 'v4', action: 'read', expires: SIGNED_URL_EXPIRY_DATE(), responseDisposition: constructDispositionHeader(fileName, options), - }) - .then((r: any) => { - if (r.data && r.data[0]) { - return r.data[0]; - } - return r; }); + return url; } async getPublicUrl( fileName: string, - _containerIsPublic?: boolean, + containerIsPublic?: boolean, ): Promise { - // GCS uses bucket/file-level ACLs, publicUrl() works regardless - return this._storage.bucket(this._activeBucket).file(fileName).publicUrl(); - } - - async store( - fileName: string, - data: any, - isPublic: boolean = false, - ): Promise { - await this._storage.bucket(this._activeBucket).file(fileName).save(data); - if (isPublic) { - await this._storage.bucket(this._activeBucket).file(fileName).makePublic(); + if (containerIsPublic) { + return this.bucket().file(fileName).publicUrl(); } - return true; + + const [url] = await this.bucket() + .file(fileName) + .getSignedUrl({ + version: 'v4', + action: 'read', + expires: new Date(new Date().setFullYear(new Date().getFullYear() + 99)), + }); + return url; } getUploadUrl(fileName: string): Promise { - return this._storage - .bucket(this._activeBucket) + return this.bucket() .file(fileName) .getSignedUrl({ + version: 'v4', action: 'write', expires: SIGNED_URL_EXPIRY_DATE(), }) - .then((r: any) => { - if (r.data && r.data[0]) { - return r.data[0]; - } - return r; - }); + .then(([url]) => url); + } + + private bucket(): Bucket { + return this._storage.bucket(this._activeBucket); + } + + private folderMarkerKey(name: string): string { + return `${name}${FOLDER_MARKER_SUFFIX}`; + } + + private async listFiles(folderName: string) { + const [files] = await this.bucket().getFiles({ prefix: folderName }); + return files; + } +} + +function createGoogleStorageClient(options: StorageConfig): Storage { + const { serviceAccountKeyPath, serviceAccountKeyJson } = options.google; + + if (serviceAccountKeyJson) { + const credentials = parseServiceAccountKeyJson(serviceAccountKeyJson); + return new Storage({ + projectId: credentials.project_id, + credentials: { + client_email: credentials.client_email, + private_key: credentials.private_key, + }, + }); } + + if (serviceAccountKeyPath) { + return new Storage({ keyFilename: serviceAccountKeyPath }); + } + + return new Storage(); +} + +function parseServiceAccountKeyJson(raw: string): GoogleServiceAccountKey { + try { + const credentials = JSON.parse(raw) as GoogleServiceAccountKey; + if (!credentials.client_email || !credentials.private_key) { + throw new Error('Missing client_email or private_key'); + } + return credentials; + } catch (error) { + throw new Error(`Invalid google.serviceAccountKeyJson: ${(error as Error).message}`); + } +} + +async function ensureUniformBucketLevelAccess(bucket: Bucket): Promise { + const [metadata] = await bucket.getMetadata(); + if (metadata.iamConfiguration?.uniformBucketLevelAccess?.enabled) { + return; + } + + await bucket.setMetadata({ + iamConfiguration: { + uniformBucketLevelAccess: { + enabled: true, + }, + }, + }); } From 8d8c2ffcebcb494eab45898afdc3385eef3fdb7a Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Thu, 25 Jun 2026 13:36:36 +0300 Subject: [PATCH 2/2] feat(storage): add public file API URIs Store stable Conduit-relative URIs for public files so private-container public files resolve through Storage without unsupported long-lived provider signatures. --- modules/storage/README.mdx | 7 + modules/storage/src/Storage.ts | 6 + .../src/adapter/StorageParamAdapter.ts | 16 +- modules/storage/src/admin/adminFile.ts | 6 +- modules/storage/src/handlers/file.ts | 6 +- .../src/interfaces/IStorageProvider.ts | 6 - .../src/migrations/fileUriMigration.ts | 40 +++ modules/storage/src/models/File.schema.ts | 4 +- modules/storage/src/providers/aliyun/index.ts | 16 +- modules/storage/src/providers/aws/index.ts | 6 +- modules/storage/src/providers/azure/index.ts | 26 +- modules/storage/src/providers/google/index.ts | 20 +- modules/storage/src/storage.proto | 2 + modules/storage/src/utils/index.ts | 239 ++++++++++-------- 14 files changed, 234 insertions(+), 166 deletions(-) create mode 100644 modules/storage/src/migrations/fileUriMigration.ts diff --git a/modules/storage/README.mdx b/modules/storage/README.mdx index 601607289..3ddcb79a4 100644 --- a/modules/storage/README.mdx +++ b/modules/storage/README.mdx @@ -11,6 +11,13 @@ Documents may have complex structures and subcollections and can be stored local - AWS S3 (or any other S3-compatible provider) - Alibaba Cloud (Aliyun) +## Public files + +Public files expose a stable Conduit-relative `uri` in the form `/storage/getFileUrl/:id`. +If both the container and file are public, `url` contains the direct provider/CDN URL. +If the container is private and the file is public, `url` is left empty and Conduit signs a provider URL on demand through `uri`. +Private files are not allowed in public containers. + ## Requirements ⚡ - [Database](../database) module diff --git a/modules/storage/src/Storage.ts b/modules/storage/src/Storage.ts index edcc5be44..39a9fe2c9 100644 --- a/modules/storage/src/Storage.ts +++ b/modules/storage/src/Storage.ts @@ -17,6 +17,7 @@ import { status } from '@grpc/grpc-js'; import { isEmpty, isNil } from 'lodash-es'; import { runMigrations } from './migrations/index.js'; import { migratePublicContainers } from './migrations/publicContainerMigration.js'; +import { migrateFileUriReferences } from './migrations/fileUriMigration.js'; import { CdnConfiguration, cdnConfigsAreEqual, @@ -82,6 +83,7 @@ export default class Storage extends ManagedModule { private enableAuthRoutes: boolean = false; private storageParamAdapter: StorageParamAdapter; private publicContainerMigrationRan: boolean = false; + private fileUriMigrationRan: boolean = false; private previousCdnConfig: CdnConfiguration = {}; private _storageAuthzResourceDispose: (() => void) | null = null; private refreshAppRoutesTimeout: NodeJS.Timeout | null = null; @@ -253,6 +255,10 @@ export default class Storage extends ManagedModule { this.publicContainerMigrationRan = true; await migratePublicContainers(this.grpcSdk, this.storageProvider); } + if (!this.fileUriMigrationRan) { + this.fileUriMigrationRan = true; + await migrateFileUriReferences(); + } // Detect CDN config changes and run migration if needed (order-independent comparison) const currentCdnConfig = (ConfigController.getInstance().config.cdnConfiguration ?? {}) as CdnConfiguration; diff --git a/modules/storage/src/adapter/StorageParamAdapter.ts b/modules/storage/src/adapter/StorageParamAdapter.ts index c604e5a2a..a00cf44b8 100644 --- a/modules/storage/src/adapter/StorageParamAdapter.ts +++ b/modules/storage/src/adapter/StorageParamAdapter.ts @@ -5,18 +5,22 @@ export class StorageParamAdapter { constructor() {} getFileResponse(response: UnparsedRouterResponse): FileResponse { + const file = response as Indexable; return { - id: (response as Indexable)._id, - url: (response as Indexable).url, - name: (response as Indexable).name, + id: file._id, + url: file.url ?? '', + uri: file.uri ?? '', + name: file.name, }; } getFileByUrlResponse(response: UnparsedRouterResponse): FileByUrlResponse { + const file = (response as Indexable).file as Indexable; return { - id: (response as Indexable).file._id, - fileUrl: (response as Indexable).file.url, - name: (response as Indexable).file.name, + id: file._id, + fileUrl: file.url ?? file.uri ?? '', + uri: file.uri ?? '', + name: file.name, uploadUrl: (response as Indexable).url, }; } diff --git a/modules/storage/src/admin/adminFile.ts b/modules/storage/src/admin/adminFile.ts index c69619b31..5284aed6f 100644 --- a/modules/storage/src/admin/adminFile.ts +++ b/modules/storage/src/admin/adminFile.ts @@ -17,6 +17,7 @@ import { applyCdnHost, deepPathHandler, normalizeFolderPath, + resolvePublicFileAccessUrl, storeNewFile, validateName, } from '../utils/index.js'; @@ -204,10 +205,11 @@ export class AdminFileHandlers { throw new GrpcError(status.NOT_FOUND, 'File does not exist'); } if (found.isPublic) { + const url = await resolvePublicFileAccessUrl(this.storageProvider, found); if (!call.request.params.redirect) { - return { result: found.url }; + return { result: url }; } - return { redirect: found.url }; + return { redirect: url }; } const options: UrlOptions = { download: call.request.params.download ?? false, diff --git a/modules/storage/src/handlers/file.ts b/modules/storage/src/handlers/file.ts index 277ed4a4e..4a6c41883 100644 --- a/modules/storage/src/handlers/file.ts +++ b/modules/storage/src/handlers/file.ts @@ -18,6 +18,7 @@ import { applyCdnHost, deepPathHandler, normalizeFolderPath, + resolvePublicFileAccessUrl, storeNewFile, validateName, } from '../utils/index.js'; @@ -270,10 +271,11 @@ export class FileHandlers { throw new GrpcError(status.NOT_FOUND, 'File does not exist'); } if (found.isPublic) { + const url = await resolvePublicFileAccessUrl(this.storageProvider, found); if (!call.request.params.redirect) { - return { result: found.url }; + return { result: url }; } - return { redirect: found.url }; + return { redirect: url }; } await this.fileAccessCheck('read', call.request, found); const options: UrlOptions = { diff --git a/modules/storage/src/interfaces/IStorageProvider.ts b/modules/storage/src/interfaces/IStorageProvider.ts index e21523954..e448447ef 100644 --- a/modules/storage/src/interfaces/IStorageProvider.ts +++ b/modules/storage/src/interfaces/IStorageProvider.ts @@ -49,12 +49,6 @@ export interface IStorageProvider { getSignedUrl(fileName: string, options?: UrlOptions): Promise; - /** - * Gets a publicly accessible URL for a file. - * @param fileName The file name/path - * @param containerIsPublic Whether the container is publicly accessible. - * If true, returns a plain URL. If false, returns a long-lived signed URL. - */ getPublicUrl(fileName: string, containerIsPublic?: boolean): Promise; getUploadUrl(fileName: string): Promise; diff --git a/modules/storage/src/migrations/fileUriMigration.ts b/modules/storage/src/migrations/fileUriMigration.ts new file mode 100644 index 000000000..9dde15984 --- /dev/null +++ b/modules/storage/src/migrations/fileUriMigration.ts @@ -0,0 +1,40 @@ +import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk'; +import { buildFileUri } from '../utils/index.js'; +import { _StorageContainer, File } from '../models/index.js'; + +export async function migrateFileUriReferences(): Promise { + const logger = ConduitGrpcSdk.Logger; + + try { + const publicFiles = await File.getInstance().findMany({ isPublic: true }); + if (publicFiles.length === 0) { + logger.log('No public files found for URI migration'); + return; + } + + const containers = await _StorageContainer.getInstance().findMany({}); + const containerIsPublic = new Map( + containers.map(container => [container.name, container.isPublic ?? false]), + ); + + let updated = 0; + for (const file of publicFiles) { + const isContainerPublic = containerIsPublic.get(file.container) ?? false; + const update: Record = { + uri: buildFileUri(file._id), + }; + + if (!isContainerPublic) { + update.url = ''; + update.sourceUrl = ''; + } + + await File.getInstance().findByIdAndUpdate(file._id, update); + updated++; + } + + logger.log(`File URI migration completed for ${updated} public file(s)`); + } catch (error) { + logger.error(`File URI migration failed: ${(error as Error).message}`); + } +} diff --git a/modules/storage/src/models/File.schema.ts b/modules/storage/src/models/File.schema.ts index 714b26d46..1112c5c01 100644 --- a/modules/storage/src/models/File.schema.ts +++ b/modules/storage/src/models/File.schema.ts @@ -28,6 +28,7 @@ const schema: ConduitModel = { default: false, }, url: TYPE.String, + uri: TYPE.String, sourceUrl: TYPE.String, mimeType: TYPE.String, createdAt: TYPE.Date, @@ -56,7 +57,8 @@ export class File extends ConduitActiveSchema { container!: string; size!: number; isPublic?: boolean; - url!: string; + url?: string; + uri?: string; sourceUrl?: string; mimeType!: string; createdAt!: Date; diff --git a/modules/storage/src/providers/aliyun/index.ts b/modules/storage/src/providers/aliyun/index.ts index 319883a6b..a16676e66 100644 --- a/modules/storage/src/providers/aliyun/index.ts +++ b/modules/storage/src/providers/aliyun/index.ts @@ -157,15 +157,13 @@ export class AliyunStorage implements IStorageProvider { }); } - async getPublicUrl(fileName: string, containerIsPublic?: boolean): Promise { - if (containerIsPublic) { - return this._ossClient.getObjectUrl(fileName); + async getPublicUrl( + fileName: string, + containerIsPublic?: boolean, + ): Promise { + if (!containerIsPublic) { + return new Error('Public URL is only available for files in public containers'); } - - // For private containers with public files, generate long-lived signed URL (100 years) - return this._ossClient.signatureUrl(fileName, { - expires: 3600 * 24 * 365 * 100, - method: 'GET', - }); + return this._ossClient.getObjectUrl(fileName); } } diff --git a/modules/storage/src/providers/aws/index.ts b/modules/storage/src/providers/aws/index.ts index b6919e511..9803f2653 100644 --- a/modules/storage/src/providers/aws/index.ts +++ b/modules/storage/src/providers/aws/index.ts @@ -267,8 +267,10 @@ export class AWSS3Storage implements IStorageProvider { }); } - async getPublicUrl(fileName: string, _containerIsPublic?: boolean) { - // AWS uses bucket-level ACLs, so public URL format is the same regardless + async getPublicUrl(fileName: string, containerIsPublic?: boolean) { + if (!containerIsPublic) { + return new Error('Public URL is only available for files in public containers'); + } const config: Config['aws'] = ConfigController.getInstance().config.aws; if (config.endpoint !== '') { // check if endpoint contains http/https or not diff --git a/modules/storage/src/providers/azure/index.ts b/modules/storage/src/providers/azure/index.ts index ce4decb90..2cccd2d79 100644 --- a/modules/storage/src/providers/azure/index.ts +++ b/modules/storage/src/providers/azure/index.ts @@ -135,27 +135,17 @@ export class AzureStorage implements IStorageProvider { return this.blobClient(fileName).generateSasUrl(sasOptions); } - async getPublicUrl(fileName: string, containerIsPublic?: boolean): Promise { - if (containerIsPublic) { - // Return direct URL without SAS token for public containers - return `${this._storage.url}${this._activeContainer}/${fileName}`; + async getPublicUrl( + fileName: string, + containerIsPublic?: boolean, + ): Promise { + if (!containerIsPublic) { + return new Error('Public URL is only available for files in public containers'); } - // For private containers with public files, generate long-lived signed URL (99 years) - const containerClient = this._storage.getContainerClient(this._activeContainer); - const sasOptions: BlobSASSignatureValues = { - containerName: containerClient.containerName, - blobName: fileName, - expiresOn: new Date(new Date().setFullYear(new Date().getFullYear() + 99)), - permissions: BlobSASPermissions.parse('r'), - }; - return this.blobClient(fileName).generateSasUrl(sasOptions); + return `${this._storage.url}${this._activeContainer}/${fileName}`; } - async store( - fileName: string, - data: any, - isPublic: boolean = false, - ): Promise { + async store(fileName: string, data: any): Promise { await this._storage .getContainerClient(this._activeContainer) .getBlockBlobClient(fileName) diff --git a/modules/storage/src/providers/google/index.ts b/modules/storage/src/providers/google/index.ts index ad0b9af5b..246cd7c8f 100644 --- a/modules/storage/src/providers/google/index.ts +++ b/modules/storage/src/providers/google/index.ts @@ -27,11 +27,7 @@ export class GoogleCloudStorage implements IStorageProvider { return this; } - async store( - fileName: string, - data: Buffer | string, - _isPublic: boolean = false, - ): Promise { + async store(fileName: string, data: Buffer | string): Promise { await this.bucket().file(fileName).save(data); return true; } @@ -179,18 +175,10 @@ export class GoogleCloudStorage implements IStorageProvider { fileName: string, containerIsPublic?: boolean, ): Promise { - if (containerIsPublic) { - return this.bucket().file(fileName).publicUrl(); + if (!containerIsPublic) { + return new Error('Public URL is only available for files in public containers'); } - - const [url] = await this.bucket() - .file(fileName) - .getSignedUrl({ - version: 'v4', - action: 'read', - expires: new Date(new Date().setFullYear(new Date().getFullYear() + 99)), - }); - return url; + return this.bucket().file(fileName).publicUrl(); } getUploadUrl(fileName: string): Promise { diff --git a/modules/storage/src/storage.proto b/modules/storage/src/storage.proto index 74a9e1d58..35f11f4ec 100644 --- a/modules/storage/src/storage.proto +++ b/modules/storage/src/storage.proto @@ -46,6 +46,7 @@ message FileResponse { string id = 1; string url = 2; string name = 3; + string uri = 4; } message GetFileDataResponse { @@ -72,6 +73,7 @@ message FileByUrlResponse { string fileUrl = 2; string name = 3; string uploadUrl = 4; + string uri = 5; } message UpdateFileByUrlRequest { diff --git a/modules/storage/src/utils/index.ts b/modules/storage/src/utils/index.ts index 9cdd67018..4957f5cf2 100644 --- a/modules/storage/src/utils/index.ts +++ b/modules/storage/src/utils/index.ts @@ -77,6 +77,79 @@ export async function deepPathHandler( } } +export function buildFileUri(fileId: string): string { + return `/storage/getFileUrl/${fileId}`; +} + +export function getStorageFileKey(folder: string, name?: string): string { + return (folder === '/' ? '' : folder) + (name ?? ''); +} + +type FileReferences = { + sourceUrl?: string; + url?: string; + uri?: string; +}; + +export async function resolveFileReferences( + storageProvider: IStorageProvider, + params: { + container: string; + folder: string; + name?: string; + isPublic?: boolean; + fileId: string; + }, +): Promise { + if (!params.isPublic) { + return {}; + } + + const containerDoc = await _StorageContainer + .getInstance() + .findOne({ name: params.container }, { readPreference: 'primary' }); + const containerIsPublic = containerDoc?.isPublic ?? false; + const uri = buildFileUri(params.fileId); + const fileName = getStorageFileKey(params.folder, params.name); + + if (!containerIsPublic) { + return { sourceUrl: '', url: '', uri }; + } + + const publicUrlResult = await storageProvider + .container(params.container) + .getPublicUrl(fileName, true); + if (publicUrlResult instanceof Error) { + throw publicUrlResult; + } + + return { + sourceUrl: publicUrlResult, + url: applyCdnHost(publicUrlResult, params.container), + uri, + }; +} + +export async function resolvePublicFileAccessUrl( + storageProvider: IStorageProvider, + file: File, +): Promise { + if (file.url) { + return file.url; + } + + const rawUrl = await storageProvider + .container(file.container) + .getSignedUrl(getStorageFileKey(file.folder, file.name), { + download: false, + fileName: file.alias ?? file.name, + }); + if (rawUrl instanceof Error) { + throw rawUrl; + } + return applyCdnHost(rawUrl, file.container) ?? rawUrl; +} + export async function storeNewFile( storageProvider: IStorageProvider, params: IFileParams, @@ -86,31 +159,12 @@ export async function storeNewFile( await validateFilePrivacy(container, isPublic); const buffer = Buffer.from(data as string, 'base64'); const size = buffer.byteLength; - const fileName = (folder === '/' ? '' : folder) + name; + const fileName = getStorageFileKey(folder, name); await storageProvider.container(container).store(fileName, buffer, isPublic); - // Get container public status for URL generation - const containerDoc = await _StorageContainer - .getInstance() - .findOne({ name: container }, { readPreference: 'primary' }); - const containerIsPublic = containerDoc?.isPublic ?? false; - - // Get raw storage URL (sourceUrl) and CDN-applied URL (url) - let sourceUrl: string | undefined; - if (isPublic) { - const publicUrlResult = await storageProvider - .container(container) - .getPublicUrl(fileName, containerIsPublic); - if (publicUrlResult instanceof Error) { - throw publicUrlResult; - } - sourceUrl = publicUrlResult; - } - const url = applyCdnHost(sourceUrl, container); - ConduitGrpcSdk.Metrics?.increment('files_total'); ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', size); - return await File.getInstance().create({ + const file = await File.getInstance().create({ name, alias, mimeType, @@ -118,9 +172,17 @@ export async function storeNewFile( container: container, size, isPublic, - sourceUrl, - url, }); + const refs = await resolveFileReferences(storageProvider, { + container, + folder, + name, + isPublic, + fileId: file._id, + }); + return refs.uri || refs.url || refs.sourceUrl + ? ((await File.getInstance().findByIdAndUpdate(file._id, refs)) as File) + : file; } export async function _createFileUploadUrl( @@ -130,30 +192,11 @@ export async function _createFileUploadUrl( const { name, alias, container, folder, mimeType, isPublic, size } = params; // Validate file privacy against container settings await validateFilePrivacy(container, isPublic); - const fileName = (folder === '/' ? '' : folder) + name; + const fileName = getStorageFileKey(folder, name); await storageProvider .container(container) .store(fileName, Buffer.from('PENDING UPLOAD'), isPublic); - // Get container public status for URL generation - const containerDoc = await _StorageContainer - .getInstance() - .findOne({ name: container }, { readPreference: 'primary' }); - const containerIsPublic = containerDoc?.isPublic ?? false; - - // Get raw storage URL (sourceUrl) and CDN-applied URL (url) - let sourceUrl: string | undefined; - if (isPublic) { - const publicUrlResult = await storageProvider - .container(container) - .getPublicUrl(fileName, containerIsPublic); - if (publicUrlResult instanceof Error) { - throw publicUrlResult; - } - sourceUrl = publicUrlResult; - } - const publicUrl = applyCdnHost(sourceUrl, container); - ConduitGrpcSdk.Metrics?.increment('files_total'); ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', size); const file = await File.getInstance().create({ @@ -164,14 +207,23 @@ export async function _createFileUploadUrl( folder: folder, container: container, isPublic, - sourceUrl, - url: publicUrl, }); + const refs = await resolveFileReferences(storageProvider, { + container, + folder, + name, + isPublic, + fileId: file._id, + }); + const storedFile = + refs.uri || refs.url || refs.sourceUrl + ? ((await File.getInstance().findByIdAndUpdate(file._id, refs)) as File) + : file; const uploadUrl = (await storageProvider .container(container) .getUploadUrl(fileName)) as string; return { - file, + file: storedFile, url: uploadUrl, }; } @@ -182,43 +234,34 @@ export async function _updateFile( params: IFileParams, ): Promise { const { name, alias, data, folder, container, mimeType } = params; + await validateFilePrivacy(container, file.isPublic); const onlyDataUpdate = name === file.name && folder === file.folder && container === file.container; await storageProvider .container(container) - .store((folder === '/' ? '' : folder) + name, data, file.isPublic); + .store(getStorageFileKey(folder, name), data, file.isPublic); if (!onlyDataUpdate) { await storageProvider .container(file.container) - .delete((file.folder === '/' ? '' : file.folder) + file.name); + .delete(getStorageFileKey(file.folder, file.name)); } - // Get container public status for URL generation - const containerDoc = await _StorageContainer - .getInstance() - .findOne({ name: container }, { readPreference: 'primary' }); - const containerIsPublic = containerDoc?.isPublic ?? false; - - // Get raw storage URL (sourceUrl) and CDN-applied URL (url) - let sourceUrl: string | undefined; - if (file.isPublic) { - const publicUrlResult = await storageProvider - .container(container) - .getPublicUrl((folder === '/' ? '' : folder) + name, containerIsPublic); - if (publicUrlResult instanceof Error) { - throw publicUrlResult; - } - sourceUrl = publicUrlResult; - } - const url = applyCdnHost(sourceUrl, container); + const refs = await resolveFileReferences(storageProvider, { + container, + folder, + name, + isPublic: file.isPublic, + fileId: file._id, + }); const updatedFile = (await File.getInstance().findByIdAndUpdate(file._id, { name, alias, folder, container, - sourceUrl, - url, + sourceUrl: refs.sourceUrl, + url: refs.url, + uri: refs.uri, mimeType, })) as File; updateFileMetrics(file.size, (data as Buffer).byteLength); @@ -231,6 +274,14 @@ export async function _updateFileUploadUrl( params: IFileParams, ): Promise<{ file: File; url: string }> { const { name, alias, folder, container, mimeType, size } = params; + await validateFilePrivacy(container, file.isPublic); + const refs = await resolveFileReferences(storageProvider, { + container, + folder, + name, + isPublic: file.isPublic, + fileId: file._id, + }); let updatedFile; const onlyDataUpdate = name === file.name && folder === file.folder && container === file.container; @@ -238,46 +289,31 @@ export async function _updateFileUploadUrl( updatedFile = await File.getInstance().findByIdAndUpdate(file._id, { mimeType, alias, + sourceUrl: refs.sourceUrl, + url: refs.url, + uri: refs.uri, ...{ size: size ?? file.size }, }); } else { await storageProvider .container(container) .store( - (folder === '/' ? '' : folder) + name, + getStorageFileKey(folder, name), Buffer.from('PENDING UPLOAD'), file.isPublic, ); await storageProvider .container(file.container) - .delete((file.folder === '/' ? '' : file.folder) + file.name); - - // Get container public status for URL generation - const containerDoc = await _StorageContainer - .getInstance() - .findOne({ name: container }, { readPreference: 'primary' }); - const containerIsPublic = containerDoc?.isPublic ?? false; - - // Get raw storage URL (sourceUrl) and CDN-applied URL (url) - let sourceUrl: string | undefined; - if (file.isPublic) { - const publicUrlResult = await storageProvider - .container(container) - .getPublicUrl((folder === '/' ? '' : folder) + name, containerIsPublic); - if (publicUrlResult instanceof Error) { - throw publicUrlResult; - } - sourceUrl = publicUrlResult; - } - const url = applyCdnHost(sourceUrl, container); + .delete(getStorageFileKey(file.folder, file.name)); updatedFile = await File.getInstance().findByIdAndUpdate(file._id, { name, alias, folder, container, - sourceUrl, - url, + sourceUrl: refs.sourceUrl, + url: refs.url, + uri: refs.uri, mimeType, ...{ size: size ?? file.size }, }); @@ -285,15 +321,17 @@ export async function _updateFileUploadUrl( if (!isNil(size)) updateFileMetrics(file.size, size!); const uploadUrl = (await storageProvider .container(container) - .getUploadUrl((folder === '/' ? '' : folder) + name)) as string; + .getUploadUrl(getStorageFileKey(folder, name))) as string; return { file: updatedFile!, url: uploadUrl }; } export function updateFileMetrics(currentSize: number, newSize: number) { const fileSizeDiff = Math.abs(currentSize - newSize); - fileSizeDiff < 0 - ? ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', fileSizeDiff) - : ConduitGrpcSdk.Metrics?.decrement('storage_size_bytes_total', fileSizeDiff); + if (newSize > currentSize) { + ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', fileSizeDiff); + } else { + ConduitGrpcSdk.Metrics?.decrement('storage_size_bytes_total', fileSizeDiff); + } } export async function validateName( @@ -369,13 +407,6 @@ export function applyCdnHost( } } -/** - * Validates file privacy settings against container's public status. - * Files in public containers cannot be marked as private. - * @param containerName The name of the container - * @param isFilePublic Whether the file is being marked as public - * @throws GrpcError if validation fails - */ export async function validateFilePrivacy( containerName: string, isFilePublic?: boolean, @@ -383,10 +414,10 @@ export async function validateFilePrivacy( const container = await _StorageContainer .getInstance() .findOne({ name: containerName }, { readPreference: 'primary' }); - if (container?.isPublic && isFilePublic === false) { + if (container?.isPublic && !isFilePublic) { throw new GrpcError( status.INVALID_ARGUMENT, - 'Files in public containers cannot be private', + 'Files in public containers must be public', ); } }