From aa18db894ee22bbb9a8e5b203a76488c164946af Mon Sep 17 00:00:00 2001 From: cherry-git999 Date: Tue, 21 Jul 2026 10:21:35 +0000 Subject: [PATCH 1/4] data store create page upload file working now --- .devcontainer/docker-compose.yml | 1 + .../app/api/endpoints/datasets/datasets.py | 35 +++++++++++++- backend/app/api/endpoints/users/role_check.py | 26 ++++++++--- backend/test_presigned.py | 7 +++ .../src/components/file-upload/FileUpload.tsx | 46 +++++++------------ 5 files changed, 78 insertions(+), 37 deletions(-) create mode 100644 backend/test_presigned.py diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index ea696c4..a18c581 100755 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -46,6 +46,7 @@ services: environment: MINIO_ROOT_USER: ${MINIO_USERNAME} MINIO_ROOT_PASSWORD: ${MINIO_PASSWORD} + MINIO_API_CORS_ALLOW_ORIGIN: "*" command: server /data/s3 --console-address ":9001" restart: unless-stopped network_mode: host diff --git a/backend/app/api/endpoints/datasets/datasets.py b/backend/app/api/endpoints/datasets/datasets.py index bd2bc73..39e16c1 100644 --- a/backend/app/api/endpoints/datasets/datasets.py +++ b/backend/app/api/endpoints/datasets/datasets.py @@ -1,3 +1,4 @@ +import os import uuid import mimetypes import pandas as pd @@ -5,7 +6,7 @@ from datetime import datetime, timezone from pymongo.collection import Collection from bson import ObjectId -from fastapi import APIRouter, HTTPException, Depends +from fastapi import APIRouter, HTTPException, Depends, UploadFile, File from app.auth.user_auth import get_current_user from app.schemas.models import ( CreateDatasetInformationRequest, @@ -32,6 +33,38 @@ def get_presigned_url(filename: str, current_user: dict = Depends(get_current_us return PresignedURLResponse(upload_url=url, object_name=object_name) +@datasets_router.post("/upload", response_model=PresignedURLResponse, operation_id="upload_dataset_file") +async def upload_dataset_file( + files: UploadFile = File(...), + current_user: dict = Depends(get_current_user) +) -> PresignedURLResponse: + try: + storage_service = get_storage_service() + user_id = str(current_user.get("_id")) + + # Read file content + file_content = await files.read() + + # Generate the unique filename using identical logic as MinioStorageService + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + name, extension = os.path.splitext(files.filename) + new_filename = f"{name}_{timestamp}{extension}" + + if user_id: + object_name = f"{user_id}/{new_filename}" + else: + object_name = new_filename + + # Upload the file directly to storage + storage_service.upload_file(file_content, object_name) + + return PresignedURLResponse(upload_url="", object_name=object_name) + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Failed to upload file to storage: {str(e)}" + ) + + @datasets_router.post("/datasets/create", response_model=CreateDatasetInformationResponse, operation_id="create_dataset") async def create_dataset(request: CreateDatasetInformationRequest, current_user: dict = Depends(get_current_user)) -> CreateDatasetInformationResponse: try: diff --git a/backend/app/api/endpoints/users/role_check.py b/backend/app/api/endpoints/users/role_check.py index 3e64354..2ccd107 100644 --- a/backend/app/api/endpoints/users/role_check.py +++ b/backend/app/api/endpoints/users/role_check.py @@ -95,13 +95,27 @@ def check_user_role_access(request: RoleCheckRequest, fastapi_request: Request, raise HTTPException(status_code=404, detail="User not found") # Get user's role - role_id = user.get("role_id") + role_ids = user.get("role_id") role_name = "user" - if role_id: - role = get_role_by_id(role_id) - if role and role.get("role_name"): - role_name = role["role_name"] - logger.debug("User role resolved: role_id=%s role_name=%s", role_id, role_name) + if role_ids: + # Handle both list of IDs and single ID cases + if not isinstance(role_ids, list): + role_ids = [role_ids] + + roles_found = [] + for r_id in role_ids: + role = get_role_by_id(r_id) + if role and role.get("role_name"): + roles_found.append(role["role_name"]) + + # Select the most privileged role if multiple exist + if "superadmin" in roles_found: + role_name = "superadmin" + elif "admin" in roles_found: + role_name = "admin" + elif roles_found: + role_name = roles_found[0] + logger.debug("User role resolved: role_ids=%s role_name=%s", role_ids, role_name) # Find matching endpoint access rule endpoint_access = find_matching_endpoint_access(role_name, request.path) diff --git a/backend/test_presigned.py b/backend/test_presigned.py new file mode 100644 index 0000000..4f11c1f --- /dev/null +++ b/backend/test_presigned.py @@ -0,0 +1,7 @@ +import requests +import os + +token = "114273893325363479564" # from the logs +headers = {"Authorization": f"Bearer {token}"} +resp = requests.get("http://localhost:8000/presignedURL?filename=test.csv", headers=headers) +print(resp.json()) diff --git a/frontend/src/components/file-upload/FileUpload.tsx b/frontend/src/components/file-upload/FileUpload.tsx index 4ec12ed..f01cf7f 100644 --- a/frontend/src/components/file-upload/FileUpload.tsx +++ b/frontend/src/components/file-upload/FileUpload.tsx @@ -164,42 +164,25 @@ export const FileUploader = forwardRef< formData.append("metadata", JSON.stringify({})); try { - // Add timestamp to filename to prevent duplicates - const timestamp = new Date().getTime(); - const fileExt = file.name.split(".").pop() || ""; - const fileNameWithoutExt = file.name.slice(0, -(fileExt.length + 1)); - const uniqueFileName = `${fileNameWithoutExt}_${timestamp}.${fileExt}`; - - // Get presigned URL using SDK - const response = await getPresignedUrl({ - query: { - filename: uniqueFileName, - }, + // Direct upload to FastAPI backend to bypass browser CORS / PNA restrictions + const backendUrl = process.env.NEXT_PUBLIC_BACKEND_URL || "http://localhost:8000"; + const uploadResponse = await fetch(`${backendUrl}/upload`, { + method: "POST", + body: formData, headers: { Authorization: `Bearer ${session?.user?.apiToken}`, - "Content-Type": "application/json", - }, - }); - const presignedUrlResponse = response.data as PresignedUrlResponse; - console.log("Presigned URL response:", presignedUrlResponse); - const upload_url = presignedUrlResponse?.upload_url; - const object_name = presignedUrlResponse?.object_name; - if (!upload_url) { - throw new Error("Failed to get presigned URL"); - } - - // Now upload the file using the presigned URL - const uploadResponse = await fetch(upload_url, { - method: "PUT", - body: file, - headers: { - "Content-Type": file.type, }, }); if (!uploadResponse.ok) { - throw new Error("Upload failed"); + const errorText = await uploadResponse.text(); + throw new Error(`Upload failed: ${errorText || uploadResponse.statusText}`); + } + const uploadResponseData = await uploadResponse.json() as PresignedUrlResponse; + console.log("Direct upload response:", uploadResponseData); + const object_name = uploadResponseData?.object_name; + if (!object_name) { + throw new Error("Failed to get object name from upload response"); } - console.log("Upload response:", uploadResponse); // Extract dataset after successful upload const extractCsvDataRequest: ExtractCsvDataRequest = { @@ -233,6 +216,9 @@ export const FileUploader = forwardRef< }; } catch (error) { console.error("Upload error:", error); + if (error instanceof Error && error.message === "Failed to fetch") { + toast.error("Upload connection failed. If you are using Brave, please try disabling Brave Shields (click the orange lion icon in the URL bar and toggle it off)."); + } throw error; } }, From 67ce56e1dfd0406f6000f6788c3ae4e47e0d2cfc Mon Sep 17 00:00:00 2001 From: cherry-git999 Date: Tue, 21 Jul 2026 10:32:56 +0000 Subject: [PATCH 2/4] fixed view file issue also --- .../app/api/endpoints/datasets/datasets.py | 39 ++++++++++++++ .../file-upload/FileDownloadButton.tsx | 51 +++++++++---------- 2 files changed, 64 insertions(+), 26 deletions(-) diff --git a/backend/app/api/endpoints/datasets/datasets.py b/backend/app/api/endpoints/datasets/datasets.py index 39e16c1..0ff90e6 100644 --- a/backend/app/api/endpoints/datasets/datasets.py +++ b/backend/app/api/endpoints/datasets/datasets.py @@ -7,6 +7,7 @@ from pymongo.collection import Collection from bson import ObjectId from fastapi import APIRouter, HTTPException, Depends, UploadFile, File +from fastapi.responses import StreamingResponse from app.auth.user_auth import get_current_user from app.schemas.models import ( CreateDatasetInformationRequest, @@ -65,6 +66,44 @@ async def upload_dataset_file( ) +@datasets_router.get("/files/{file_id}/view", operation_id="view_file") +def view_file( + file_id: str, + current_user: dict = Depends(get_current_user) +): + try: + # Find the file in files_collection + file_doc = files_collection.find_one({"_id": ObjectId(file_id)}) + if not file_doc: + raise HTTPException(status_code=404, detail="File record not found") + + file_location = file_doc.get("file_location") + if not file_location: + raise HTTPException(status_code=404, detail="File location not found in record") + + storage_service = get_storage_service() + + # Get the object stream from storage + obj = storage_service.get_object(file_location) + + filename = os.path.basename(file_location) + media_type = file_doc.get("file_type") or "application/octet-stream" + + return StreamingResponse( + obj, + media_type=media_type, + headers={ + "Content-Disposition": f"attachment; filename={filename}" + } + ) + except Exception as e: + if isinstance(e, HTTPException): + raise e + raise HTTPException( + status_code=500, detail=f"Failed to retrieve file from storage: {str(e)}" + ) + + @datasets_router.post("/datasets/create", response_model=CreateDatasetInformationResponse, operation_id="create_dataset") async def create_dataset(request: CreateDatasetInformationRequest, current_user: dict = Depends(get_current_user)) -> CreateDatasetInformationResponse: try: diff --git a/frontend/src/components/file-upload/FileDownloadButton.tsx b/frontend/src/components/file-upload/FileDownloadButton.tsx index fbc5a98..be36265 100644 --- a/frontend/src/components/file-upload/FileDownloadButton.tsx +++ b/frontend/src/components/file-upload/FileDownloadButton.tsx @@ -30,7 +30,8 @@ const FileDownloadButton = React.forwardRef< ) => { const theToast = useToast(); - const handleDownload = async () => { + const handleDownload = async (e: React.MouseEvent) => { + e.preventDefault(); try { // If accessToken is "", show an error toast if (!accessToken || accessToken === "") { @@ -54,36 +55,33 @@ const FileDownloadButton = React.forwardRef< return; } - // // Get the download URL from the API - // const response = await getNimbusFileDownloadUrl({ - // path: { - // fileId: fileID, - // }, - // headers: { - // Authorization: `Bearer ${accessToken}`, - // }, - // }); - - const response = { - data: "https://example.com/file.csv", - }; + // Direct download from FastAPI backend to bypass browser CORS / PNA / SSL restrictions + const backendUrl = process.env.NEXT_PUBLIC_BACKEND_URL || "http://localhost:8000"; + const downloadResponse = await fetch(`${backendUrl}/files/${fileID}/view`, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }); - if (response.data) { - window.open(response.data as string, "_blank"); - } else { - theToast.toast({ - title: "Error", - description: "Failed to get file download URL", - variant: "destructive", - }); - console.error("No download URL received"); - throw new Error("No download URL received"); + if (!downloadResponse.ok) { + const errorText = await downloadResponse.text(); + throw new Error(errorText || `Download request failed: ${downloadResponse.statusText}`); } + + const blob = await downloadResponse.blob(); + const blobUrl = window.URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = blobUrl; + a.download = downloadName.endsWith(".csv") ? downloadName : `${downloadName}.csv`; + document.body.appendChild(a); + a.click(); + a.remove(); + window.URL.revokeObjectURL(blobUrl); } catch (error) { - console.error("Error getting file download URL:", error); + console.error("Error downloading file:", error); theToast.toast({ title: "Error", - description: "Failed to get file download URL", + description: error instanceof Error ? error.message : "Failed to download file", variant: "destructive", }); } @@ -92,6 +90,7 @@ const FileDownloadButton = React.forwardRef< return (