diff --git a/backend/app/routers/datasets.py b/backend/app/routers/datasets.py index 76ab085f1..51950a9b0 100644 --- a/backend/app/routers/datasets.py +++ b/backend/app/routers/datasets.py @@ -2,9 +2,9 @@ import io import os from typing import List, Optional - +import zipfile from bson import ObjectId -from fastapi import APIRouter, HTTPException, Depends, File, UploadFile +from fastapi import APIRouter, HTTPException, Depends, File, UploadFile, Response from fastapi import Form from minio import Minio from pymongo import MongoClient @@ -41,6 +41,28 @@ clowder_bucket = os.getenv("MINIO_BUCKET_NAME", "clowder") +async def get_folder_hierarchy( + folder_id: str, + hierarchy: str, + db: MongoClient, +): + found = await db["folders"].find_one({"_id": ObjectId(folder_id)}) + folder = FolderOut.from_mongo(found) + folder_name = folder.name + hierarchy = folder_name + "/" + hierarchy + folder_parent = folder.parent_folder + if folder_parent is not None: + parent_folder_found = await db["folders"].find_one( + {"_id": ObjectId(folder_parent)} + ) + parent_folder = FolderOut.from_mongo(parent_folder_found) + hierarchy = parent_folder.name + "/" + hierarchy + parent_folder_parent = parent_folder.parent_folder + if parent_folder_parent is not None: + hierarchy = await get_folder_hierarchy(str(parent_folder.id), hierarchy, db) + return hierarchy + + @router.post("", response_model=DatasetOut) async def save_dataset( dataset_in: DatasetIn, @@ -174,7 +196,7 @@ async def delete_dataset( await db["datasets"].delete_one({"_id": ObjectId(dataset_id)}) async for file in db["files"].find({"dataset_id": ObjectId(dataset_id)}): fs.remove_object(clowder_bucket, str(file)) - db["file_versions"].delete_many({"file_id": file['_id']}) + db["file_versions"].delete_many({"file_id": file["_id"]}) files_deleted = await db.files.delete_many({"dataset_id": ObjectId(dataset_id)}) folders_delete = await db["folders"].delete_many( {"dataset_id": ObjectId(dataset_id)} @@ -263,17 +285,13 @@ async def save_file( status_code=401, detail=f"User not found. Session might have expired." ) - dataset = await db["datasets"].find_one({"_id": ObjectId(dataset_id)}) - if dataset is None: - raise HTTPException( - status_code=404, detail=f"Dataset {dataset_id} not found" - ) fileDB = FileDB(name=file.filename, creator=user, dataset_id=dataset["_id"]) if folder_id is not None: if ( folder := await db["folders"].find_one({"_id": ObjectId(folder_id)}) ) is not None: + folder = FolderOut.from_mongo(folder) fileDB.folder_id = folder.id else: raise HTTPException( @@ -314,3 +332,38 @@ async def save_file( return fileDB else: raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") + + +@router.get("/{dataset_id}/download", response_model=DatasetOut) +async def download_dataset( + dataset_id: str, + user=Depends(get_current_user), + db: MongoClient = Depends(dependencies.get_db), + fs: Minio = Depends(dependencies.get_fs), +): + if ( + dataset := await db["datasets"].find_one({"_id": ObjectId(dataset_id)}) + ) is not None: + dataset = DatasetOut.from_mongo(dataset) + stream = io.BytesIO() + z = zipfile.ZipFile(stream, "w") + async for f in db["files"].find({"dataset_id": dataset.id}): + file = FileOut.from_mongo(f) + file_name = file.name + if file.folder_id is not None: + hierarchy = await get_folder_hierarchy(file.folder_id, "", db) + file_name = "/" + hierarchy + file_name + content = fs.get_object(settings.MINIO_BUCKET_NAME, str(file.id)) + z.writestr(file_name, content.data) + content.close() + content.release_conn() + z.close() + return Response( + stream.getvalue(), + media_type="application/x-zip-compressed", + headers={ + "Content-Disposition": f'attachment;filename={dataset.name + ".zip"}' + }, + ) + else: + raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") diff --git a/backend/app/routers/files.py b/backend/app/routers/files.py index 40eb31ac6..f6cbeb47a 100644 --- a/backend/app/routers/files.py +++ b/backend/app/routers/files.py @@ -116,7 +116,9 @@ async def delete_file( # TODO: Deleting individual versions may require updating version_id in mongo, or deleting entire document fs.remove_object(settings.MINIO_BUCKET_NAME, str(file_id)) removed_file = await db["files"].delete_one({"_id": ObjectId(file_id)}) - removed_vers = await db["file_versions"].delete_many({"file_id": ObjectId(file_id)}) + removed_vers = await db["file_versions"].delete_many( + {"file_id": ObjectId(file_id)} + ) return {"deleted": file_id} else: raise HTTPException(status_code=404, detail=f"File {file_id} not found") diff --git a/frontend/src/actions/dataset.js b/frontend/src/actions/dataset.js index 3c8c0b185..f70de5d8a 100644 --- a/frontend/src/actions/dataset.js +++ b/frontend/src/actions/dataset.js @@ -186,7 +186,7 @@ export function datasetDownloaded(datasetId, filename = "") { } else { filename = `${datasetId}.zip`; } - const endpoint = `${config.hostname}/datasets/${datasetId}/download?superAdmin=true`; + const endpoint = `${config.hostname}/api/v2/datasets/${datasetId}/download`; const response = await fetch(endpoint, {method: "GET", mode: "cors", headers: await getHeader()}); if (response.status === 200) { diff --git a/frontend/src/components/datasets/Dataset.tsx b/frontend/src/components/datasets/Dataset.tsx index f5ce7a840..049a68d6f 100644 --- a/frontend/src/components/datasets/Dataset.tsx +++ b/frontend/src/components/datasets/Dataset.tsx @@ -246,8 +246,8 @@ export const Dataset = (): JSX.Element => { { handleOptionClose(); - }} disabled={true}> - Download All + }}> + Download Dataset { diff --git a/frontend/src/components/datasets/DatasetCard.tsx b/frontend/src/components/datasets/DatasetCard.tsx index 213312e4b..e9cd257cc 100644 --- a/frontend/src/components/datasets/DatasetCard.tsx +++ b/frontend/src/components/datasets/DatasetCard.tsx @@ -6,6 +6,8 @@ import Button from '@mui/material/Button'; import Typography from '@mui/material/Typography'; import {useNavigate} from "react-router-dom"; import {parseDate} from "../../utils/common"; +import {datasetDownloaded} from "../../actions/dataset"; +import {useDispatch} from "react-redux"; type DatasetCardProps = { id: string, @@ -17,6 +19,10 @@ type DatasetCardProps = { export default function DatasetCard(props: DatasetCardProps) { const { id, name, author, created, description} = props; + + const dispatch = useDispatch(); + const downloadDataset = (datasetId:string|undefined, filename:string|undefined) => dispatch(datasetDownloaded(datasetId, filename)) + const formattedCreated = parseDate(created); // use history hook to redirect/navigate between routes const history = useNavigate(); @@ -42,7 +48,7 @@ export default function DatasetCard(props: DatasetCardProps) { - + ); diff --git a/frontend/src/openapi/v2/services/DatasetsService.ts b/frontend/src/openapi/v2/services/DatasetsService.ts index 9d877777a..24129db28 100644 --- a/frontend/src/openapi/v2/services/DatasetsService.ts +++ b/frontend/src/openapi/v2/services/DatasetsService.ts @@ -251,4 +251,22 @@ export class DatasetsService { }); } -} \ No newline at end of file + /** + * Download Dataset + * @param datasetId + * @returns DatasetOut Successful Response + * @throws ApiError + */ + public static downloadDatasetApiV2DatasetsDatasetIdDownloadGet( + datasetId: string, + ): CancelablePromise { + return __request({ + method: 'GET', + path: `/api/v2/datasets/${datasetId}/download`, + errors: { + 422: `Validation Error`, + }, + }); + } + +}