diff --git a/backend/app/database/errors.py b/backend/app/database/errors.py index 105462675..a811072c2 100644 --- a/backend/app/database/errors.py +++ b/backend/app/database/errors.py @@ -11,13 +11,6 @@ logger = logging.getLogger(__name__) -async def _get_db() -> Generator: - """Duplicate of app.dependencies.get_db(), but importing that causes circular import.""" - mongo_client = motor.motor_asyncio.AsyncIOMotorClient(settings.MONGODB_URL) - db = mongo_client[settings.MONGO_DATABASE] - yield db - - async def log_error( exception: Exception, resource: Optional[MongoDBRef] = None, @@ -30,7 +23,6 @@ async def log_error( resource -- if error relates to a specific resource, you can include it user --- if error relates to actions performed by a user, you can include them """ - db = _get_db() message = str(exception) trace = traceback.format_exc(exception, limit=4) diff --git a/backend/app/dependencies.py b/backend/app/dependencies.py index 1f69a7d2a..2da09d542 100644 --- a/backend/app/dependencies.py +++ b/backend/app/dependencies.py @@ -15,15 +15,6 @@ from app.search.connect import connect_elasticsearch -async def get_db() -> Generator: - mongo_client = motor.motor_asyncio.AsyncIOMotorClient(settings.MONGODB_URL) - db = mongo_client[settings.MONGO_DATABASE] - if db is None: - raise HTTPException(status_code=503, detail="Service not available") - return - yield db - - async def get_fs() -> Generator: file_system = Minio( settings.MINIO_SERVER_URL, diff --git a/backend/app/deps/authorization_deps.py b/backend/app/deps/authorization_deps.py index e4b0a2ad4..297602de7 100644 --- a/backend/app/deps/authorization_deps.py +++ b/backend/app/deps/authorization_deps.py @@ -2,13 +2,11 @@ from beanie.operators import Or from bson import ObjectId from fastapi import Depends, HTTPException -from pymongo import MongoClient -from app.dependencies import get_db from app.keycloak_auth import get_current_username from app.models.authorization import RoleType, AuthorizationDB from app.models.datasets import DatasetDB -from app.models.files import FileOut +from app.models.files import FileOut, FileDB from app.models.groups import GroupOut, GroupDB from app.models.metadata import MetadataDB from app.models.pyobjectid import PyObjectId @@ -32,13 +30,11 @@ async def get_role( async def get_role_by_file( file_id: str, - db: MongoClient = Depends(get_db), current_user=Depends(get_current_username), ) -> RoleType: - if (file := await db["files"].find_one({"_id": ObjectId(file_id)})) is not None: - file_out = FileOut.from_mongo(file) + if (file := await FileDB.get(PydanticObjectId(file_id))) is not None: authorization = await AuthorizationDB.find_one( - AuthorizationDB.dataset_id == file_out.dataset_id, + AuthorizationDB.dataset_id == file.dataset_id, Or( AuthorizationDB.creator == current_user, AuthorizationDB.user_ids == current_user, @@ -50,19 +46,15 @@ async def get_role_by_file( async def get_role_by_metadata( metadata_id: str, - db: MongoClient = Depends(get_db), current_user=Depends(get_current_username), ) -> RoleType: if (md_out := await MetadataDB.get(PydanticObjectId(metadata_id))) is not None: resource_type = md_out.resource.collection resource_id = md_out.resource.resource_id if resource_type == "files": - if ( - file := await db["files"].find_one({"_id": ObjectId(resource_id)}) - ) is not None: - file_out = FileOut.from_mongo(file) + if (file := await FileDB.get(PydanticObjectId(resource_id))) is not None: authorization = await AuthorizationDB.find_one( - AuthorizationDB.dataset_id == file_out.dataset_id, + AuthorizationDB.dataset_id == file.dataset_id, Or( AuthorizationDB.creator == current_user, AuthorizationDB.user_ids == current_user, @@ -114,7 +106,6 @@ def __init__(self, role: str): async def __call__( self, dataset_id: str, - db: MongoClient = Depends(get_db), current_user: str = Depends(get_current_username), ): # TODO: Make sure we enforce only one role per user per dataset, or find_one could yield wrong answer here. @@ -150,13 +141,11 @@ def __init__(self, role: str): async def __call__( self, file_id: str, - db: MongoClient = Depends(get_db), current_user: str = Depends(get_current_username), ): - if (file := await db["files"].find_one({"_id": ObjectId(file_id)})) is not None: - file_out = FileOut.from_mongo(file) + if (file := await FileDB.get(PydanticObjectId(file_id))) is not None: authorization = await AuthorizationDB.find_one( - AuthorizationDB.dataset_id == file_out.dataset_id, + AuthorizationDB.dataset_id == file.dataset_id, Or( AuthorizationDB.creator == current_user, AuthorizationDB.user_ids == current_user, @@ -183,7 +172,6 @@ def __init__(self, role: str): async def __call__( self, metadata_id: str, - db: MongoClient = Depends(get_db), current_user: str = Depends(get_current_username), ): if (md_out := await MetadataDB.get(PydanticObjectId(metadata_id))) is not None: @@ -191,11 +179,10 @@ async def __call__( resource_id = md_out.resource.resource_id if resource_type == "files": if ( - file := await db["files"].find_one({"_id": ObjectId(resource_id)}) + file := await FileDB.get(PydanticObjectId(resource_id)) ) is not None: - file_out = FileOut.from_mongo(file) authorization = await AuthorizationDB.find_one( - AuthorizationDB.dataset_id == file_out.dataset_id, + AuthorizationDB.dataset_id == file.dataset_id, Or( AuthorizationDB.creator == current_user, AuthorizationDB.user_ids == current_user, @@ -246,7 +233,6 @@ def __init__(self, role: str): async def __call__( self, group_id: str, - db: MongoClient = Depends(get_db), current_user: str = Depends(get_current_username), ): if (group := await GroupDB.get(group_id)) is not None: diff --git a/backend/app/models/datasets.py b/backend/app/models/datasets.py index cb84a92f1..223941ae2 100644 --- a/backend/app/models/datasets.py +++ b/backend/app/models/datasets.py @@ -3,7 +3,7 @@ from typing import Optional, List import pymongo -from beanie import Document, View, PydanticObjectId +from beanie import Document, View from pydantic import BaseModel, Field from app.models.authorization import RoleType, AuthorizationDB @@ -56,8 +56,6 @@ class Settings: class DatasetDBViewList(View, DatasetBase): - # FIXME This seems to be required to return _id. Otherwise _id is null in the response. - id: PydanticObjectId = Field(None, alias="_id") creator: UserOut created: datetime = Field(default_factory=datetime.utcnow) modified: datetime = Field(default_factory=datetime.utcnow) diff --git a/backend/app/models/feeds.py b/backend/app/models/feeds.py index 28faab823..97faef5fa 100644 --- a/backend/app/models/feeds.py +++ b/backend/app/models/feeds.py @@ -27,8 +27,6 @@ class FeedIn(JobFeed): class FeedDB(Document, JobFeed, Provenance): - id: PydanticObjectId = Field(None, alias="_id") - class Settings: name = "feeds" indexes = [ diff --git a/backend/app/models/files.py b/backend/app/models/files.py index 323541531..26f3ed1cf 100644 --- a/backend/app/models/files.py +++ b/backend/app/models/files.py @@ -1,9 +1,10 @@ from datetime import datetime -from typing import Optional +from typing import Optional, List -from beanie import Document, PydanticObjectId +from beanie import Document, View from pydantic import Field, BaseModel +from app.models.authorization import AuthorizationDB from app.models.pyobjectid import PyObjectId from app.models.users import UserOut @@ -29,8 +30,6 @@ class FileVersion(BaseModel): class FileVersionDB(Document, FileVersion): - id: PydanticObjectId = Field(None, alias="_id") - class Settings: name = "file_versions" @@ -44,7 +43,6 @@ class FileIn(FileBase): class FileDB(Document, FileBase): - id: PydanticObjectId = Field(None, alias="_id") creator: UserOut created: datetime = Field(default_factory=datetime.utcnow) version_id: str = "N/A" @@ -60,5 +58,31 @@ class Settings: name = "files" +class FileDBViewList(View, FileBase): + creator: UserOut + created: datetime = Field(default_factory=datetime.utcnow) + modified: datetime = Field(default_factory=datetime.utcnow) + auth: List[AuthorizationDB] + + class Settings: + source = FileDB + name = "files_view" + pipeline = [ + { + "$lookup": { + "from": "authorization", + "localField": "dataset_id", + "foreignField": "dataset_id", + "as": "auth", + } + }, + ] + # Needs fix to work https://github.com/roman-right/beanie/pull/521 + # use_cache = True + # cache_expiration_time = timedelta(seconds=10) + # cache_capacity = 5 + + class FileOut(FileDB): - pass + class Config: + fields = {"id": "id"} diff --git a/backend/app/models/folders.py b/backend/app/models/folders.py index 1dd49fd2f..83a3f763c 100644 --- a/backend/app/models/folders.py +++ b/backend/app/models/folders.py @@ -1,9 +1,10 @@ from datetime import datetime -from typing import Optional +from typing import Optional, List -from beanie import Document, PydanticObjectId +from beanie import Document, View from pydantic import Field, BaseModel +from app.models.authorization import AuthorizationDB from app.models.pyobjectid import PyObjectId from app.models.users import UserOut @@ -17,7 +18,6 @@ class FolderIn(FolderBase): class FolderDB(Document, FolderBase): - id: PydanticObjectId = Field(None, alias="_id") dataset_id: PyObjectId parent_folder: Optional[PyObjectId] creator: UserOut @@ -28,5 +28,31 @@ class Settings: name = "folders" +class FolderDBViewList(View, FolderBase): + creator: UserOut + created: datetime = Field(default_factory=datetime.utcnow) + modified: datetime = Field(default_factory=datetime.utcnow) + auth: List[AuthorizationDB] + + class Settings: + source = FolderDB + name = "folders_view" + pipeline = [ + { + "$lookup": { + "from": "authorization", + "localField": "dataset_id", + "foreignField": "dataset_id", + "as": "auth", + } + }, + ] + # Needs fix to work https://github.com/roman-right/beanie/pull/521 + # use_cache = True + # cache_expiration_time = timedelta(seconds=10) + # cache_capacity = 5 + + class FolderOut(FolderDB): - pass + class Config: + fields = {"id": "id"} diff --git a/backend/app/models/listeners.py b/backend/app/models/listeners.py index 41a4c1e81..051fc6a59 100644 --- a/backend/app/models/listeners.py +++ b/backend/app/models/listeners.py @@ -67,7 +67,6 @@ class LegacyEventListenerIn(ExtractorInfo): class EventListenerDB(Document, EventListenerBase): """EventListeners have a name, version, author, description, and optionally properties where extractor_info will be saved.""" - id: PydanticObjectId = Field(None, alias="_id") creator: Optional[UserOut] = None created: datetime = Field(default_factory=datetime.now) modified: datetime = Field(default_factory=datetime.now) @@ -132,8 +131,6 @@ class Config: class EventListenerJobDB(Document, EventListenerJobBase): """This summarizes a submission to an extractor. All messages from that extraction should include this job's ID.""" - id: PydanticObjectId = Field(None, alias="_id") - class Settings: name = "listener_jobs" indexes = [ @@ -175,7 +172,6 @@ class EventListenerDatasetJobMessage(BaseModel): class EventListenerJobUpdateBase(BaseModel): """This is a status update message coming from the extractors back to Clowder.""" - id: PydanticObjectId = Field(None, alias="_id") job_id: str timestamp: datetime = Field(default_factory=datetime.utcnow) status: str @@ -195,8 +191,6 @@ class Settings: class EventListenerJobViewList(View, EventListenerJobBase): """Get associated resource information for each job""" - # FIXME This seems to be required to return _id. Otherwise _id is null in the response. - id: PydanticObjectId = Field(None, alias="_id") creator: UserOut created: datetime = Field(default_factory=datetime.utcnow) modified: datetime = Field(default_factory=datetime.utcnow) @@ -262,8 +256,6 @@ class Settings: class EventListenerJobUpdateViewList(View, EventListenerJobUpdateBase): """Get associated resource information for each job update""" - # FIXME This seems to be required to return _id. Otherwise _id is null in the response. - id: PydanticObjectId = Field(None, alias="_id") creator: UserOut created: datetime = Field(default_factory=datetime.utcnow) modified: datetime = Field(default_factory=datetime.utcnow) diff --git a/backend/app/models/metadata.py b/backend/app/models/metadata.py index d64516c2b..4464d59e8 100644 --- a/backend/app/models/metadata.py +++ b/backend/app/models/metadata.py @@ -107,7 +107,6 @@ class MetadataDefinitionIn(MetadataDefinitionBase): class MetadataDefinitionDB(Document, MetadataDefinitionBase): - id: PydanticObjectId = Field(None, alias="_id") creator: UserOut class Settings: @@ -228,7 +227,6 @@ class MetadataDelete(BaseModel): class MetadataDB(Document, MetadataBase): - id: PydanticObjectId = Field(None, alias="_id") resource: MongoDBRef agent: MetadataAgent created: datetime = Field(default_factory=datetime.utcnow) diff --git a/backend/app/models/users.py b/backend/app/models/users.py index d65d3ecb9..eb6b4b22b 100644 --- a/backend/app/models/users.py +++ b/backend/app/models/users.py @@ -4,7 +4,6 @@ from beanie import Document from passlib.context import CryptContext from pydantic import Field, EmailStr, BaseModel -from pymongo import MongoClient pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") @@ -33,6 +32,9 @@ class UserDB(Document, UserBase): def verify_password(self, password): return pwd_context.verify(password, self.hashed_password) + class Settings: + name = "users" + class UserOut(UserBase): first_name: str @@ -62,8 +64,5 @@ class UserAPIKeyOut(BaseModel): created: datetime = Field(default_factory=datetime.utcnow) expires: Optional[datetime] = None - -async def get_user_out(user_id: str, db: MongoClient) -> UserOut: - """Retrieve user from Mongo based on email address.""" - user_out = await db["users"].find_one({"email": user_id}) - return UserOut.from_mongo(user_out) + class Config: + fields: {"id": "id"} diff --git a/backend/app/rabbitmq/heartbeat_listener_sync.py b/backend/app/rabbitmq/heartbeat_listener_sync.py index 14def89ea..759753a0b 100644 --- a/backend/app/rabbitmq/heartbeat_listener_sync.py +++ b/backend/app/rabbitmq/heartbeat_listener_sync.py @@ -4,7 +4,6 @@ import pika from packaging import version -from pymongo import MongoClient from app.config import settings from app.models.listeners import EventListenerDB, EventListenerOut, ExtractorInfo @@ -28,9 +27,6 @@ def callback(ch, method, properties, body): **extractor_info, properties=ExtractorInfo(**extractor_info) ) - mongo_client = MongoClient(settings.MONGODB_URL) - db = mongo_client[settings.MONGO_DATABASE] - # check to see if extractor alredy exists and update if so existing_extractor = await EventListenerDB.find_one( EventListenerDB.name == msg["queue"] @@ -43,7 +39,7 @@ def callback(ch, method, properties, body): # if this is a new version, add it to the database new_extractor = await extractor_db.insert() # TODO - for now we are not deleting an older version of the extractor, just adding a new one - # removed = db["listeners"].delete_one({"_id": existing_extractor["_id"]}) + # await existing_extractor.delete() extractor_out = EventListenerOut(**new_extractor.dict()) logger.info( "%s updated from %s to %s" @@ -62,7 +58,7 @@ def callback(ch, method, properties, body): processed_feed = await _process_incoming_v1_extractor_info( extractor_name, extractor_out.id, process ) - db["feeds"].insert_one(processed_feed) + await processed_feed.insert() return extractor_out diff --git a/backend/app/rabbitmq/listeners.py b/backend/app/rabbitmq/listeners.py index 71e095dc7..b1efd24a4 100644 --- a/backend/app/rabbitmq/listeners.py +++ b/backend/app/rabbitmq/listeners.py @@ -3,7 +3,6 @@ import string import random from fastapi import Request, HTTPException, Depends -from pymongo import MongoClient from pika.adapters.blocking_connection import BlockingChannel from app.config import settings @@ -23,10 +22,6 @@ async def create_reply_queue(): - # TODO: Dependency injection not working here - mongo_client = MongoClient(settings.MONGODB_URL) - db = mongo_client[settings.MONGO_DATABASE] - credentials = pika.PlainCredentials("guest", "guest") parameters = pika.ConnectionParameters("localhost", credentials=credentials) connection = pika.BlockingConnection(parameters) diff --git a/backend/app/rabbitmq/message_listener_sync.py b/backend/app/rabbitmq/message_listener_sync.py index ba1e7bcd3..3cea49d1e 100644 --- a/backend/app/rabbitmq/message_listener_sync.py +++ b/backend/app/rabbitmq/message_listener_sync.py @@ -7,7 +7,6 @@ import pika from bson import ObjectId -from pymongo import MongoClient from app.config import settings from app.models.config import ConfigEntryDB, ConfigEntryOut @@ -83,13 +82,10 @@ def callback(ch, method, properties, body): ) # incoming format: '2023-01-20T08:30:27-05:00' timestamp = timestamp.replace(tzinfo=datetime.utcnow().tzinfo) - mongo_client = MongoClient(settings.MONGODB_URL) - db = mongo_client[settings.MONGO_DATABASE] - # TODO: Updating an event message could go in rabbitmq/listeners # Check if the job exists, and update if so - job = EventListenerDB.find_one(EventListenerDB.id == ObjectId(job_id)) + job = await EventListenerDB.find_one(EventListenerDB.id == ObjectId(job_id)) if job: # Update existing job with newest info job.updated = timestamp @@ -141,9 +137,9 @@ async def listen_for_messages(): connection = pika.BlockingConnection(parameters) channel = connection.channel() - mongo_client = MongoClient(settings.MONGODB_URL) - db = mongo_client[settings.MONGO_DATABASE] - if (config_entry := ConfigEntryOut.find_one({"key": "instance_id"})) is not None: + if ( + config_entry := await ConfigEntryOut.find_one({"key": "instance_id"}) + ) is not None: instance_id = config_entry.value else: # If no ID has been generated for this instance, generate a 10-digit alphanumeric identifier diff --git a/backend/app/routers/authentication.py b/backend/app/routers/authentication.py index 0774302ef..d90b8545b 100644 --- a/backend/app/routers/authentication.py +++ b/backend/app/routers/authentication.py @@ -7,7 +7,6 @@ KeycloakPostError, ) from passlib.hash import bcrypt -from pymongo import MongoClient from app import dependencies from app.keycloak_auth import create_user @@ -18,7 +17,7 @@ @router.post("/users", response_model=UserOut) -async def save_user(userIn: UserIn, db: MongoClient = Depends(dependencies.get_db)): +async def save_user(userIn: UserIn): try: keycloak_user = await create_user( userIn.email, userIn.password, userIn.first_name, userIn.last_name @@ -39,17 +38,17 @@ async def save_user(userIn: UserIn, db: MongoClient = Depends(dependencies.get_d # create local user hashed_password = bcrypt.hash(userIn.password) - userDB = UserDB( + user = UserDB( **userIn.dict(), hashed_password=hashed_password, keycloak_id=keycloak_user, ) - await userDB.insert() - return UserOut(**userDB.dict()) + await user.insert() + return user.dict() @router.post("/login") -async def login(userIn: UserLogin, db: MongoClient = Depends(dependencies.get_db)): +async def login(userIn: UserLogin): try: token = keycloak_openid.token(userIn.email, userIn.password) return {"token": token["access_token"]} @@ -69,7 +68,7 @@ async def login(userIn: UserLogin, db: MongoClient = Depends(dependencies.get_db ) -async def authenticate_user(email: str, password: str, db: MongoClient): +async def authenticate_user(email: str, password: str): user = await UserDB.find_one({"email": email}) if not user: return None diff --git a/backend/app/routers/authorization.py b/backend/app/routers/authorization.py index 5d3f4edb1..5b90e1ca2 100644 --- a/backend/app/routers/authorization.py +++ b/backend/app/routers/authorization.py @@ -1,15 +1,11 @@ from typing import List from beanie import PydanticObjectId -from beanie.operators import Or +from beanie.operators import Or, In from bson import ObjectId from fastapi import APIRouter, Depends from fastapi.exceptions import HTTPException -from pydantic.networks import EmailStr -from pymongo import MongoClient -from app import dependencies -from app.dependencies import get_db from app.deps.authorization_deps import ( Authorization, get_role_by_file, @@ -30,9 +26,9 @@ DatasetRoles, DatasetDB, ) -from app.models.groups import GroupOut +from app.models.groups import GroupOut, GroupDB from app.models.pyobjectid import PyObjectId -from app.models.users import UserOut +from app.models.users import UserOut, UserDB router = APIRouter() @@ -42,24 +38,21 @@ async def save_authorization( dataset_id: str, authorization_in: AuthorizationBase, user=Depends(get_current_username), - db: MongoClient = Depends(dependencies.get_db), allow: bool = Depends(Authorization("editor")), ): """Save authorization info in Mongo. This is a triple of dataset_id/user_id/role/group_id.""" # Retrieve users from groups in mongo - user_ids: List[EmailStr] = authorization_in.user_ids - group_q_list = db["groups"].find({"_id": {"$in": authorization_in.group_ids}}) + user_ids = authorization_in.user_ids + group_list = await GroupDB.find(In(GroupDB.id, authorization_in.group_ids)) found_groups = 0 - async for group_q in group_q_list: + async for group in group_list: found_groups += 1 - group = GroupOut.from_mongo(group_q) for u in group.users: user_ids.append(u.user.email) if found_groups != len(authorization_in.group_ids): missing_groups = authorization_in.group_ids - async for group_q in group_q_list: - group = GroupOut.from_mongo(group_q) + async for group in group_list: missing_groups.remove(group.id) raise HTTPException( status_code=404, detail=f"Groups not found: {missing_groups}" @@ -68,19 +61,17 @@ async def save_authorization( authorization = await AuthorizationDB( **authorization_in.dict(), creator=user, user_ids=user_ids ) - authorization_db = authorization.insert() - return authorization_db + await authorization.insert() + return authorization @router.get("/datasets/{dataset_id}/role", response_model=AuthorizationDB) async def get_dataset_role( dataset_id: str, current_user=Depends(get_current_username), - db: MongoClient = Depends(get_db), ): """Retrieve role of user for a specific dataset.""" # Get group id and the associated users from authorization - if ( auth_db := await AuthorizationDB.find_one( AuthorizationDB.dataset_id == PyObjectId(dataset_id), @@ -153,18 +144,14 @@ async def set_dataset_group_role( dataset_id: str, group_id: str, role: RoleType, - db: MongoClient = Depends(get_db), user_id=Depends(get_user), allow: bool = Depends(Authorization("editor")), ): """Assign an entire group a specific role for a dataset.""" if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None: - if ( - group_q := await db["groups"].find_one({"_id": ObjectId(group_id)}) - ) is not None: - group = GroupOut.from_mongo(group_q) + if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: # First, remove any existing role the group has on the dataset - await remove_dataset_group_role(dataset_id, group_id, db, user_id, allow) + await remove_dataset_group_role(dataset_id, group_id, user_id, allow) if ( auth_db := await AuthorizationDB.find_one( AuthorizationDB.dataset_id == PyObjectId(dataset_id), @@ -182,13 +169,14 @@ async def set_dataset_group_role( user_ids = [] for u in group.users: user_ids.append(u.user.email) - auth_db = await AuthorizationDB( + auth_db = AuthorizationDB( creator=user_id, dataset_id=PyObjectId(dataset_id), role=role, group_ids=[PyObjectId(group_id)], user_ids=user_ids, - ).insert() + ) + await auth_db.insert() return auth_db else: raise HTTPException(status_code=404, detail=f"Group {group_id} not found") @@ -203,16 +191,15 @@ async def set_dataset_user_role( dataset_id: str, username: str, role: RoleType, - db: MongoClient = Depends(get_db), user_id=Depends(get_user), allow: bool = Depends(Authorization("editor")), ): """Assign a single user a specific role for a dataset.""" if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None: - if (user_q := await db["users"].find_one({"email": username})) is not None: + if (user := await UserDB.find_one(UserDB.email == username)) is not None: # First, remove any existing role the user has on the dataset - await remove_dataset_user_role(dataset_id, username, db, user_id, allow) + await remove_dataset_user_role(dataset_id, username, user_id, allow) auth_db = await AuthorizationDB.find_one( AuthorizationDB.dataset_id == PyObjectId(dataset_id), AuthorizationDB.role == role, @@ -222,12 +209,9 @@ async def set_dataset_user_role( await auth_db.save() if username in auth_db.user_ids: # Only add user entry if all the others occurrences are from associated groups - group_q_list = db["groups"].find( - {"_id": {"$in": auth_db.group_ids}} - ) + group_list = await GroupDB.find(In(GroupDB.id, auth_db.group_ids)) group_occurrences = 0 - async for group_q in group_q_list: - group = GroupOut.from_mongo(group_q) + async for group in group_list: for u in group.users: if u.user.email == username: group_occurrences += 1 @@ -240,12 +224,13 @@ async def set_dataset_user_role( return auth_db else: # Create a new entry - auth_db = await AuthorizationDB( + auth_db = AuthorizationDB( creator=user_id, dataset_id=PyObjectId(dataset_id), role=role, user_ids=[username], - ).insert() + ) + await auth_db.insert() return auth_db else: @@ -261,17 +246,13 @@ async def set_dataset_user_role( async def remove_dataset_group_role( dataset_id: str, group_id: str, - db: MongoClient = Depends(get_db), user_id=Depends(get_user), allow: bool = Depends(Authorization("editor")), ): """Remove any role the group has with a specific dataset.""" if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None: - if ( - group_q := await db["groups"].find_one({"_id": ObjectId(group_id)}) - ) is not None: - group = GroupOut.from_mongo(group_q) + if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: if ( auth_db := await AuthorizationDB.find_one( AuthorizationDB.dataset_id == PyObjectId(dataset_id), @@ -297,14 +278,13 @@ async def remove_dataset_group_role( async def remove_dataset_user_role( dataset_id: str, username: str, - db: MongoClient = Depends(get_db), user_id=Depends(get_user), allow: bool = Depends(Authorization("editor")), ): """Remove any role the user has with a specific dataset.""" if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None: - if (user_q := await db["users"].find_one({"email": username})) is not None: + if (user := await UserDB.find_one(UserDB.email == username)) is not None: if ( auth_db := await AuthorizationDB.find_one( AuthorizationDB.dataset_id == PyObjectId(dataset_id), @@ -323,22 +303,19 @@ async def remove_dataset_user_role( @router.get("/datasets/{dataset_id}/roles", response_model=DatasetRoles) async def get_dataset_roles( dataset_id: str, - db: MongoClient = Depends(dependencies.get_db), allow: bool = Depends(Authorization("editor")), ): """Get a list of all users and groups that have assigned roles on this dataset.""" if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None: roles = DatasetRoles(dataset_id=str(dataset.id)) - async for auth_q in AuthorizationDB.find( + async for auth in AuthorizationDB.find( AuthorizationDB.dataset_id == ObjectId(dataset_id) ): - auth = AuthorizationOut.from_mongo(auth_q) - # First, fetch all groups that have a role on the dataset + group_list = GroupDB.find(In(GroupDB.id, auth.group_ids)) group_user_counts = {} - async for group_q in db["groups"].find({"_id": {"$in": auth.group_ids}}): - group = GroupOut.from_mongo(group_q) + async for group in group_list: group.id = str(group.id) for u in group.users: u.user.id = str(u.user.id) @@ -350,8 +327,7 @@ async def get_dataset_roles( roles.group_roles.append(GroupAndRole(group=group, role=auth.role)) # Next, get all users but omit those that are included in a group above - async for user_q in db["users"].find({"email": {"$in": auth.user_ids}}): - user = UserOut.from_mongo(user_q) + async for user in UserDB.find(In(UserDB.email, auth.user_ids)): if ( user.email in group_user_counts and auth.user_ids.count(user.email) == group_user_counts[user.email] diff --git a/backend/app/routers/datasets.py b/backend/app/routers/datasets.py index 82eb13176..b11ba793c 100644 --- a/backend/app/routers/datasets.py +++ b/backend/app/routers/datasets.py @@ -6,7 +6,7 @@ import tempfile import zipfile from collections.abc import Mapping, Iterable -from typing import List, Optional, Union +from typing import List, Optional from beanie import PydanticObjectId from beanie.operators import Or @@ -23,7 +23,7 @@ from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from minio import Minio from pika.adapters.blocking_connection import BlockingChannel -from pymongo import MongoClient, DESCENDING +from pymongo import DESCENDING from rocrate.model.person import Person from rocrate.rocrate import ROCrate @@ -42,8 +42,8 @@ DatasetPatch, DatasetDBViewList, ) -from app.models.files import FileOut, FileDB -from app.models.folders import FolderOut, FolderIn, FolderDB +from app.models.files import FileOut, FileDB, FileDBViewList +from app.models.folders import FolderOut, FolderIn, FolderDB, FolderDBViewList from app.models.metadata import MetadataDB from app.models.pyobjectid import PyObjectId from app.models.users import UserOut @@ -133,7 +133,6 @@ async def _create_folder_structure( folder_path: str, folder_lookup: dict, user: UserOut, - db: MongoClient, parent_folder_id: Optional[str] = None, ): """Recursively create folders encountered in folder_path until the target folder is created. @@ -154,15 +153,15 @@ async def _create_folder_structure( "name": k, "parent_folder": parent_folder_id, } - folder_db = await FolderDB(**folder_dict, creator=user).insert() - new_folder_id = folder_db.id + folder_db = FolderDB(**folder_dict, creator=user) + await folder_db.insert() # Store ID and call recursively on child folders new_folder_path = folder_path + os.path.sep + k - folder_lookup[new_folder_path] = new_folder_id + folder_lookup[new_folder_path] = folder_db.id if isinstance(v, Mapping): folder_lookup = await _create_folder_structure( - dataset_id, v, new_folder_path, folder_lookup, user, db, new_folder_id + dataset_id, v, new_folder_path, folder_lookup, user, folder_db.id ) return folder_lookup @@ -171,30 +170,19 @@ async def _create_folder_structure( async def _get_folder_hierarchy( folder_id: str, hierarchy: str, - db: MongoClient, ): """Generate a string of nested path to folder for use in zip file creation.""" - found = await db["folders"].find_one({"_id": ObjectId(folder_id)}) - folder = FolderOut.from_mongo(found) + folder = await FolderDB.get(PydanticObjectId(folder_id)) hierarchy = folder.name + "/" + hierarchy if folder.parent_folder is not None: - hierarchy = await _get_folder_hierarchy(folder.parent_folder, hierarchy, db) + hierarchy = await _get_folder_hierarchy(folder.parent_folder, hierarchy) return hierarchy -async def remove_folder_entry( - folder_id: Union[str, ObjectId], - db: MongoClient, -): - """Remove FolderDB object into MongoDB""" - await db["folders"].delete_one({"_id": ObjectId(folder_id)}) - - @router.post("", response_model=DatasetOut) async def save_dataset( dataset_in: DatasetIn, user=Depends(keycloak_auth.get_current_user), - db: MongoClient = Depends(dependencies.get_db), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), ): dataset = DatasetDB(**dataset_in.dict(), creator=user) @@ -217,18 +205,16 @@ async def save_dataset( "download": dataset.downloads, } insert_record(es, "dataset", doc, dataset.id) - return DatasetOut(**dataset.dict()) + return dataset.dict() @router.get("", response_model=List[DatasetOut]) async def get_datasets( user_id=Depends(get_user), - db: MongoClient = Depends(dependencies.get_db), skip: int = 0, limit: int = 10, mine: bool = False, ): - # TODO: Other endpoints convert DB response to DatasetOut(**response.dict()) if mine: datasets = await DatasetDBViewList.find( { @@ -270,68 +256,30 @@ async def get_dataset_files( dataset_id: str, folder_id: Optional[str] = None, user_id=Depends(get_user), - db: MongoClient = Depends(dependencies.get_db), allow: bool = Depends(Authorization("viewer")), skip: int = 0, limit: int = 10, ): - files = [] - if folder_id is not None: - for f in ( - await db["files_view"] - .find( - { - "$and": [ - { - "dataset_id": ObjectId(dataset_id), - "folder_id": ObjectId(folder_id), - }, - { - "$or": [ - {"creator.email": user_id}, - {"auth": {"$elemMatch": {"user_ids": user_id}}}, - ] - }, - ] - } - ) - .skip(skip) - .limit(limit) - .to_list(length=limit) - ): - files.append(FileOut.from_mongo(f)) - else: - for f in ( - await db["files_view"] - .find( - { - "$and": [ - { - "dataset_id": ObjectId(dataset_id), - "folder_id": None, - }, - { - "$or": [ - {"creator.email": user_id}, - {"auth": {"$elemMatch": {"user_ids": user_id}}}, - ] - }, - ] - } - ) - .skip(skip) - .limit(limit) - .to_list(length=limit) - ): - files.append(FileOut.from_mongo(f)) - return files + files = ( + await FileDBViewList.find( + FileDBViewList.dataset_id == ObjectId(dataset_id), + FileDBViewList.folder_id == ObjectId(folder_id), + Or( + FileDBViewList.creator.email == user_id, + FileDBViewList.auth.user_ids == user_id, + ), + ) + .skip(skip) + .limit(limit) + .to_list() + ) + return [file.dict() for file in files] @router.put("/{dataset_id}", response_model=DatasetOut) async def edit_dataset( dataset_id: str, dataset_info: DatasetBase, - db: MongoClient = Depends(dependencies.get_db), user=Depends(get_current_user), es=Depends(dependencies.get_elasticsearchclient), allow: bool = Depends(Authorization("editor")), @@ -364,7 +312,7 @@ async def edit_dataset( } } update_record(es, "metadata", doc, str(metadata.id)) - return DatasetOut(**dataset.dict()) + return dataset.dict() raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") @@ -390,7 +338,7 @@ async def patch_dataset( } update_record(es, "dataset", doc, dataset_id) # updating metadata in elasticsearch - metadata = MetadataDB.find_one( + metadata = await MetadataDB.find_one( MetadataDB.resource.resource_id == ObjectId(dataset_id) ) if metadata: @@ -401,13 +349,12 @@ async def patch_dataset( } } update_record(es, "metadata", doc, str(metadata["_id"])) - return DatasetOut(**dataset.dict()) + return dataset.dict() @router.delete("/{dataset_id}") async def delete_dataset( dataset_id: str, - db: MongoClient = Depends(dependencies.get_db), fs: Minio = Depends(dependencies.get_fs), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), allow: bool = Depends(Authorization("editor")), @@ -422,10 +369,9 @@ async def delete_dataset( await MetadataDB.delete_all( MetadataDB.resource.resource_id == ObjectId(dataset_id) ) - async for file in db["files"].find({"dataset_id": ObjectId(dataset_id)}): - file = FileOut(**file) + async for file in FileDB.find(FileDB.dataset_id == ObjectId(dataset_id)): await remove_file_entry(file.id, fs, es) - await db["folders"].delete_many({"dataset_id": ObjectId(dataset_id)}) + await FolderDB.delete_all(FolderDB.dataset_id == ObjectId(dataset_id)) return {"deleted": dataset_id} raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") @@ -435,59 +381,47 @@ async def add_folder( dataset_id: str, folder_in: FolderIn, user=Depends(get_current_user), - db: MongoClient = Depends(dependencies.get_db), allow: bool = Depends(Authorization("uploader")), ): if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None: parent_folder = folder_in.parent_folder if parent_folder is not None: - folder = await db["folders"].find_one({"_id": ObjectId(parent_folder)}) - if folder is None: + if (await FolderDB.get(PydanticObjectId(parent_folder))) is None: raise HTTPException( status_code=400, detail=f"Parent folder {parent_folder} not found" ) - new_folder = await FolderDB( + new_folder = FolderDB( **folder_in.dict(), creator=user, dataset_id=PyObjectId(dataset_id) - ).insert() - folder_out = FolderOut(**new_folder.dict()) - return folder_out + ) + await new_folder.insert() + return new_folder.dict() raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") -@router.get("/{dataset_id}/folders") +@router.get("/{dataset_id}/folders", response_model=List[FolderOut]) async def get_dataset_folders( dataset_id: str, parent_folder: Optional[str] = None, user_id=Depends(get_user), - db: MongoClient = Depends(dependencies.get_db), allow: bool = Depends(Authorization("viewer")), + skip: int = 0, + limit: int = 10, ): if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None: - folders = [] - if parent_folder is None: - async for f in db["folders"].find( - {"dataset_id": ObjectId(dataset_id), "parent_folder": None} - ): - folders.append(FolderDB.from_mongo(f)) - else: - async for f in db["folders"].find( - { - "$and": [ - { - "dataset_id": ObjectId(dataset_id), - "parent_folder": ObjectId(parent_folder), - }, - { - "$or": [ - {"author.email": user_id}, - {"auth": {"$elemMatch": {"user_ids": user_id}}}, - ] - }, - ] - } - ): - folders.append(FolderDB.from_mongo(f)) - return folders + folders = ( + await FolderDBViewList.find( + FolderDBViewList.dataset_id == ObjectId(dataset_id), + FolderDBViewList.folder_id == ObjectId(parent_folder), + Or( + FolderDBViewList.creator.email == user_id, + FolderDBViewList.auth.user_ids == user_id, + ), + ) + .skip(skip) + .limit(limit) + .to_list() + ) + return [folder.dict() for folder in folders] raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") @@ -495,61 +429,35 @@ async def get_dataset_folders( async def delete_folder( dataset_id: str, folder_id: str, - db: MongoClient = Depends(dependencies.get_db), fs: Minio = Depends(dependencies.get_fs), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), allow: bool = Depends(Authorization("editor")), ): if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None: - if (await db["folders"].find_one({"_id": ObjectId(folder_id)})) is not None: + if (folder := await FolderDB.get(PydanticObjectId(folder_id))) is not None: # delete current folder and files - await remove_folder_entry(folder_id, db) - async for file in db["files"].find({"folder_id": ObjectId(folder_id)}): - file = FileOut(**file) + async for file in FileDB.find(FileDB.folder_id == ObjectId(folder_id)): await remove_file_entry(file.id, fs, es) - # list all child folders and delete child folders/files - parent_folder_id = folder_id - + # recursively delete child folder and files async def _delete_nested_folders(parent_folder_id): while ( - folders := await db["folders"].find_one( - { - "dataset_id": ObjectId(dataset_id), - "parent_folder": ObjectId(parent_folder_id), - } + await FolderDB.find_one( + FolderDB.dataset_id == ObjectId(dataset_id), + FolderDB.parent_folder == ObjectId(parent_folder_id), ) ) is not None: - async for folder in db["folders"].find( - { - "dataset_id": ObjectId(dataset_id), - "parent_folder": ObjectId(parent_folder_id), - } + async for subfolder in FolderDB.find( + FolderDB.dataset_id == PydanticObjectId(dataset_id), + FolderDB.parent_folder == PydanticObjectId(parent_folder_id), ): - folder = FolderOut(**folder) - parent_folder_id = folder.id - - # recursively delete child folder and files - await _delete_nested_folders(parent_folder_id) - - await remove_folder_entry(folder.id, db) - async for file in db["files"].find( - {"folder_id": ObjectId(folder.id)} - ): - folder = FolderOut(**folder.dict()) - parent_folder_id = folder.id - - # recursively delete child folder and files - await _delete_nested_folders(parent_folder_id) - - await remove_folder_entry(folder.id, db) - async for file in db["files"].find( - {"folder_id": ObjectId(folder.id)} - ): - file = FileOut(**file) - await remove_file_entry(file.id, fs, es) - - await _delete_nested_folders(parent_folder_id) + async for file in FileDB.find(FileDB.folder_id == subfolder.id): + await remove_file_entry(file.id, fs, es) + await _delete_nested_folders(subfolder.id) + await subfolder.delete() + + await _delete_nested_folders(folder_id) + await folder.delete() return {"deleted": folder_id} else: raise HTTPException(status_code=404, detail=f"Folder {folder_id} not found") @@ -561,7 +469,6 @@ async def save_file( dataset_id: str, folder_id: Optional[str] = None, user=Depends(get_current_user), - db: MongoClient = Depends(dependencies.get_db), fs: Minio = Depends(dependencies.get_fs), file: UploadFile = File(...), es=Depends(dependencies.get_elasticsearchclient), @@ -575,14 +482,11 @@ async def save_file( status_code=401, detail=f"User not found. Session might have expired." ) - fileDB = FileDB(name=file.filename, creator=user, dataset_id=dataset.id) + new_file = FileDB(name=file.filename, creator=user, dataset_id=dataset.id) if folder_id is not None: - if ( - folder := await db["folders"].find_one({"_id": ObjectId(folder_id)}) - ) is not None: - folder = FolderOut.from_mongo(folder) - fileDB.folder_id = folder.id + if (folder := await FolderDB.get(PydanticObjectId(folder_id))) is not None: + new_file.folder_id = folder.id else: raise HTTPException( status_code=404, detail=f"Folder {folder_id} not found" @@ -590,9 +494,8 @@ async def save_file( access_token = credentials.credentials await add_file_entry( - fileDB, + new_file, user, - db, fs, es, rabbitmq_client, @@ -600,15 +503,13 @@ async def save_file( file.file, content_type=file.content_type, ) - - return fileDB + return new_file raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") @router.post("/createFromZip", response_model=DatasetOut) async def create_dataset_from_zip( user=Depends(get_current_user), - db: MongoClient = Depends(dependencies.get_db), fs: Minio = Depends(dependencies.get_fs), file: UploadFile = File(...), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), @@ -635,7 +536,7 @@ async def create_dataset_from_zip( # Create folders folder_lookup = await _create_folder_structure( - dataset.id, zip_directory, "", {}, user, db + dataset.id, zip_directory, "", {}, user ) # Go back through zipfile, this time uploading files to folders @@ -652,21 +553,20 @@ async def create_dataset_from_zip( extracted = zip_file.extract(entry, path="/tmp") if foldername in folder_lookup: folder_id = folder_lookup[foldername] - fileDB = FileDB( + new_file = FileDB( name=filename, creator=user, dataset_id=dataset.id, folder_id=folder_id, ) else: - fileDB = FileDB( + new_file = FileDB( name=filename, creator=user, dataset_id=dataset.id ) with open(extracted, "rb") as file_reader: await add_file_entry( - fileDB, + new_file, user, - db, fs, es, rabbitmq_client, @@ -676,19 +576,17 @@ async def create_dataset_from_zip( if os.path.isfile(extracted): os.remove(extracted) - return DatasetOut(**dataset.dict()) + return dataset.dict() @router.get("/{dataset_id}/download", response_model=DatasetOut) async def download_dataset( dataset_id: str, user=Depends(get_current_user), - db: MongoClient = Depends(dependencies.get_db), fs: Minio = Depends(dependencies.get_fs), allow: bool = Depends(Authorization("viewer")), ): if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None: - dataset = DatasetOut(**dataset) current_temp_dir = tempfile.mkdtemp(prefix="rocratedownload") crate = ROCrate() user_full_name = user.first_name + " " + user.last_name @@ -731,12 +629,11 @@ async def download_dataset( bag_size = 0 # bytes file_count = 0 - async for f in db["files"].find({"dataset_id": ObjectId(dataset_id)}): + async for file in FileDB.find(FileDB.dataset_id == ObjectId(dataset_id)): file_count += 1 - file = FileOut.from_mongo(f) file_name = file.name if file.folder_id is not None: - hierarchy = await _get_folder_hierarchy(file.folder_id, "", db) + hierarchy = await _get_folder_hierarchy(file.folder_id, "") dest_folder = os.path.join(current_temp_dir, hierarchy.lstrip("/")) if not os.path.isdir(dest_folder): os.mkdir(dest_folder) diff --git a/backend/app/routers/feeds.py b/backend/app/routers/feeds.py index 7af2a6bdd..5fa1aa869 100644 --- a/backend/app/routers/feeds.py +++ b/backend/app/routers/feeds.py @@ -80,7 +80,7 @@ async def save_feed( """Create a new Feed (i.e. saved search) in the database.""" feed = FeedDB(**feed_in.dict(), creator=user) await feed.save() - return FeedOut(**feed.dict()) + return feed.dict() @router.get("", response_model=List[FeedOut]) @@ -116,7 +116,7 @@ async def get_feed( ): """Fetch an existing saved search Feed.""" if (feed := await FeedDB.get(PydanticObjectId(feed_id))) is not None: - return FeedOut(**feed.dict()) + return feed.dict() else: raise HTTPException(status_code=404, detail=f"Feed {feed_id} not found") @@ -147,11 +147,11 @@ async def associate_listener( """ if (feed := await FeedDB.get(PydanticObjectId(feed_id))) is not None: if ( - exists := await EventListenerDB.get(PydanticObjectId(listener.listener_id)) + await EventListenerDB.get(PydanticObjectId(listener.listener_id)) ) is not None: feed.listeners.append(listener) await feed.save() - return FeedOut(**feed.dict()) + return feed.dict() raise HTTPException( status_code=404, detail=f"listener {listener.listener_id} not found" ) diff --git a/backend/app/routers/files.py b/backend/app/routers/files.py index 610680afa..5de7b3cb0 100644 --- a/backend/app/routers/files.py +++ b/backend/app/routers/files.py @@ -12,13 +12,11 @@ from fastapi import ( File, UploadFile, - Request, ) from fastapi.responses import StreamingResponse from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from minio import Minio from pika.adapters.blocking_connection import BlockingChannel -from pymongo import MongoClient from app import dependencies from app.config import settings @@ -59,12 +57,11 @@ async def _resubmit_file_extractors( Arguments: file_id: Id of file credentials: credentials of logged in user - db: MongoDB Client rabbitmq_client: Rabbitmq Client """ resubmitted_jobs = [] - jobs = EventListenerJobDB.find( + jobs = await EventListenerJobDB.find( EventListenerJobDB.resource_ref.resource_id == ObjectId(file.id), EventListenerJobDB.resource_ref.version == file.version_num - 1, ) @@ -91,9 +88,8 @@ async def _resubmit_file_extractors( # TODO: Move this to MongoDB middle layer async def add_file_entry( - file_db: FileDB, + new_file: FileDB, user: UserOut, - db: MongoClient, fs: Minio, es: Elasticsearch, rabbitmq_client: BlockingChannel, @@ -114,10 +110,10 @@ async def add_file_entry( raise HTTPException(status_code=503, detail="Service not available") return - new_file = await file_db.insert() + await new_file.insert() new_file_id = new_file.id if content_type is None: - content_type = mimetypes.guess_type(file_db.name) + content_type = mimetypes.guess_type(new_file.name) content_type = content_type[0] if len(content_type) > 1 else content_type type_main = content_type.split("/")[0] if type(content_type) is str else "N/A" content_type_obj = FileContentType(content_type=content_type, main_type=type_main) @@ -135,12 +131,11 @@ async def add_file_entry( if version_id is None: # TODO: This occurs in testing when minio is not running version_id = 999999999 - file_db.version_id = version_id - file_db.version_num = 1 - file_db.bytes = bytes - file_db.content_type = content_type_obj - await file_db.replace() - file_out = FileOut(**file_db.dict()) + new_file.version_id = version_id + new_file.version_num = 1 + new_file.bytes = bytes + new_file.content_type = content_type_obj + await new_file.replace() # Add FileVersion entry and update file new_version = FileVersionDB( @@ -153,20 +148,22 @@ async def add_file_entry( # Add entry to the file index doc = { - "name": file_db.name, - "creator": file_db.creator.email, - "created": file_db.created, - "download": file_db.downloads, - "dataset_id": str(file_db.dataset_id), - "folder_id": str(file_db.folder_id), - "bytes": file_db.bytes, + "name": new_file.name, + "creator": new_file.creator.email, + "created": new_file.created, + "download": new_file.downloads, + "dataset_id": str(new_file.dataset_id), + "folder_id": str(new_file.folder_id), + "bytes": new_file.bytes, "content_type": content_type_obj.content_type, "content_type_main": content_type_obj.main_type, } - insert_record(es, "file", doc, file_db.id) + insert_record(es, "file", doc, new_file.id) # Submit file job to any qualifying feeds - await check_feed_listeners(es, file_out, user, rabbitmq_client, token) + await check_feed_listeners( + es, FileOut(**new_file.dict()), user, rabbitmq_client, token + ) # TODO: Move this to MongoDB middle layer @@ -185,7 +182,7 @@ async def remove_file_entry( delete_document_by_id(es, "file", str(file_id)) query = {"match": {"resource_id": str(file_id)}} delete_document_by_query(es, "metadata", query) - await FileDB.find_one(FileDB.id == ObjectId(file_id)).delete_one() + await FileDB.get(PydanticObjectId(file_id)).delete() await MetadataDB.find(MetadataDB.resource.resource_id == ObjectId(file_id)).delete() await FileVersionDB.find(FileVersionDB.file_id == ObjectId(file_id)).delete() @@ -207,10 +204,7 @@ async def update_file( raise HTTPException(status_code=503, detail="Service not available") return - file_q = await FileDB.get(PydanticObjectId(file_id)) - if file_q is not None: - updated_file = FileOut(**file_q.dict()) - + if (updated_file := await FileDB.get(PydanticObjectId(file_id))) is not None: if ( file.filename != updated_file.name or file.content_type != updated_file.content_type.content_type @@ -221,7 +215,6 @@ async def update_file( ) # Update file in Minio and get the new version IDs - version_id = None response = fs.put_object( settings.MINIO_BUCKET_NAME, str(updated_file.id), @@ -285,7 +278,7 @@ async def update_file( } } update_record(es, "metadata", doc, str(metadata.id)) - return updated_file + return updated_file.dict() else: raise HTTPException(status_code=404, detail=f"File {file_id} not found") @@ -294,14 +287,11 @@ async def update_file( async def download_file( file_id: str, version: Optional[int] = None, - db: MongoClient = Depends(dependencies.get_db), fs: Minio = Depends(dependencies.get_fs), allow: bool = Depends(FileAuthorization("viewer")), ): # If file exists in MongoDB, download from Minio - file = await FileDB.get(PydanticObjectId(file_id)) - if file is not None: - file_obj = FileOut(**file.dict()) + if (file := await FileDB.get(PydanticObjectId(file_id))) is not None: if version is not None: # Version is specified, so get the minio ID from versions table if possible file_vers = await FileVersionDB.find_one( @@ -324,12 +314,9 @@ async def download_file( # Get content type & open file stream response = StreamingResponse(content.stream(settings.MINIO_UPLOAD_CHUNK_SIZE)) - response.headers["Content-Disposition"] = ( - "attachment; filename=%s" % file_obj.name - ) + response.headers["Content-Disposition"] = "attachment; filename=%s" % file.name # Increment download count - await file.update(Inc({FileDB.downloads: 1})) - + await file.update(Inc({FileDB.downloads, 1})) return response else: raise HTTPException(status_code=404, detail=f"File {file_id} not found") @@ -338,31 +325,27 @@ async def download_file( @router.delete("/{file_id}") async def delete_file( file_id: str, - db: MongoClient = Depends(dependencies.get_db), fs: Minio = Depends(dependencies.get_fs), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), allow: bool = Depends(FileAuthorization("editor")), ): - file = await FileDB.get(PydanticObjectId(file_id)) - if file is not None: + if (await FileDB.get(PydanticObjectId(file_id))) is not None: await remove_file_entry(file_id, fs, es) return {"deleted": file_id} else: raise HTTPException(status_code=404, detail=f"File {file_id} not found") -@router.get("/{file_id}/summary") +@router.get("/{file_id}/summary", response_model=FileOut) async def get_file_summary( file_id: str, - db: MongoClient = Depends(dependencies.get_db), allow: bool = Depends(FileAuthorization("viewer")), ): - file = await FileDB.get(PydanticObjectId(file_id)) - if file is not None: + if (file := await FileDB.get(PydanticObjectId(file_id))) is not None: # TODO: Incrementing too often (3x per page view) - # file["views"] += 1 - # db["files"].replace_one({"_id": ObjectId(file_id)}, file) - return FileOut(**file.dict()) + # file.views += 1 + # await file.replace() + return file.dict() else: raise HTTPException(status_code=404, detail=f"File {file_id} not found") @@ -438,7 +421,6 @@ async def resubmit_file_extractions( Arguments: file_id: Id of file credentials: credentials of logged in user - db: MongoDB Client rabbitmq_client: Rabbitmq Client """ diff --git a/backend/app/routers/groups.py b/backend/app/routers/groups.py index 1def6cdcb..bdb9fb473 100644 --- a/backend/app/routers/groups.py +++ b/backend/app/routers/groups.py @@ -6,9 +6,7 @@ from bson.objectid import ObjectId from fastapi import HTTPException, Depends, APIRouter from pymongo import DESCENDING -from pymongo.mongo_client import MongoClient -from app import dependencies from app.deps.authorization_deps import AuthorizationDB, GroupAuthorization from app.keycloak_auth import get_current_user, get_user from app.models.authorization import RoleType @@ -28,7 +26,7 @@ async def save_group( if user_member not in group_db.users: group_db.users.append(user_member) await group_db.insert() - return GroupOut(**group_db.dict()) + return group_db.dict() @router.get("", response_model=List[GroupOut]) @@ -89,7 +87,7 @@ async def get_group( allow: bool = Depends(GroupAuthorization("viewer")), ): if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: - return GroupOut(**group.dict()) + return group.dict() raise HTTPException(status_code=404, detail=f"Group {group_id} not found") @@ -120,7 +118,7 @@ async def edit_group( await group.replace() except Exception as e: raise HTTPException(status_code=500, detail=e.args[0]) - return GroupOut(**group.dict()) + return group.dict() raise HTTPException(status_code=404, detail=f"Group {group_id} not found") @@ -131,7 +129,7 @@ async def delete_group( ): if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: await group.delete() - return GroupOut(**group.dict()) + return group.dict() # TODO: Do we need to return what we just deleted? else: raise HTTPException(status_code=404, detail=f"Dataset {group_id} not found") @@ -144,10 +142,8 @@ async def add_member( allow: bool = Depends(GroupAuthorization("editor")), ): """Add a new user to a group.""" - user_q = await UserDB.find_one(UserDB.email == username) - if user_q is not None: - user_out = UserOut(**user_q.dict()) - new_member = Member(user=user_out) + if (user := await UserDB.find_one(UserDB.email == username)) is not None: + new_member = Member(user=UserOut(**user.dict())) if (group := await GroupDB.get(PydanticObjectId(group_id))) is not None: found_already = False for u in group.users: @@ -169,7 +165,7 @@ async def add_member( AuthorizationDB.group_ids == ObjectId(group_id), Push({AuthorizationDB.user_ids: username}), ) - return GroupOut(**group.dict()) + return group.dict() raise HTTPException(status_code=404, detail=f"Group {group_id} not found") raise HTTPException(status_code=404, detail=f"User {username} not found") @@ -202,7 +198,7 @@ async def remove_member( group.users.remove(found_user) await group.replace() - return GroupOut(**group.dict()) + return group.dict() raise HTTPException(status_code=404, detail=f"Group {group_id} not found") @@ -235,6 +231,6 @@ async def update_member( status_code=404, detail=f"User {username} does not belong to this group!", ) - return GroupOut(**group.dict()) + return group.dict() raise HTTPException(status_code=404, detail=f"Group {group_id} not found") raise HTTPException(status_code=404, detail=f"User {username} not found") diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py index 5b7080901..ac5a3304a 100644 --- a/backend/app/routers/listeners.py +++ b/backend/app/routers/listeners.py @@ -9,9 +9,7 @@ from bson import ObjectId from fastapi import APIRouter, HTTPException, Depends from packaging import version -from pymongo import MongoClient -from app.dependencies import get_db from app.keycloak_auth import get_user, get_current_user, get_current_username from app.models.config import ConfigEntryDB from app.models.feeds import FeedDB, FeedListener @@ -100,7 +98,7 @@ async def save_listener( listener = EventListenerDB(**listener_in.dict(), creator=user) # TODO: Check for duplicates somehow? await listener.save() - return EventListenerOut(**listener.dict()) + return listener.dict() @legacy_router.post("", response_model=EventListenerOut) @@ -125,10 +123,10 @@ async def save_legacy_listener( if version.parse(listener.version) > version.parse(existing.version): await listener.save() # TODO: Should older extractor version entries be deleted? - return EventListenerOut(**listener.dict()) + return listener.dict() else: # TODO: Should this fail the POST instead? - return EventListenerOut(**existing.dict()) + return listener.dict() else: # Register new listener await listener.save() @@ -137,7 +135,7 @@ async def save_legacy_listener( await _process_incoming_v1_extractor_info( legacy_in.name, listener.id, listener.properties.process ) - return EventListenerOut(**listener.dict()) + return listener.dict() @router.get("/search", response_model=List[EventListenerOut]) @@ -181,9 +179,9 @@ async def list_default_labels(user=Depends(get_current_username)): async def get_listener(listener_id: str, user=Depends(get_current_username)): """Return JSON information about an Event Listener if it exists.""" if ( - listener := EventListenerDB.find_one(PydanticObjectId(listener_id)) + listener := await EventListenerDB.find_one(PydanticObjectId(listener_id)) ) is not None: - return EventListenerOut(**listener.dict()) + return listener.dict() raise HTTPException(status_code=404, detail=f"listener {listener_id} not found") @@ -224,7 +222,9 @@ async def edit_listener( listener_id -- UUID of the listener to be udpated listener_in -- JSON object including updated information """ - listener = EventListenerDB.find_one(EventListenerDB.id == ObjectId(listener_id)) + listener = await EventListenerDB.find_one( + EventListenerDB.id == ObjectId(listener_id) + ) if listener: # TODO: Refactor this with permissions checks etc. listener_update = dict(listener_in) if listener_in is not None else {} @@ -232,7 +232,7 @@ async def edit_listener( try: listener.update(listener_update) await listener.save() - return EventListenerOut(**listener.dict()) + return listener.dict() except Exception as e: raise HTTPException(status_code=500, detail=e.args[0]) raise HTTPException(status_code=404, detail=f"listener {listener_id} not found") @@ -244,10 +244,10 @@ async def delete_listener( user=Depends(get_current_username), ): """Remove an Event Listener from the database. Will not clear event history for the listener.""" - listener = EventListenerDB.find(EventListenerDB.id == ObjectId(listener_id)) + listener = await EventListenerDB.find(EventListenerDB.id == ObjectId(listener_id)) if listener: # unsubscribe the listener from any feeds - feeds = FeedDB.find(FeedDB.listeners.listener_id == ObjectId(listener_id)) + feeds = await FeedDB.find(FeedDB.listeners.listener_id == ObjectId(listener_id)) for feed in feeds: await disassociate_listener_db(feed.id, listener_id) await listener.delete() diff --git a/backend/app/routers/metadata.py b/backend/app/routers/metadata.py index 6c613e725..ee2d3e1f1 100644 --- a/backend/app/routers/metadata.py +++ b/backend/app/routers/metadata.py @@ -40,7 +40,7 @@ async def save_metadata_definition( else: md_def = MetadataDefinitionDB(**definition_in.dict(), creator=user) await md_def.save() - return MetadataDefinitionOut(**md_def.dict()) + return md_def.dict() @router.get("/definition", response_model=List[MetadataDefinitionOut]) diff --git a/backend/app/routers/metadata_datasets.py b/backend/app/routers/metadata_datasets.py index 50e9b2cba..a48cfa528 100644 --- a/backend/app/routers/metadata_datasets.py +++ b/backend/app/routers/metadata_datasets.py @@ -111,7 +111,9 @@ async def add_dataset_metadata( raise HTTPException( 409, f"Metadata for {definition} already exists on this dataset" ) - md = await _build_metadata_db_obj(metadata_in, dataset, user) + md = await _build_metadata_db_obj( + metadata_in, DatasetOut(**dataset.dict()), user + ) await md.insert() # Add an entry to the metadata index @@ -130,7 +132,7 @@ async def add_dataset_metadata( } insert_record(es, "metadata", doc, md.id) - return MetadataOut(**md.dict()) + return md.dict() @router.put("/{dataset_id}/metadata", response_model=MetadataOut) @@ -152,7 +154,7 @@ async def replace_dataset_metadata( # Filter by MetadataAgent if metadata_in.extractor is not None: - listener = EventListenerDB.find_one( + listener = await EventListenerDB.find_one( EventListenerDB.name == metadata_in.extractor.name, EventListenerDB.version == metadata_in.extractor.version, ) @@ -179,12 +181,11 @@ async def replace_dataset_metadata( md = new_md md.id = tmp_md_id new_metadata = await md.replace() - metadata_out = MetadataOut(**new_metadata.dict()) # Update entry to the metadata index - doc = {"doc": {"content": metadata_out.content}} - update_record(es, "metadata", doc, metadata_out.id) - return metadata_out + doc = {"doc": {"content": new_metadata.content}} + update_record(es, "metadata", doc, new_metadata.id) + return new_metadata.dict() else: raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") @@ -230,7 +231,7 @@ async def update_dataset_metadata( # Filter by MetadataAgent if metadata_in.extractor is not None: - listener = EventListenerDB.find_one( + listener = await EventListenerDB.find_one( EventListenerDB.name == metadata_in.extractor.name, EventListenerDB.version == metadata_in.extractor.version, ) @@ -276,16 +277,15 @@ async def get_dataset_metadata( metadata = [] async for md in MetadataDB.find(*query): - md_out = MetadataOut(**md.dict()) - if md_out.definition is not None: - md_df = await MetadataDefinitionDB.find_one( - MetadataDefinitionDB.name == md_out.definition - ) - if md_df is not None: - md_def = MetadataDefinitionOut(**md_df.dict()) - md_out.description = md_def.description - metadata.append(md_out) - return metadata + if md.definition is not None: + if ( + md_def := await MetadataDefinitionDB.find_one( + MetadataDefinitionDB.name == md.definition + ) + ) is not None: + md.description = md_def.description + metadata.append(md) + return [md.dict() for md in metadata] else: raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") @@ -319,7 +319,7 @@ async def delete_dataset_metadata( # Filter by MetadataAgent extractor_info = metadata_in.extractor_info if extractor_info is not None: - listener = EventListenerDB.find_one( + listener = await EventListenerDB.find_one( EventListenerDB.name == metadata_in.extractor.name, EventListenerDB.version == metadata_in.extractor.version, ) @@ -341,8 +341,8 @@ async def delete_dataset_metadata( md = await MetadataDB.find_one(*query) if md is not None: - if await md.delete() is not None: - return MetadataOut(**md.dict()) + await md.delete() + return md.dict() # TODO: Do we need to return what we just deleted? else: raise HTTPException( status_code=404, detail=f"No metadata found with that criteria" diff --git a/backend/app/routers/metadata_files.py b/backend/app/routers/metadata_files.py index 394a4ff16..39ebc1fa8 100644 --- a/backend/app/routers/metadata_files.py +++ b/backend/app/routers/metadata_files.py @@ -1,5 +1,6 @@ from typing import Optional, List +from beanie import PydanticObjectId from bson import ObjectId from elasticsearch import Elasticsearch from fastapi import ( @@ -8,12 +9,11 @@ Depends, Form, ) -from pymongo import MongoClient from app import dependencies from app.deps.authorization_deps import FileAuthorization from app.keycloak_auth import get_current_user, UserOut -from app.models.files import FileOut +from app.models.files import FileOut, FileDB, FileVersionDB from app.models.listeners import EventListenerDB from app.models.metadata import ( MongoDBRef, @@ -34,7 +34,6 @@ async def _build_metadata_db_obj( - db: MongoClient, metadata_in: MetadataIn, file: FileOut, user: UserOut, @@ -55,8 +54,9 @@ async def _build_metadata_db_obj( file_version = metadata_in.file_version if file_version is not None: if ( - await db["file_versions"].find_one( - {"file_id": file.id, "version_num": file_version} + await FileVersionDB.find_one( + FileVersionDB.file_id == file.id, + FileVersionDB.version_num == file_version, ) ) is None: raise HTTPException( @@ -105,7 +105,6 @@ async def add_file_metadata( metadata_in: MetadataIn, file_id: str, user=Depends(get_current_user), - db: MongoClient = Depends(dependencies.get_db), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), allow: bool = Depends(FileAuthorization("uploader")), ): @@ -115,8 +114,7 @@ async def add_file_metadata( Returns: Metadata document that was added to database """ - if (file := await db["files"].find_one({"_id": ObjectId(file_id)})) is not None: - file = FileOut(**file) + if (file := await FileDB.get(PydanticObjectId(file_id))) is not None: current_file_version = file.version_num # if metadata does not already specify a file version # change metadata_in file version to match the current file version @@ -147,7 +145,7 @@ async def add_file_metadata( 409, f"Metadata for {definition} already exists on this file" ) - md = await _build_metadata_db_obj(metadata_in, file, user) + md = await _build_metadata_db_obj(metadata_in, FileOut(**file.dict()), user) await md.insert() # Add an entry to the metadata index @@ -168,7 +166,7 @@ async def add_file_metadata( "bytes": file.bytes, } insert_record(es, "metadata", doc, md.id) - return MetadataOut(**md.dict()) + return md.dict() @router.put("/{file_id}/metadata", response_model=MetadataOut) @@ -176,7 +174,6 @@ async def replace_file_metadata( metadata_in: MetadataPatch, file_id: str, user=Depends(get_current_user), - db: MongoClient = Depends(dependencies.get_db), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), allow: bool = Depends(FileAuthorization("editor")), ): @@ -185,16 +182,16 @@ async def replace_file_metadata( Returns: Metadata document that was updated """ - if (file := await db["files"].find_one({"_id": ObjectId(file_id)})) is not None: + if (file := await FileDB.get(PydanticObjectId(file_id))) is not None: # First, make sure the metadata we are replacing actually exists. query = [MetadataDB.resource.resource_id == ObjectId(file_id)] - file = FileOut(**file) version = metadata_in.file_version if version is not None: if ( - version_q := await db["file_versions"].find_one( - {"file_id": ObjectId(file_id), "version_num": version} + await FileVersionDB.find_one( + FileVersionDB.file_id == ObjectId(file_id), + FileVersionDB.version_num == version, ) ) is None: raise HTTPException( @@ -228,14 +225,18 @@ async def replace_file_metadata( if (md := await MetadataDB(*query).find_one()) is not None: # Metadata exists, so prepare the new document we are going to replace it with md = await _build_metadata_db_obj( - metadata_in, file, user, agent=agent, version=target_version + metadata_in, + FileOut(**file.dict()), + user, + agent=agent, + version=target_version, ) await md.save() # Update entry to the metadata index doc = {"doc": {"content": md.content}} update_record(es, "metadata", doc, md.id) - return MetadataOut(**md.dict()) + return md.dict() else: raise HTTPException(status_code=404, detail=f"No metadata found to update") else: @@ -247,7 +248,6 @@ async def update_file_metadata( metadata_in: MetadataPatch, file_id: str, user=Depends(get_current_user), - db: MongoClient = Depends(dependencies.get_db), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), allow: bool = Depends(FileAuthorization("editor")), ): @@ -260,17 +260,16 @@ async def update_file_metadata( # check if metadata with file version exists, replace metadata if none exists if ( - version_md := await MetadataDB( + await MetadataDB( MetadataDB.resource.resource_id == ObjectId(file_id), MetadataDB.resource.version == metadata_in.file_version, ).find_one() ) is None: - result = await replace_file_metadata(metadata_in, file_id, user, db, es) + result = await replace_file_metadata(metadata_in, file_id, user, es) return result - if (file := await db["files"].find_one({"_id": ObjectId(file_id)})) is not None: + if (file := await FileDB.get(PydanticObjectId(file_id))) is not None: query = [MetadataDB.resource.resource_id == ObjectId(file_id)] - file = FileOut(**file) content = metadata_in.content if metadata_in.metadata_id is not None: @@ -294,18 +293,18 @@ async def update_file_metadata( if definition is not None: query.append(MetadataDB.definition == definition) - version = metadata_in.file_version - if version is not None: + if metadata_in.file_version is not None: if ( - version_q := await db["file_versions"].find_one( - {"file_id": ObjectId(file_id), "version_num": version} + await FileVersionDB.find_one( + FileVersionDB.file_id == ObjectId(file_id), + FileVersionDB.version_num == metadata_in.file_version, ) ) is None: raise HTTPException( status_code=404, - detail=f"File version {version} does not exist", + detail=f"File version {metadata_in.file_version} does not exist", ) - target_version = version + target_version = metadata_in.file_version else: target_version = file.version_num query.append(MetadataDB.resource.version == target_version) @@ -351,20 +350,19 @@ async def get_file_metadata( extractor_name: Optional[str] = Form(None), extractor_version: Optional[float] = Form(None), user=Depends(get_current_user), - db: MongoClient = Depends(dependencies.get_db), allow: bool = Depends(FileAuthorization("viewer")), ): """Get file metadata.""" - if (file := await db["files"].find_one({"_id": ObjectId(file_id)})) is not None: + if (file := await FileDB.get(PydanticObjectId(file_id))) is not None: query = [MetadataDB.resource.resource_id == ObjectId(file_id)] - file = FileOut.from_mongo(file) # Validate specified version, or use latest by default if not all_versions: if version is not None: if ( - version_q := await db["file_versions"].find_one( - {"file_id": ObjectId(file_id), "version_num": version} + await FileVersionDB.find_one( + FileVersionDB.file_id == ObjectId(file_id), + FileVersionDB.version_num == version, ) ) is None: raise HTTPException( @@ -386,17 +384,16 @@ async def get_file_metadata( query.append(MetadataDB.agent.extractor.version == extractor_version) metadata = [] - async for md in db["metadata"].find(query): - md_out = MetadataOut.from_mongo(md) - if md_out.definition is not None: + async for md in MetadataDB.find(query): + if md.definition is not None: if ( md_def := await MetadataDefinitionDB( - MetadataDefinitionDB.name == md_out.definition + MetadataDefinitionDB.name == md.definition ) ) is not None: md_def = MetadataDefinitionOut(**md_def.dict()) - md_out.description = md_def.description - metadata.append(md_out) + md.description = md_def.description + metadata.append(md.dict()) return metadata else: raise HTTPException(status_code=404, detail=f"File {file_id} not found") @@ -408,19 +405,18 @@ async def delete_file_metadata( file_id: str, # version: Optional[int] = Form(None), user=Depends(get_current_user), - db: MongoClient = Depends(dependencies.get_db), es: Elasticsearch = Depends(dependencies.get_elasticsearchclient), allow: bool = Depends(FileAuthorization("editor")), ): - if (file := await db["files"].find_one({"_id": ObjectId(file_id)})) is not None: + if (file := await FileDB.get(PydanticObjectId(file_id))) is not None: query = [MetadataDB.resource.resource_id == ObjectId(file_id)] - file = FileOut.from_mongo(file) # # Validate specified version, or use latest by default # if version is not None: # if ( - # version_q := await db["file_versions"].find_one( - # {"file_id": ObjectId(file_id), "version_num": version} + # version_q := await FileVersionDB.find_one( + # FileVersionDB.file_id == ObjectId(file_id), + # FileVersionDB.version_num == version, # ) # ) is None: # raise HTTPException( @@ -436,7 +432,7 @@ async def delete_file_metadata( if metadata_in.metadata_id is not None: # If a specific metadata_id is provided, delete the matching entry if ( - existing_md := await MetadataDB( + await MetadataDB( MetadataDB.metadata_id == ObjectId(metadata_in.metadata_id) ).find_one() ) is not None: @@ -473,10 +469,8 @@ async def delete_file_metadata( delete_document_by_id(es, "metadata", str(metadata_in.id)) if (md := await MetadataDB(*query).find_one()) is not None: - metadata_deleted = md - # if await db["metadata"].delete_one({"_id": md["_id"]}) is not None: - if await MetadataDB(MetadataDB.id == md.id).delete_one() is not None: - return MetadataOut(**metadata_deleted.dict()) + await md.delete() + return md.dict() # TODO: Do we need to return the object we just deleted? else: raise HTTPException( status_code=404, detail=f"No metadata found with that criteria" diff --git a/backend/app/routers/users.py b/backend/app/routers/users.py index 62df9d54e..4a4093853 100644 --- a/backend/app/routers/users.py +++ b/backend/app/routers/users.py @@ -2,10 +2,10 @@ from secrets import token_urlsafe from typing import List +from beanie import PydanticObjectId from bson import ObjectId from fastapi import APIRouter, HTTPException, Depends from itsdangerous.url_safe import URLSafeSerializer -from pymongo import MongoClient, DESCENDING from app import dependencies from app.config import settings @@ -17,7 +17,6 @@ @router.get("/keys", response_model=List[UserAPIKeyOut]) async def get_user_api_keys( - db: MongoClient = Depends(dependencies.get_db), current_user=Depends(get_current_username), skip: int = 0, limit: int = 10, @@ -29,15 +28,14 @@ async def get_user_api_keys( limit: number to limit per page """ apikeys = [] - for doc in ( - await db["user_keys"] - .find({"user": current_user}) - .sort([("created", DESCENDING)]) + for key in ( + await UserAPIKey.find(UserAPIKey.user == current_user) + .sort(-UserAPIKey.created) .skip(skip) .limit(limit) .to_list(length=limit) ): - apikeys.append(UserAPIKeyOut.from_mongo(doc)) + apikeys.append(key.dict()) return apikeys @@ -68,7 +66,6 @@ async def generate_user_api_key( @router.delete("/keys/{key_id}", response_model=UserAPIKeyOut) async def delete_user_api_key( key_id: str, - db: MongoClient = Depends(dependencies.get_db), current_user=Depends(get_current_username), ): """Delete API keys given ID @@ -90,36 +87,29 @@ async def delete_user_api_key( @router.get("", response_model=List[UserOut]) -async def get_users( - db: MongoClient = Depends(dependencies.get_db), skip: int = 0, limit: int = 2 -): - return await UserDB.find({}, skip=skip, limit=limit).to_list() +async def get_users(skip: int = 0, limit: int = 2): + users = await UserDB.find({}, skip=skip, limit=limit).to_list() + return [user.dict() for user in users] @router.get("/profile", response_model=UserOut) async def get_profile( username=Depends(get_current_username), - db: MongoClient = Depends(dependencies.get_db), ): - user = await UserDB.find_one(UserDB.email == username) - if user is not None: - return UserOut(**user.dict()) + if (user := await UserDB.find_one(UserDB.email == username)) is not None: + return user.dict() raise HTTPException(status_code=404, detail=f"User {username} not found") @router.get("/{user_id}", response_model=UserOut) -async def get_user(user_id: str, db: MongoClient = Depends(dependencies.get_db)): - user = await UserDB.get(user_id) - if user is not None: - return UserOut(**user.dict()) +async def get_user(user_id: str): + if (user := await UserDB.get(PydanticObjectId(user_id))) is not None: + return user.dict() raise HTTPException(status_code=404, detail=f"User {user_id} not found") @router.get("/username/{username}", response_model=UserOut) -async def get_user_by_name( - username: str, db: MongoClient = Depends(dependencies.get_db) -): - user = UserDB.find_one(UserDB.email == username) - if user is not None: - return UserOut(**user.dict()) +async def get_user_by_name(username: str): + if (user := await UserDB.find_one(UserDB.email == username)) is not None: + return user.dict() raise HTTPException(status_code=404, detail=f"User {username} not found") diff --git a/backend/app/tests/conftest.py b/backend/app/tests/conftest.py index 0233f1288..cfc33cca3 100644 --- a/backend/app/tests/conftest.py +++ b/backend/app/tests/conftest.py @@ -3,20 +3,12 @@ from typing import Generator from fastapi.testclient import TestClient from app.config import settings -from app.dependencies import get_db from app.main import app from app.tests.utils import user_example -async def override_get_db() -> Generator: - mongo_client = motor.motor_asyncio.AsyncIOMotorClient(settings.MONGODB_URL) - db = mongo_client["clowder-tests"] - yield db - - @pytest.fixture(scope="session") def client() -> Generator: - app.dependency_overrides[get_db] = override_get_db with TestClient(app) as c: yield c diff --git a/frontend/src/openapi/v2/models/FileOut.ts b/frontend/src/openapi/v2/models/FileOut.ts index e54f16035..185ff3ee6 100644 --- a/frontend/src/openapi/v2/models/FileOut.ts +++ b/frontend/src/openapi/v2/models/FileOut.ts @@ -20,7 +20,7 @@ import type { UserOut } from './UserOut'; */ export type FileOut = { name?: string; - _id?: string; + id?: string; creator: UserOut; created?: string; version_id?: string; diff --git a/frontend/src/openapi/v2/models/FolderOut.ts b/frontend/src/openapi/v2/models/FolderOut.ts index 162bad076..d2ba5771b 100644 --- a/frontend/src/openapi/v2/models/FolderOut.ts +++ b/frontend/src/openapi/v2/models/FolderOut.ts @@ -19,7 +19,7 @@ import type { UserOut } from './UserOut'; */ export type FolderOut = { name?: string; - _id?: string; + id?: string; dataset_id: string; parent_folder?: string; creator: UserOut; diff --git a/frontend/src/openapi/v2/services/DatasetsService.ts b/frontend/src/openapi/v2/services/DatasetsService.ts index fdb35ea11..06283f272 100644 --- a/frontend/src/openapi/v2/services/DatasetsService.ts +++ b/frontend/src/openapi/v2/services/DatasetsService.ts @@ -202,18 +202,24 @@ export class DatasetsService { * Get Dataset Folders * @param datasetId * @param parentFolder - * @returns any Successful Response + * @param skip + * @param limit + * @returns FolderOut Successful Response * @throws ApiError */ public static getDatasetFoldersApiV2DatasetsDatasetIdFoldersGet( datasetId: string, parentFolder?: string, - ): CancelablePromise { + skip?: number, + limit: number = 10, + ): CancelablePromise> { return __request({ method: 'GET', path: `/api/v2/datasets/${datasetId}/folders`, query: { 'parent_folder': parentFolder, + 'skip': skip, + 'limit': limit, }, errors: { 422: `Validation Error`, diff --git a/frontend/src/openapi/v2/services/FilesService.ts b/frontend/src/openapi/v2/services/FilesService.ts index 8187a19f3..4e4ea5824 100644 --- a/frontend/src/openapi/v2/services/FilesService.ts +++ b/frontend/src/openapi/v2/services/FilesService.ts @@ -75,12 +75,12 @@ export class FilesService { /** * Get File Summary * @param fileId - * @returns any Successful Response + * @returns FileOut Successful Response * @throws ApiError */ public static getFileSummaryApiV2FilesFileIdSummaryGet( fileId: string, - ): CancelablePromise { + ): CancelablePromise { return __request({ method: 'GET', path: `/api/v2/files/${fileId}/summary`, @@ -152,7 +152,6 @@ export class FilesService { * Arguments: * file_id: Id of file * credentials: credentials of logged in user - * db: MongoDB Client * rabbitmq_client: Rabbitmq Client * @param fileId * @returns any Successful Response