491 beanie for file and folder (#496)

* import container (#462)

* 446 gui to manage api keys (#465)

* add new routes placeholder

* add list and delete endpoint; inline docs

* codegen and pytest

* ugly ui?

* onstart clear user keys

* delete works but pytest doesn't work yet

* pytest all works

* codegen

* redux

* reducer and types

* write the table

* correctly wire in the apikey and error handling

* wire in the delete modal of apikeys

* wire in the creation api key

* everything working in the gui

* formatting

---------

Co-authored-by: toddn <[email protected]>

* Sharing tab cleanup (#430)

* fix bug where users not being fetched

* simplifying role query pt 1

* refactor tables and API calls

* cleanup

* start moving GroupAndRole to separate table

* add expanding sub table

* remove AddGroup buttons for now

* formatting

* codegen

* add response model

* codegen

* remove container

* fix bug with Roles return object and autocomplete on groups

* codegen

* Fix reload bug (#451)

* 461 fix library version in pipfile (#468)

* fix pipfile version

* regenerate the piplock file

* 443 frontend need to display more verbose backend error (#458)

* Updated frontend error message to be more verbose

* Backend returns verbose logs, frontend displays it as generic messages depending on the HTTP status code

* Updated the error message logic to include original backend logs wherever possible

* Fixed report button issue

* Rootcause: The onClick event for the report button has a semantic error, which is causing the issue reason string to be replaced with an object

* Also added encode for the reason

* Fixing issue with error message

* Revert "Updated frontend error message to be more verbose"

This reverts commit 7d230d3.

* Fixing issue with error message

* 469 create profile page (#471)

* profile page exists

* new route for user profile
profile page includes layout

* codegen new route for get current user profile

* we now get the profile, but nothing shows yet

* page has fake data

* we see the profile now

* formatting

* Implemented role delete in sharing tab (#472)

* Tested by deleting a user, group. Verified entry is removed upon refresh

* Show Creator on Group Page (#428)

* the owner is visible, and we cannot change the owner in the table

* showing group creator on top

* creator link is wrong, need to fix

* bold text for word 'creator'

* using more conditionals to reduce duplicate code

* missing modal added
creator name matches other names

---------

Co-authored-by: Chen Wang <[email protected]>

* Fixed bug where error detail might be empty (#478)

* Fixed bug where error detail might be empty

* Added null check to the detail string to prevent errors when detail is not set

* Second check to validate the body field

* filter the option without group owner

* filter group owner out

* Made the updaterole function async, and added a call to fetch roles to refresh the list of roles in state

* Updated remove role logic to automatically refresh the roles list

* Executed eslint

* remove _beanie

* wire in folder db

* download folder

* refactored everything in file/folder

* reformat

* missed a spot

* codegen

* forget to add FileVersionDB

* add await

* fix more merge conflict

* duplicate

* fix inc

* black fromat

* codegen

* fix inconsistency

* black

* fix frontend author/creator

* fix detail

* fix some of the async bug

* download works

* init & use TokenDB everywhere

* fix files/submit endpoint

* remove manual index creation

* fix extractor submission

---------

Co-authored-by: toddn <[email protected]>
Co-authored-by: Max Burnette <[email protected]>
Co-authored-by: Aruna Parameswaran <[email protected]>
Co-authored-by: Todd Nicholson <[email protected]>

Author: 4 people

backend/app/database/errors.py

@@ -6,7 +6,6 @@
 from pymongo import MongoClient
 
 from app.config import settings
-from app.mongo import create_mongo_indexes
 from app.models.errors import Error
 from app.models.mongomodel import MongoDBRef
 
@@ -17,7 +16,6 @@ async def _get_db() -> Generator:
     """Duplicate of app.dependencies.get_db(), but importing that causes circular import."""
     mongo_client = motor.motor_asyncio.AsyncIOMotorClient(settings.MONGODB_URL)
     db = mongo_client[settings.MONGO_DATABASE]
-    await create_mongo_indexes(db)
     yield db
 
 

backend/app/dependencies.py

@@ -12,14 +12,12 @@
 from app.config import settings
 from minio.commonconfig import ENABLED
 from minio.versioningconfig import VersioningConfig
-from app.mongo import create_mongo_indexes
 from app.search.connect import connect_elasticsearch
 
 
 async def get_db() -> Generator:
     mongo_client = motor.motor_asyncio.AsyncIOMotorClient(settings.MONGODB_URL)
     db = mongo_client[settings.MONGO_DATABASE]
-    await create_mongo_indexes(db)
     if db is None:
         raise HTTPException(status_code=503, detail="Service not available")
         return

backend/app/deps/authorization_deps.py

@@ -70,7 +70,6 @@ async def get_role_by_metadata(
                 )
                 return authorization.role
         elif resource_type == "datasets":
-
             if (
                 dataset := await DatasetDB.get(PydanticObjectId(resource_id))
             ) is not None:

backend/app/keycloak_auth.py

@@ -17,6 +17,7 @@
 
 from . import dependencies
 from .config import settings
+from .models.tokens import TokenDB
 from .models.users import UserOut, UserAPIKey, UserDB
 
 logger = logging.getLogger(__name__)
@@ -144,8 +145,8 @@ async def get_current_user(
     if token:
         try:
             userinfo = keycloak_openid.userinfo(token)
-            user_out = await db["users"].find_one({"email": userinfo["email"]})
-            return UserOut.from_mongo(user_out)
+            user = await UserDB.find_one({"email": userinfo["email"]})
+            return UserOut(**user.dict())
         except KeycloakAuthenticationError as e:
             raise HTTPException(
                 status_code=e.response_code,
@@ -296,18 +297,16 @@ async def create_user(email: str, password: str, firstName: str, lastName: str):
 async def retreive_refresh_token(
     email: str, db: MongoClient = Depends(dependencies.get_db)
 ):
-    if (token_exist := await db["tokens"].find_one({"email": email})) is not None:
+    if (token_exist := await TokenDB.find_one({"email": email})) is not None:
         try:
             new_tokens = keycloak_openid.refresh_token(token_exist["refresh_token"])
             # update the refresh token in the database
             token_exist.update({"refresh_token": new_tokens["refresh_token"]})
-            await db["tokens"].replace_one(
-                {"_id": ObjectId(token_exist["_id"])}, token_exist
-            )
+            await token_exist.save()
             return {"access_token": new_tokens["access_token"]}
         except KeycloakGetError as e:
             # refresh token invalid; remove from database
-            await db["tokens"].delete_one({"_id": ObjectId(token_exist["_id"])})
+            await token_exist.delete()
             raise HTTPException(
                 status_code=401,
                 detail=str(e),  # "Invalid authentication credentials",

backend/app/main.py

@@ -10,10 +10,12 @@
 from app.config import settings
 from app.keycloak_auth import get_current_username
 from app.models.authorization import AuthorizationDB
-from app.models.users import UserDB, UserAPIKey
-from app.models.groups import GroupDB
+from app.models.config import ConfigEntryDB
 from app.models.datasets import DatasetDB, DatasetDBViewList
 from app.models.feeds import FeedDB
+from app.models.files import FileDB, FileVersionDB
+from app.models.folders import FolderDB
+from app.models.groups import GroupDB
 from app.models.listeners import (
     EventListenerDB,
     EventListenerJobDB,
@@ -22,6 +24,8 @@
     EventListenerJobUpdateViewList,
 )
 from app.models.metadata import MetadataDB, MetadataDefinitionDB
+from app.models.tokens import TokenDB
+from app.models.users import UserDB, UserAPIKey
 from app.routers import (
     folders,
     groups,
@@ -179,11 +183,15 @@ async def startup_beanie():
         database=getattr(client, settings.MONGO_DATABASE),
         # Make sure to include all models. If one depends on another that is not in the list it is not clear which one is missing.
         document_models=[
+            ConfigEntryDB,
             DatasetDB,
             DatasetDBViewList,
             AuthorizationDB,
             MetadataDB,
             MetadataDefinitionDB,
+            FolderDB,
+            FileDB,
+            FileVersionDB,
             FeedDB,
             EventListenerDB,
             EventListenerJobDB,
@@ -193,6 +201,7 @@ async def startup_beanie():
             UserDB,
             UserAPIKey,
             GroupDB,
+            TokenDB,
         ],
         recreate_views=True,
     )

backend/app/models/config.py

@@ -1,11 +1,14 @@
 import pymongo
 from beanie import Document
+from pydantic import BaseModel
 
 
-class ConfigEntryBase(Document):
+class ConfigEntryBase(BaseModel):
     key: str
     value: str
 
+
+class ConfigEntryDB(Document, ConfigEntryBase):
     class Settings:
         name = "config"
         indexes = [
@@ -15,9 +18,5 @@ class Settings:
         ]
 
 
-class ConfigEntryDB(ConfigEntryBase):
-    pass
-
-
-class ConfigEntryOut(ConfigEntryBase):
+class ConfigEntryOut(ConfigEntryDB):
     pass

backend/app/models/files.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 from typing import Optional
 
+from beanie import Document, PydanticObjectId
 from pydantic import Field, BaseModel
 
 from app.models.mongomodel import MongoModel
@@ -28,6 +29,13 @@ class FileVersion(MongoModel):
     bytes: int = 0
 
 
+class FileVersionDB(Document, FileVersion):
+    id: PydanticObjectId = Field(None, alias="_id")
+
+    class Settings:
+        name = "file_versions"
+
+
 class FileBase(MongoModel):
     name: str = "N/A"
 
@@ -36,7 +44,8 @@ class FileIn(FileBase):
     pass
 
 
-class FileDB(FileBase):
+class FileDB(Document, FileBase):
+    id: PydanticObjectId = Field(None, alias="_id")
     creator: UserOut
     created: datetime = Field(default_factory=datetime.utcnow)
     version_id: str = "N/A"
@@ -48,6 +57,9 @@ class FileDB(FileBase):
     bytes: int = 0
     content_type: FileContentType = FileContentType()
 
+    class Settings:
+        name = "files"
+
 
 class FileOut(FileDB):
     pass

backend/app/models/folders.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 from typing import Optional
 
+from beanie import Document, PydanticObjectId
 from pydantic import Field
 
 from app.models.mongomodel import MongoModel
@@ -16,13 +17,17 @@ class FolderIn(FolderBase):
     parent_folder: Optional[PyObjectId]
 
 
-class FolderDB(FolderBase):
+class FolderDB(Document, FolderBase):
+    id: PydanticObjectId = Field(None, alias="_id")
     dataset_id: PyObjectId
     parent_folder: Optional[PyObjectId]
     creator: UserOut
     created: datetime = Field(default_factory=datetime.utcnow)
     modified: datetime = Field(default_factory=datetime.utcnow)
 
+    class Settings:
+        name = "folders"
+
 
 class FolderOut(FolderDB):
     pass

backend/app/models/tokens.py

@@ -1,5 +1,7 @@
-from app.models.mongomodel import MongoModel, BaseModel
-from beanie import Document, View, PydanticObjectId
+import pymongo
+from beanie import Document
+
+from app.models.mongomodel import BaseModel
 
 
 class TokenBase(BaseModel):
@@ -8,4 +10,10 @@ class TokenBase(BaseModel):
 
 
 class TokenDB(Document, TokenBase):
-    pass
+    class Settings:
+        name = "tokens"
+        indexes = [
+            [
+                ("email", pymongo.TEXT),
+            ],
+        ]