Beanie - Listeners, Metadata

backend/app/beanie_views_test.py

@@ -85,7 +85,7 @@ async def example():
     # print(results)
 
     results = await DatasetDBViewList.find(
-        DatasetDBViewList.author.email == "[email protected]"
+        DatasetDBViewList.creator.email == "[email protected]"
     ).to_list()
     print(results)
 

backend/app/deps/authorization_deps.py

@@ -1,3 +1,4 @@
+from beanie import PydanticObjectId
 from beanie.operators import Or
 from bson import ObjectId
 from fastapi import Depends, HTTPException
@@ -6,16 +7,15 @@
 from app.dependencies import get_db
 from app.keycloak_auth import get_current_username
 from app.models.authorization import RoleType, AuthorizationDB
-from app.models.datasets import DatasetOut
+from app.models.datasets import DatasetDB
 from app.models.files import FileOut
 from app.models.groups import GroupOut
-from app.models.metadata import MetadataOut
+from app.models.metadata import MetadataDB
 from app.models.pyobjectid import PyObjectId
 
 
 async def get_role(
     dataset_id: str,
-    db: MongoClient = Depends(get_db),
     current_user=Depends(get_current_username),
 ) -> RoleType:
     """Returns the role a specific user has on a dataset. If the user is a creator (owner), they are not listed in
@@ -28,16 +28,6 @@ async def get_role(
         ),
     )
     return authorization.role
-    # authorization = await db["authorization"].find_one(
-    #     {
-    #         "$and": [
-    #             {"dataset_id": ObjectId(dataset_id)},
-    #             {"$or": [{"creator": current_user}, {"user_ids": current_user}]},
-    #         ]
-    #     }
-    # )
-    # role = AuthorizationDB.from_mongo(authorization).role
-    # return role
 
 
 async def get_role_by_file(
@@ -55,17 +45,6 @@ async def get_role_by_file(
             ),
         )
         return authorization.role
-        # authorization = await db["authorization"].find_one(
-        #     {
-        #         "$and": [
-        #             {"dataset_id": ObjectId(file_out.dataset_id)},
-        #             {"$or": [{"creator": current_user}, {"user_ids": current_user}]},
-        #         ]
-        #     }
-        # )
-        # role = AuthorizationDB.from_mongo(authorization).role
-        # return role
-
     raise HTTPException(status_code=404, detail=f"File {file_id} not found")
 
 
@@ -74,10 +53,7 @@ async def get_role_by_metadata(
     db: MongoClient = Depends(get_db),
     current_user=Depends(get_current_username),
 ) -> RoleType:
-    if (
-        metadata := await db["metadata"].find_one({"_id": ObjectId(metadata_id)})
-    ) is not None:
-        md_out = MetadataOut.from_mongo(metadata)
+    if (md_out := await MetadataDB.get(PydanticObjectId(metadata_id))) is not None:
         resource_type = md_out.resource.collection
         resource_id = md_out.resource.resource_id
         if resource_type == "files":
@@ -93,49 +69,19 @@ async def get_role_by_metadata(
                     ),
                 )
                 return authorization.role
-                # authorization = await db["authorization"].find_one(
-                #     {
-                #         "$and": [
-                #             {"dataset_id": ObjectId(file_out.dataset_id)},
-                #             {
-                #                 "$or": [
-                #                     {"creator": current_user},
-                #                     {"user_ids": current_user},
-                #                 ]
-                #             },
-                #         ]
-                #     }
-                # )
-                # role = AuthorizationDB.from_mongo(authorization).role
-                # return role
         elif resource_type == "datasets":
+
             if (
-                dataset := await db["datasets"].find_one({"_id": ObjectId(resource_id)})
+                dataset := await DatasetDB.get(PydanticObjectId(resource_id))
             ) is not None:
-                dataset_out = DatasetOut.from_mongo(dataset)
                 authorization = await AuthorizationDB.find_one(
-                    AuthorizationDB.dataset_id == dataset_out.dataset_id,
+                    AuthorizationDB.dataset_id == dataset.id,
                     Or(
                         AuthorizationDB.creator == current_user,
                         AuthorizationDB.user_ids == current_user,
                     ),
                 )
                 return authorization.role
-                # authorization = await db["authorization"].find_one(
-                #     {
-                #         "$and": [
-                #             {"dataset_id": ObjectId(dataset_out.dataset_id)},
-                #             {
-                #                 "$or": [
-                #                     {"creator": current_user},
-                #                     {"user_ids": current_user},
-                #                 ]
-                #             },
-                #         ]
-                #     }
-                # )
-                # role = AuthorizationDB.from_mongo(authorization).role
-                # return role
 
 
 async def get_role_by_group(
@@ -182,22 +128,6 @@ async def __call__(
                 AuthorizationDB.user_ids == current_user,
             ),
         )
-        # if (
-        #         authorization_q := await db["authorization"].find_one(
-        #             {
-        #                 "$and": [
-        #                     {"dataset_id": ObjectId(dataset_id)},
-        #                     {
-        #                         "$or": [
-        #                             {"creator": current_user},
-        #                             {"user_ids": current_user},
-        #                         ]
-        #                     },
-        #                 ]
-        #             }
-        #         )
-        # ) is not None:
-        #     authorization = AuthorizationDB.from_mongo(authorization_q)
         if authorization is not None:
             if access(authorization.role, self.role):
                 return True
@@ -235,22 +165,6 @@ async def __call__(
                     AuthorizationDB.user_ids == current_user,
                 ),
             )
-            # if (
-            #         authorization_q := await db["authorization"].find_one(
-            #             {
-            #                 "$and": [
-            #                     {"dataset_id": ObjectId(file_out.dataset_id)},
-            #                     {
-            #                         "$or": [
-            #                             {"creator": current_user},
-            #                             {"user_ids": current_user},
-            #                         ]
-            #                     },
-            #                 ]
-            #             }
-            #         )
-            # ) is not None:
-            #     authorization = AuthorizationDB.from_mongo(authorization_q)
             if authorization is not None:
                 if access(authorization.role, self.role):
                     return True
@@ -275,10 +189,7 @@ async def __call__(
         db: MongoClient = Depends(get_db),
         current_user: str = Depends(get_current_username),
     ):
-        if (
-            metadata := await db["metadata"].find_one({"_id": ObjectId(metadata_id)})
-        ) is not None:
-            md_out = MetadataOut.from_mongo(metadata)
+        if (md_out := await MetadataDB.get(PydanticObjectId(metadata_id))) is not None:
             resource_type = md_out.resource.collection
             resource_id = md_out.resource.resource_id
             if resource_type == "files":
@@ -293,22 +204,6 @@ async def __call__(
                             AuthorizationDB.user_ids == current_user,
                         ),
                     )
-                    # if (
-                    #         authorization_q := await db["authorization"].find_one(
-                    #             {
-                    #                 "$and": [
-                    #                     {"dataset_id": ObjectId(file_out.dataset_id)},
-                    #                     {
-                    #                         "$or": [
-                    #                             {"creator": current_user},
-                    #                             {"user_ids": current_user},
-                    #                         ]
-                    #                     },
-                    #                 ]
-                    #             }
-                    #         )
-                    # ) is not None:
-                    #     authorization = AuthorizationDB.from_mongo(authorization_q)
                     if authorization is not None:
                         if access(authorization.role, self.role):
                             return True
@@ -322,34 +217,16 @@ async def __call__(
                         )
             elif resource_type == "datasets":
                 if (
-                    dataset := await db["datasets"].find_one(
-                        {"_id": ObjectId(resource_id)}
-                    )
-                ) is not None:
-                    dataset_out = DatasetOut.from_mongo(dataset)
+                    dataset_out := await DatasetDB.get(PydanticObjectId(resource_id))
+                    is not None
+                ):
                     authorization = await AuthorizationDB.find_one(
                         AuthorizationDB.dataset_id == dataset_out.dataset_id,
                         Or(
                             AuthorizationDB.creator == current_user,
                             AuthorizationDB.user_ids == current_user,
                         ),
                     )
-                    # if (
-                    #         authorization_q := await db["authorization"].find_one(
-                    #             {
-                    #                 "$and": [
-                    #                     {"dataset_id": ObjectId(dataset_out.id)},
-                    #                     {
-                    #                         "$or": [
-                    #                             {"creator": current_user},
-                    #                             {"user_ids": current_user},
-                    #                         ]
-                    #                     },
-                    #                 ]
-                    #             }
-                    #         )
-                    # ) is not None:
-                    #     authorization = AuthorizationDB.from_mongo(authorization_q)
                     if authorization is not None:
                         if access(authorization.role, self.role):
                             return True

backend/app/main.py

@@ -11,6 +11,14 @@
 from app.keycloak_auth import get_current_username
 from app.models.authorization import AuthorizationDB
 from app.models.datasets import DatasetDB, DatasetDBViewList
+from app.models.feeds import FeedDB
+from app.models.listeners import (
+    EventListenerDB,
+    EventListenerJobDB,
+    EventListenerJobUpdateDB,
+    EventListenerJobViewList,
+    EventListenerJobUpdateViewList,
+)
 from app.models.metadata import MetadataDB, MetadataDefinitionDB
 from app.routers import (
     folders,
@@ -173,6 +181,12 @@ async def startup_beanie():
             AuthorizationDB,
             MetadataDB,
             MetadataDefinitionDB,
+            FeedDB,
+            EventListenerDB,
+            EventListenerJobDB,
+            EventListenerJobUpdateDB,
+            EventListenerJobViewList,
+            EventListenerJobUpdateViewList,
         ],
         recreate_views=True,
     )

backend/app/models/authorization.py

@@ -35,10 +35,17 @@ class Config:
         use_enum_values = True
 
 
-class AuthorizationDB(MongoModel, AuthorizationBase):
+class Provenance:
     pass
 
 
+class AuthorizationDB(Document, AuthorizationBase, Provenance):
+    """The creator of the Authorization object should also be the creator of the dataset itself."""
+
+    class Settings:
+        name = "authorization"
+
+
 class AuthorizationOut(AuthorizationDB):
     pass
 
@@ -75,10 +82,3 @@ class Provenance(BaseModel):
     creator: EmailStr
     created: datetime = Field(default_factory=datetime.utcnow)
     modified: datetime = Field(default_factory=datetime.utcnow)
-
-
-class AuthorizationDB(Document, AuthorizationBase, Provenance):
-    """The creator of the Authorization object should also be the creator of the dataset itself."""
-
-    class Settings:
-        name = "authorization"

backend/app/models/config.py

@@ -1,10 +1,19 @@
-from app.models.mongomodel import MongoModel
+import pymongo
+from beanie import Document
 
 
-class ConfigEntryBase(MongoModel):
+class ConfigEntryBase(Document):
     key: str
     value: str
 
+    class Settings:
+        name = "config"
+        indexes = [
+            [
+                ("key", pymongo.TEXT),
+            ],
+        ]
+
 
 class ConfigEntryDB(ConfigEntryBase):
     pass

backend/app/models/datasets.py

@@ -39,7 +39,8 @@ class DatasetPatch(BaseModel):
 
 
 class DatasetDB(Document, DatasetBase):
-    author: UserOut
+    id: PydanticObjectId = Field(None, alias="_id")
+    creator: UserOut
     created: datetime = Field(default_factory=datetime.utcnow)
     modified: datetime = Field(default_factory=datetime.utcnow)
     status: str = DatasetStatus.PRIVATE.name
@@ -59,7 +60,7 @@ class Settings:
 class DatasetDBViewList(View, DatasetBase):
     # FIXME This seems to be required to return _id. Otherwise _id is null in the response.
     id: PydanticObjectId = Field(None, alias="_id")
-    author: UserOut
+    creator: UserOut
     created: datetime = Field(default_factory=datetime.utcnow)
     modified: datetime = Field(default_factory=datetime.utcnow)
     auth: List[AuthorizationDB]

backend/app/models/feeds.py

@@ -1,10 +1,12 @@
-from datetime import datetime
-from pydantic import Field, BaseModel
-from typing import Optional, List, Union
-from app.models.mongomodel import MongoModel
-from app.models.users import UserOut
+from typing import List
+
+import pymongo
+from beanie import Document, PydanticObjectId
+from pydantic import BaseModel, Field
+
+from app.models.authorization import Provenance
+from app.models.listeners import FeedListener
 from app.models.search import SearchObject
-from app.models.listeners import EventListenerOut, FeedListener
 
 
 class JobFeed(BaseModel):
@@ -24,9 +26,17 @@ class FeedIn(JobFeed):
     pass
 
 
-class FeedDB(JobFeed, MongoModel):
-    author: Optional[UserOut] = None
-    updated: datetime = Field(default_factory=datetime.utcnow)
+class FeedDB(Document, JobFeed, Provenance):
+    id: PydanticObjectId = Field(None, alias="_id")
+
+    class Settings:
+        name = "feeds"
+        indexes = [
+            [
+                ("name", pymongo.TEXT),
+                ("description", pymongo.TEXT),
+            ],
+        ]
 
 
 class FeedOut(FeedDB):