diff --git a/.circleci/config.yml b/.circleci/config.yml index 27eb017..34abae6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: MINIO_SECRET_KEY: minioSecretKey - image: arangodb/arangodb:3.5.6 environment: - ARANGO_ROOT_PASSWORD=letmein + ARANGO_ROOT_PASSWORD: letmein steps: - checkout - run: @@ -31,6 +31,7 @@ jobs: DJANGO_MINIO_STORAGE_SECRET_KEY: minioSecretKey DJANGO_MULTINET_ARANGO_URL: http://localhost:8529 DJANGO_MULTINET_ARANGO_PASSWORD: letmein + DJANGO_MULTINET_ARANGO_READONLY_PASSWORD: letmein workflows: version: 2 ci: diff --git a/Procfile b/Procfile index 8f7a9e8..ce9c55e 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ -release: ./manage.py migrate +release: ./manage.py migrate && ./manage.py createarangoreadonlyuser web: gunicorn --bind 0.0.0.0:$PORT multinet.wsgi worker: REMAP_SIGTERM=SIGQUIT celery --app multinet.celery worker --loglevel INFO --without-heartbeat diff --git a/dev/.env.docker-compose b/dev/.env.docker-compose index 5234ae7..29dd5f7 100644 --- a/dev/.env.docker-compose +++ b/dev/.env.docker-compose @@ -8,3 +8,4 @@ DJANGO_STORAGE_BUCKET_NAME=django-storage DJANGO_MINIO_STORAGE_MEDIA_URL=http://localhost:9000/django-storage DJANGO_MULTINET_ARANGO_URL=http://arangodb:8529 DJANGO_MULTINET_ARANGO_PASSWORD=letmein +DJANGO_MULTINET_ARANGO_READONLY_PASSWORD=letmein diff --git a/dev/.env.docker-compose-native b/dev/.env.docker-compose-native index c80606f..04c2ae7 100644 --- a/dev/.env.docker-compose-native +++ b/dev/.env.docker-compose-native @@ -7,3 +7,4 @@ DJANGO_MINIO_STORAGE_SECRET_KEY=minioSecretKey DJANGO_STORAGE_BUCKET_NAME=django-storage DJANGO_MULTINET_ARANGO_URL=http://localhost:8529 DJANGO_MULTINET_ARANGO_PASSWORD=letmein +DJANGO_MULTINET_ARANGO_READONLY_PASSWORD=letmein diff --git a/multinet/api/management/__init__.py b/multinet/api/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/multinet/api/management/commands/__init__.py b/multinet/api/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/multinet/api/management/commands/createarangoreadonlyuser.py b/multinet/api/management/commands/createarangoreadonlyuser.py new file mode 100644 index 0000000..e71b686 --- /dev/null +++ b/multinet/api/management/commands/createarangoreadonlyuser.py @@ -0,0 +1,58 @@ +from arango.exceptions import ( + ArangoClientError, + ArangoServerError, + PermissionUpdateError, + UserCreateError, +) +from django.conf import settings +from django.core.management.base import BaseCommand + +from multinet.api.utils.arango import arango_system_db + +READONLY_USERNAME = 'readonly' + + +class Command(BaseCommand): + help = ( + 'Ensure the existence of a user with universal read-only priveleges on the' + 'Arango DB server. Replaces the readonly user with a new one if it already exists.' + ) + + def handle(self, *args, **kwargs): + try: + password = settings.MULTINET_ARANGO_READONLY_PASSWORD + system_db = arango_system_db(readonly=False) + readonly_user_exists = system_db.has_user(READONLY_USERNAME) + + if not readonly_user_exists: + system_db.create_user(READONLY_USERNAME, password, active=True) + self.stdout.write( + self.style.SUCCESS(f'Successfully created user: {READONLY_USERNAME}') + ) + else: + system_db.replace_user(READONLY_USERNAME, password, active=True) + self.stdout.write( + self.style.SUCCESS(f'Successfully replaced user: {READONLY_USERNAME}') + ) + + system_db.update_permission(username=READONLY_USERNAME, permission='ro', database='*') + self.stdout.write( + self.style.SUCCESS( + 'Successfully set universal read-only permission for user: ' + f'{READONLY_USERNAME}' + ) + ) + except AttributeError: + self.stderr.write( + self.style.ERROR( + 'Environment variable MULTINET_ARANGO_READONLY_PASSWORD is required.' + ) + ) + except UserCreateError: + self.stderr.write(self.style.ERROR(f'Failed to create user: {READONLY_USERNAME}')) + except PermissionUpdateError: + self.stderr.write( + self.style.ERROR(f'Failed to set permissions for user: {READONLY_USERNAME}') + ) + except (ArangoClientError, ArangoServerError) as error: + self.stderr.write(self.style.ERROR(str(error))) diff --git a/multinet/api/models/network.py b/multinet/api/models/network.py index 93d66ea..2e8dbd7 100644 --- a/multinet/api/models/network.py +++ b/multinet/api/models/network.py @@ -68,7 +68,7 @@ def create_with_edge_definition( ) -> Network: """Create a network with an edge definition, using the provided arguments.""" # Create graph in arango before creating the Network object here - workspace.get_arango_db().create_graph( + workspace.get_arango_db(readonly=False).create_graph( name, edge_definitions=[ { @@ -93,7 +93,7 @@ def __str__(self) -> str: def arango_graph_save(sender: Type[Network], instance: Network, **kwargs): workspace: Workspace = instance.workspace - db = workspace.get_arango_db() + db = workspace.get_arango_db(readonly=False) if not db.has_graph(instance.name): db.create_graph(instance.name) @@ -102,6 +102,6 @@ def arango_graph_save(sender: Type[Network], instance: Network, **kwargs): def arango_graph_delete(sender: Type[Network], instance: Network, **kwargs): workspace: Workspace = instance.workspace - db = workspace.get_arango_db() + db = workspace.get_arango_db(readonly=False) if db.has_graph(instance.name): db.delete_graph(instance.name) diff --git a/multinet/api/models/table.py b/multinet/api/models/table.py index 511ace4..32a0874 100644 --- a/multinet/api/models/table.py +++ b/multinet/api/models/table.py @@ -43,9 +43,9 @@ class Meta: def count(self) -> int: return self.get_arango_collection().count() - def get_arango_collection(self) -> StandardCollection: + def get_arango_collection(self, readonly=True) -> StandardCollection: workspace: Workspace = self.workspace - return workspace.get_arango_db().collection(self.name) + return workspace.get_arango_db(readonly=readonly).collection(self.name) def get_row(self, query: Optional[Dict] = None) -> Cursor: """Return a specific document.""" @@ -58,7 +58,7 @@ def get_rows(self, limit: Optional[int] = None, offset: Optional[int] = None) -> def put_rows(self, rows: List[Dict]) -> RowInsertionResponse: """Insert/update rows in the underlying arangodb collection.""" - res = self.get_arango_collection().insert_many(rows, overwrite=True) + res = self.get_arango_collection(readonly=False).insert_many(rows, overwrite=True) errors = [ RowModifyError(index=i, message=doc.error_message) for i, doc in enumerate(res) @@ -70,7 +70,7 @@ def put_rows(self, rows: List[Dict]) -> RowInsertionResponse: def delete_rows(self, rows: List[Dict]) -> RowDeletionResponse: """Delete rows in the underlying arangodb collection.""" - res = self.get_arango_collection().delete_many(rows) + res = self.get_arango_collection(readonly=False).delete_many(rows) errors = [ RowModifyError(index=i, message=doc.error_message) for i, doc in enumerate(res) @@ -127,7 +127,7 @@ def __str__(self) -> str: def arango_coll_save(sender: Type[Table], instance: Table, **kwargs): workspace: Workspace = instance.workspace - db = workspace.get_arango_db() + db = workspace.get_arango_db(readonly=False) if not db.has_collection(instance.name): db.create_collection(instance.name, edge=instance.edge) @@ -136,6 +136,6 @@ def arango_coll_save(sender: Type[Table], instance: Table, **kwargs): def arango_coll_delete(sender: Type[Table], instance: Table, **kwargs): workspace: Workspace = instance.workspace - db = workspace.get_arango_db() + db = workspace.get_arango_db(readonly=False) if db.has_collection(instance.name): db.delete_collection(instance.name) diff --git a/multinet/api/models/workspace.py b/multinet/api/models/workspace.py index b1cf279..6e5fa3a 100644 --- a/multinet/api/models/workspace.py +++ b/multinet/api/models/workspace.py @@ -10,7 +10,7 @@ from django.dispatch import receiver from django_extensions.db.models import TimeStampedModel -from multinet.api.utils.arango import ensure_db_created, ensure_db_deleted, get_or_create_db +from multinet.api.utils.arango import db, ensure_db_created, ensure_db_deleted def create_default_arango_db_name(): @@ -165,8 +165,14 @@ def set_user_permissions_bulk( [*new_reader_roles, *new_writer_roles, *new_maintainer_roles] ) - def get_arango_db(self) -> StandardDatabase: - return get_or_create_db(self.arango_db_name) + def get_arango_db(self, readonly=True) -> StandardDatabase: + """ + Return the arango database associated with this workspace. + + The workspace must be saved before accessing this function. Otherwise, this will result in + errors, as the arango database is created on model save. + """ + return db(self.arango_db_name, readonly) def __str__(self) -> str: return self.name diff --git a/multinet/api/tests/conftest.py b/multinet/api/tests/conftest.py index b4f05ad..c11a279 100644 --- a/multinet/api/tests/conftest.py +++ b/multinet/api/tests/conftest.py @@ -88,9 +88,10 @@ def pytest_sessionfinish(session, exitstatus): # `pytest.mark.django_db` decorator doesn't run the model save/delete methods, meaning the sync # between arangodb and django doesn't happen. - for db in arango_system_db().databases(): + system_db = arango_system_db(readonly=False) + for db in system_db.databases(): if db not in pytest.before_session_arango_databases: - arango_system_db().delete_database(db, ignore_missing=True) + system_db.delete_database(db, ignore_missing=True) register(UserFactory) diff --git a/multinet/api/tests/test_command.py b/multinet/api/tests/test_command.py new file mode 100644 index 0000000..51d8795 --- /dev/null +++ b/multinet/api/tests/test_command.py @@ -0,0 +1,18 @@ +from django.core.management import call_command + +from multinet.api.management.commands.createarangoreadonlyuser import READONLY_USERNAME +from multinet.api.utils.arango import arango_system_db + + +def test_createarangoreadonlyuser(): + system_db = arango_system_db(readonly=False) + + if system_db.has_user(READONLY_USERNAME): + system_db.delete_user(READONLY_USERNAME) + assert not system_db.has_user(READONLY_USERNAME) + + call_command('createarangoreadonlyuser') + + assert system_db.has_user(READONLY_USERNAME) + readonly_permissions = system_db.permission(READONLY_USERNAME, '*') + assert readonly_permissions == 'ro' diff --git a/multinet/api/tests/test_workspace.py b/multinet/api/tests/test_workspace.py index 910a535..a666de7 100644 --- a/multinet/api/tests/test_workspace.py +++ b/multinet/api/tests/test_workspace.py @@ -1,5 +1,6 @@ from typing import Dict, List +from arango.cursor import Cursor from django.contrib.auth.models import User from faker import Faker import pytest @@ -14,6 +15,7 @@ from multinet.api.tests.utils import create_users_with_permissions from multinet.api.utils.arango import arango_system_db +from .conftest import populated_table from .fuzzy import TIMESTAMP_RE, workspace_re @@ -415,3 +417,60 @@ def test_workspace_rest_get_user_permission_public( 'permission': WorkspaceRoleChoice.READER.value, 'permission_label': WorkspaceRoleChoice.READER.label, } + + +@pytest.mark.django_db +@pytest.mark.parametrize( + 'permission,is_owner,status_code,success', + [ + (None, False, 404, False), + (WorkspaceRoleChoice.READER, False, 200, True), + (WorkspaceRoleChoice.WRITER, False, 200, True), + (WorkspaceRoleChoice.MAINTAINER, False, 200, True), + (None, True, 200, True), + ], +) +def test_workspace_rest_aql( + workspace: Workspace, + user: User, + authenticated_api_client: APIClient, + permission: WorkspaceRoleChoice, + is_owner: bool, + status_code: int, + success: bool, +): + if permission is not None: + workspace.set_user_permission(user, WorkspaceRoleChoice.READER) + elif is_owner: + workspace.set_owner(user) + + node_table = populated_table(workspace, False) + nodes: Cursor = node_table.get_rows() + nodes_list = list(nodes) + # try and execute a valid non-mutating query on the data + query = f'FOR document IN {node_table.name} RETURN document' + r = authenticated_api_client.get( + f'/api/workspaces/{workspace.name}/aql/', data={'query': query} + ) + assert r.status_code == status_code + + if success: + results = r.json() + for node in nodes_list: + assert node in results + + +@pytest.mark.django_db +def test_workspace_rest_aql_mutating_query( + workspace: Workspace, user: User, authenticated_api_client: APIClient +): + workspace.set_user_permission(user, WorkspaceRoleChoice.READER) + fake = Faker() + + node_table = populated_table(workspace, False) + # Mutating query + query = f"INSERT {{ 'name': {fake.pystr()} }} INTO {node_table.name}" + r = authenticated_api_client.get( + f'/api/workspaces/{workspace.name}/aql/', data={'query': query} + ) + assert r.status_code == 400 diff --git a/multinet/api/utils/arango.py b/multinet/api/utils/arango.py index 7e0c13b..c0a1729 100644 --- a/multinet/api/utils/arango.py +++ b/multinet/api/utils/arango.py @@ -23,32 +23,33 @@ def arango_client(): return ArangoClient(hosts=settings.MULTINET_ARANGO_URL, http_client=NoTimeoutHttpClient()) -def db(name: str): - return arango_client().db(name, username='root', password=settings.MULTINET_ARANGO_PASSWORD) +def db(name: str, readonly): + username = 'readonly' if readonly else 'root' + password = ( + settings.MULTINET_ARANGO_READONLY_PASSWORD + if readonly + else settings.MULTINET_ARANGO_PASSWORD + ) + return arango_client().db(name, username=username, password=password) @lru_cache() -def arango_system_db(): - return db('_system') +def arango_system_db(readonly=True): + return db('_system', readonly) def ensure_db_created(name: str) -> None: - sysdb = arango_system_db() + sysdb = arango_system_db(readonly=False) if not sysdb.has_database(name): sysdb.create_database(name) def ensure_db_deleted(name: str) -> None: - sysdb = arango_system_db() + sysdb = arango_system_db(readonly=False) if sysdb.has_database(name): sysdb.delete_database(name) -def get_or_create_db(name: str) -> StandardDatabase: - ensure_db_created(name) - return db(name) - - class ArangoQuery: """A class to represent an AQL query.""" @@ -57,10 +58,14 @@ def __init__( db: StandardDatabase, query_str: Optional[str] = None, bind_vars: Optional[Dict[str, str]] = None, + time_limit_secs: int = 30, + memory_limit_bytes: int = 20000000, # 20MB ) -> None: self.db = db self.query_str = query_str self.bind_vars = bind_vars + self.time_limit_secs = time_limit_secs + self.memory_limit_bytes = memory_limit_bytes @staticmethod def from_collections(db: StandardDatabase, collections: List[str]) -> ArangoQuery: @@ -132,4 +137,11 @@ def execute(self, **kwargs) -> Cursor: Accepts the same keyword arguments as `arango.database.StandardDatabase.aql.execute`. """ + # Use time and memory limit of the query object unless different values + # are explicitly passed. + if 'max_runtime' not in kwargs: + kwargs['max_runtime'] = self.time_limit_secs + if 'memory_limit' not in kwargs: + kwargs['memory_limit'] = self.memory_limit_bytes + return self.db.aql.execute(query=self.query_str, bind_vars=self.bind_vars, **kwargs) diff --git a/multinet/api/views/serializers.py b/multinet/api/views/serializers.py index 329440d..0010843 100644 --- a/multinet/api/views/serializers.py +++ b/multinet/api/views/serializers.py @@ -88,6 +88,10 @@ class SingleUserWorkspacePermissionSerializer(serializers.Serializer): permission_label = serializers.CharField(allow_null=True) +class AqlQuerySerializer(serializers.Serializer): + query = serializers.CharField() + + class LimitOffsetSerializer(serializers.Serializer): limit = serializers.IntegerField(required=False) offset = serializers.IntegerField(required=False) diff --git a/multinet/api/views/workspace.py b/multinet/api/views/workspace.py index 96a674b..db86202 100644 --- a/multinet/api/views/workspace.py +++ b/multinet/api/views/workspace.py @@ -1,5 +1,7 @@ from typing import OrderedDict +from arango.cursor import Cursor +from arango.exceptions import AQLQueryExecuteError, ArangoServerError from django.contrib.auth.models import User from django.db.models import Q from django.shortcuts import get_object_or_404 @@ -13,7 +15,9 @@ from multinet.api.auth.decorators import require_workspace_ownership, require_workspace_permission from multinet.api.models import Workspace, WorkspaceRole, WorkspaceRoleChoice +from multinet.api.utils.arango import ArangoQuery from multinet.api.views.serializers import ( + AqlQuerySerializer, PermissionsCreateSerializer, PermissionsReturnSerializer, SingleUserWorkspacePermissionSerializer, @@ -173,3 +177,32 @@ def put_workspace_permissions(self, request, name: str): workspace.set_owner(new_owner) return Response(PermissionsReturnSerializer(workspace).data, status=status.HTTP_200_OK) + + @swagger_auto_schema(query_serializer=AqlQuerySerializer()) + @action(detail=True) + @require_workspace_permission(WorkspaceRoleChoice.READER) + def aql(self, request, name: str): + """Execute AQL in a workspace.""" + serializer = AqlQuerySerializer(data=request.query_params) + serializer.is_valid(raise_exception=True) + query_str = serializer.validated_data['query'] + workspace: Workspace = get_object_or_404(Workspace, name=name) + database = workspace.get_arango_db() + query = ArangoQuery(database, query_str) + + try: + cursor: Cursor = query.execute() + return Response( + cursor, + status=status.HTTP_200_OK, + ) + except AQLQueryExecuteError as err: + # Invalid query, time/memory limit reached, or + # attempt to run a mutating query as the readonly user + return Response( + err.error_message, + status=status.HTTP_400_BAD_REQUEST, + ) + except ArangoServerError as err: + # Arango server errors unrelated to the client's query + return Response(err.error_message, status=status.HTTP_500_INTERNAL_SERVER_ERROR) diff --git a/multinet/settings.py b/multinet/settings.py index 55a3ba1..9b7a644 100644 --- a/multinet/settings.py +++ b/multinet/settings.py @@ -38,6 +38,7 @@ def before_binding(configuration: ComposedConfiguration) -> None: MULTINET_ARANGO_URL = values.Value(environ_required=True) MULTINET_ARANGO_PASSWORD = values.Value(environ_required=True) + MULTINET_ARANGO_READONLY_PASSWORD = values.Value(environ_required=True) SWAGGER_SETTINGS = { 'DEFAULT_AUTO_SCHEMA_CLASS': 'multinet.api.utils.swagger.ImprovedAutoSchema' } diff --git a/tox.ini b/tox.ini index 000451f..49b275a 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,8 @@ commands = black {posargs:.} [testenv:test] +setenv = + DJANGO_CONFIGURATION = TestingConfiguration passenv = DJANGO_CELERY_BROKER_URL DJANGO_DATABASE_URL @@ -47,6 +49,7 @@ passenv = DJANGO_MINIO_STORAGE_SECRET_KEY DJANGO_MULTINET_ARANGO_URL DJANGO_MULTINET_ARANGO_PASSWORD + DJANGO_MULTINET_ARANGO_READONLY_PASSWORD extras = dev deps = @@ -56,6 +59,8 @@ deps = pytest-django pytest-factoryboy pytest-mock +commands_pre = + ./manage.py createarangoreadonlyuser commands = pytest {posargs} @@ -70,6 +75,7 @@ passenv = DJANGO_MINIO_STORAGE_SECRET_KEY DJANGO_MULTINET_ARANGO_URL DJANGO_MULTINET_ARANGO_PASSWORD + DJANGO_MULTINET_ARANGO_READONLY_PASSWORD commands = {envpython} ./manage.py makemigrations --check --dry-run @@ -88,7 +94,6 @@ ignore = [pytest] DJANGO_SETTINGS_MODULE = multinet.settings -DJANGO_CONFIGURATION = TestingConfiguration addopts = --strict-markers --showlocals --verbose filterwarnings = ignore::DeprecationWarning:minio