diff --git a/.github/workflows/codegen.yaml b/.github/workflows/codegen.yaml new file mode 100644 index 000000000..4b315c3f8 --- /dev/null +++ b/.github/workflows/codegen.yaml @@ -0,0 +1,76 @@ +name: Codegen + +on: + push: + branches: + - 'main' + - 'release/*' + tags: + - 'v*' + pull_request: + branches: + - 'main' + - 'release/*' + +jobs: + docker: + timeout-minutes: 10 + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + node: + - "14" + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Checkout + uses: actions/checkout@v3 + + - name: Build Backend no push + uses: docker/build-push-action@v3 + with: + context: ./backend + platforms: linux/amd64 + push: false + tags: github-action + + - name: Start containers + run: docker-compose -f "docker-compose.test.yml" up -d --build + + # setup node + - name: Install node + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node }} + + - name: Install dependencies + working-directory: frontend + run: npm install + + - name: Run Codegen + working-directory: frontend + run: npm run codegen:v2:test + + - name: Verify Changed files + uses: tj-actions/verify-changed-files@v14 + id: verify-changed-files + with: + files: frontend/src/openapi + + - name: Perform action when openapi file changes + if: contains(steps.verify-changed-files.outputs.changed_files, 'frontend/src/openapi') + run: | + echo "Codegen is out of sync" + echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}" + exit 1 + + - name: Stop containers + if: always() + run: docker-compose -f "docker-compose.test.yml" down diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml index 6784494f4..fa6789523 100644 --- a/.github/workflows/frontend.yaml +++ b/.github/workflows/frontend.yaml @@ -54,7 +54,7 @@ jobs: with: context: ./frontend target: clowder-build - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} @@ -64,7 +64,7 @@ jobs: with: context: ./frontend target: clowder-runtime - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file + labels: ${{ steps.meta.outputs.labels }} diff --git a/README.md b/README.md index 13b5aa4fd..ca40db5e2 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,9 @@ following steps. 3. Run for development: `npm run start:dev` 4. By default backend runs at `http://localhost:8000`. If running at different url/port, use: `CLOWDER_REMOTE_HOSTNAME=http://somewhere:9999 npm start` -5. After modifying backend API, update autogenerated client function calls (backend must be running): -`CLOWDER_REMOTE_HOSTNAME=http://localhost:8000 npm run codegen:v2` +5. After modifying backend API, update autogenerated client function calls: + - Backend must be running + - Run codege: `npm run codegen:v2:dev` ### Configuring Keycloak diff --git a/backend/app/routers/keycloak.py b/backend/app/routers/keycloak.py index 81ee6bc1d..a3bddbcdf 100644 --- a/backend/app/routers/keycloak.py +++ b/backend/app/routers/keycloak.py @@ -188,27 +188,28 @@ async def refresh_token( ) -@router.get("/broker/{identity_provider}/token") -def get_identity_provider_token( - identity_provider: str, access_token: str = Security(oauth2_scheme) -) -> Json: - """Get identity provider JWT token from keyclok. Keycloak must be configured to store external tokens.""" - if identity_provider in settings.keycloak_ipds: - idp_url = f"{settings.auth_base}/auth/realms/{settings.auth_realm}/broker/{identity_provider}/token" - idp_headers = { - "Content-Type": "application/x-www-form-urlencoded", - "Authorization": f"Bearer {access_token}", - } - idp_token = requests.request("GET", idp_url, headers=idp_headers) - # FIXME is there a better way to know if the token as expired and the above call did not go through? - idp_token.raise_for_status() - itp_token_body = json.loads(idp_token.content) - return itp_token_body - else: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail={ - "error_msg": f"Identy provider [{identity_provider}] not recognized." - }, - headers={"WWW-Authenticate": "Bearer"}, - ) +# FIXME: we need to parse and return a consistent response +# @router.get("/broker/{identity_provider}/token") +# def get_idenity_provider_token( +# identity_provider: str, access_token: str = Security(oauth2_scheme) +# ) -> Json: +# """Get identity provider JWT token from keyclok. Keycloak must be configured to store external tokens.""" +# if identity_provider in settings.keycloak_ipds: +# idp_url = f"{settings.auth_base}/auth/realms/{settings.auth_realm}/broker/{identity_provider}/token" +# idp_headers = { +# "Content-Type": "application/x-www-form-urlencoded", +# "Authorization": f"Bearer {access_token}", +# } +# idp_token = requests.request("GET", idp_url, headers=idp_headers) +# # FIXME is there a better way to know if the token as expired and the above call did not go through? +# idp_token.raise_for_status() +# itp_token_body = json.loads(idp_token.content) +# return itp_token_body +# else: +# raise HTTPException( +# status_code=status.HTTP_400_BAD_REQUEST, +# detail={ +# "error_msg": f"Identy provider [{identity_provider}] not recognized." +# }, +# headers={"WWW-Authenticate": "Bearer"}, +# ) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5d4f1a284..37ffa3581 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -167,7 +167,6 @@ services: - RABBITMQ_HOST=rabbitmq - RABBITMQ_USER=${RABBITMQ_USER:-guest} - RABBITMQ_PASS=${RABBITMQ_PASS:-guest} - restart: unless-stopped networks: - clowder2 restart: unless-stopped diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 000000000..830cd7a34 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,198 @@ +version: '3.7' + +# Settings and configurations that are common for all minio containers +x-minio-common: &minio-common + image: quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z + command: server --console-address ":9001" http://minio{1...4}/data + restart: unless-stopped + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + networks: + - clowder2 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + +services: + + reverse-proxy: + image: traefik:v2.5 + restart: unless-stopped + command: + - --api.insecure=true + - --providers.docker +# - --entryPoints.web.address=:80 + ports: + # The HTTP port + - "80:80" + # The Web UI (enabled by --api.insecure=true) + - "8080:8080" + networks: + - clowder2 + volumes: + # So that Traefik can listen to the Docker events + - /var/run/docker.sock:/var/run/docker.sock + + backend: + image: 'clowder/clowder2-backend:github-action' + restart: unless-stopped + build: + context: ./backend + networks: + - clowder2 + environment: + MONGODB_URL: mongodb://mongo:27017 + MINIO_SERVER_URL: minio-nginx:9000 + RABBITMQ_HOST: rabbitmq:15672 + elasticsearch_url: http://elasticsearch:9200 + auth_base: http://localhost + auth_url: http://localhost/keycloak/realms/clowder/protocol/openid-connect/auth?client_id=clowder2-backend&response_type=code + oauth2_scheme_auth_url: http://keycloak:8080/keycloak/realms/clowder/protocol/openid-connect/auth?client_id=clowder2-backend&response_type=code + auth_register_url: http://localhost/keycloak/realms/clowder/protocol/openid-connect/registrations?client_id=clowder2-backend&response_type=code + auth_token_url: http://keycloak:8080/keycloak/realms/clowder/protocol/openid-connect/token + auth_server_url: http://keycloak:8080/keycloak/ + keycloak_base: http://localhost/api + frontend_url: http://localhost + depends_on: + - mongo + - minio-nginx + - keycloak + - elasticsearch + labels: + - "traefik.enable=true" + - "traefik.http.routers.backend.rule=PathPrefix(`/api`)" + - "traefik.http.services.backend.loadbalancer.server.port=80" + - "traefik.http.routers.backend.priority=5" + + mongo: + image: mongo:5.0 + restart: unless-stopped + networks: + - clowder2 + volumes: + - mongo:/data/db + - ./scripts/mongoviews/mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js:ro + + minio1: + <<: *minio-common + hostname: minio1 + volumes: + - data1:/data + + minio2: + <<: *minio-common + hostname: minio2 + volumes: + - data2:/data + + minio3: + <<: *minio-common + hostname: minio3 + volumes: + - data3:/data + + minio4: + <<: *minio-common + hostname: minio4 + volumes: + - data4:/data + + minio-nginx: + image: nginx:1.19.2-alpine + restart: unless-stopped + hostname: nginx + networks: + - clowder2 + volumes: + - ./docker/minio-nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + - minio1 + - minio2 + - minio3 + - minio4 + + postgres: + image: postgres + restart: unless-stopped + networks: + - clowder2 + volumes: + - postgres_data:/var/lib/postgresql/data + environment: + POSTGRES_DB: keycloak_prod + POSTGRES_USER: keycloak + POSTGRES_PASSWORD: password + + keycloak: + image: quay.io/keycloak/keycloak:19.0 + restart: unless-stopped + networks: + - clowder2 + volumes: + - ./scripts/keycloak/clowder-realm-prod.json:/opt/keycloak/data/import/realm.json:ro + - ./scripts/keycloak/clowder-theme/:/opt/keycloak/themes/clowder-theme/:ro + command: + - start-dev + - --http-relative-path /keycloak + - --import-realm + environment: + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: admin + KC_DB: postgres + KC_DB_URL_HOST: postgres + KC_DB_URL_DATABASE: keycloak_prod + KC_DB_USERNAME: keycloak + KC_DB_PASSWORD: password + depends_on: + - postgres + labels: + - "traefik.enable=true" + - "traefik.http.routers.keycloak.rule=PathPrefix(`/keycloak`)" + - "traefik.http.services.keycloak.loadbalancer.server.port=8080" + - "traefik.http.routers.keycloak.priority=10" + + # message broker + rabbitmq: + image: rabbitmq:3-management-alpine + environment: + - RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=-rabbitmq_management path_prefix "/rabbitmq" + - RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER:-guest} + - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS:-guest} + ports: + - "5672:5672" + - "15672:15672" + networks: + - clowder2 + volumes: + - rabbitmq:/var/lib/rabbitmq + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.3.3 + restart: unless-stopped + networks: + - clowder2 + environment: + - "cluster.name=clowder2" + - "discovery.type=single-node" + - "xpack.security.enabled=false" + - "xpack.security.http.ssl.enabled=false" + volumes: + - elasticsearch:/usr/share/elasticsearch/data + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + mongo: + data1: + data2: + data3: + data4: + postgres_data: + elasticsearch: + rabbitmq: + +networks: + clowder2: diff --git a/frontend/package.json b/frontend/package.json index 0c3978951..e344d77cf 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -27,6 +27,7 @@ "codegen:v1": "./node_modules/.bin/openapi -i https://clowder.ncsa.illinois.edu/clowder/swagger -o src/openapi/v1", "codegen:v2": "./node_modules/.bin/openapi -i ${CLOWDER_REMOTE_HOSTNAME}/api/v2/openapi.json -o src/openapi/v2", "codegen:v2:dev": "export CLOWDER_REMOTE_HOSTNAME=http://localhost:8000 && ./node_modules/.bin/openapi -i ${CLOWDER_REMOTE_HOSTNAME}/api/v2/openapi.json -o src/openapi/v2", + "codegen:v2:test": "export CLOWDER_REMOTE_HOSTNAME=http://localhost && ./node_modules/.bin/openapi -i ${CLOWDER_REMOTE_HOSTNAME}/api/v2/openapi.json -o src/openapi/v2", "codegen": "npm run codegen:v1 && npm run codegen:v2", "docs": "typedoc" }, diff --git a/frontend/src/openapi/v2/services/AuthService.ts b/frontend/src/openapi/v2/services/AuthService.ts index 916350773..f6533346f 100644 --- a/frontend/src/openapi/v2/services/AuthService.ts +++ b/frontend/src/openapi/v2/services/AuthService.ts @@ -100,23 +100,4 @@ export class AuthService { }); } - /** - * Get Idenity Provider Token - * Get identity provider JWT token from keyclok. Keycloak must be configured to store external tokens. - * @param identityProvider - * @returns string Successful Response - * @throws ApiError - */ - public static getIdenityProviderTokenApiV2AuthBrokerIdentityProviderTokenGet( - identityProvider: string, - ): CancelablePromise { - return __request({ - method: 'GET', - path: `/api/v2/auth/broker/${identityProvider}/token`, - errors: { - 422: `Validation Error`, - }, - }); - } - } \ No newline at end of file diff --git a/frontend/src/openapi/v2/services/AuthorizationService.ts b/frontend/src/openapi/v2/services/AuthorizationService.ts index 398e0d239..159c20de4 100644 --- a/frontend/src/openapi/v2/services/AuthorizationService.ts +++ b/frontend/src/openapi/v2/services/AuthorizationService.ts @@ -241,6 +241,7 @@ export class AuthorizationService { /** * Get Dataset Users And Roles + * Returns a list of UserAndRole objects. These show what users have what permission on a dataset * @param datasetId * @returns UserAndRole Successful Response * @throws ApiError @@ -259,6 +260,8 @@ export class AuthorizationService { /** * Get Dataset Groups And Roles + * Returns a list of Group objects. These show what groups have what permission on a dataset Group and + * role has the id, name, and roleType * @param datasetId * @returns GroupAndRole Successful Response * @throws ApiError diff --git a/frontend/src/openapi/v2/services/UsersService.ts b/frontend/src/openapi/v2/services/UsersService.ts index 699a16eb1..78ffdeb30 100644 --- a/frontend/src/openapi/v2/services/UsersService.ts +++ b/frontend/src/openapi/v2/services/UsersService.ts @@ -67,4 +67,16 @@ export class UsersService { }); } + /** + * Generate User Api Key + * @returns string Successful Response + * @throws ApiError + */ + public static generateUserApiKeyApiV2UsersKeysPost(): CancelablePromise { + return __request({ + method: 'POST', + path: `/api/v2/users/keys`, + }); + } + } \ No newline at end of file