379 GitHub action to run codegen automatically when there is backend change

.github/workflows/codegen.yaml

@@ -0,0 +1,76 @@
+name: Codegen
+
+on:
+  push:
+    branches:
+      - 'main'
+      - 'release/*'
+    tags:
+      - 'v*'
+  pull_request:
+    branches:
+      - 'main'
+      - 'release/*'
+
+jobs:
+  docker:
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        node:
+          - "14"
+
+    steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Build Backend no push
+        uses: docker/build-push-action@v3
+        with:
+          context: ./backend
+          platforms: linux/amd64
+          push: false
+          tags: github-action
+
+      - name: Start containers
+        run: docker-compose -f "docker-compose.test.yml" up -d --build
+
+      # setup node
+      - name: Install node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Install dependencies
+        working-directory: frontend
+        run: npm install
+
+      - name: Run Codegen
+        working-directory: frontend
+        run: npm run codegen:v2:test
+
+      - name: Verify Changed files
+        uses: tj-actions/verify-changed-files@v14
+        id: verify-changed-files
+        with:
+          files: frontend/src/openapi
+
+      - name: Perform action when openapi file changes
+        if: contains(steps.verify-changed-files.outputs.changed_files, 'frontend/src/openapi')
+        run: |
+          echo "Codegen is out of sync"
+          echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}"
+          exit 1
+
+      - name: Stop containers
+        if: always()
+        run: docker-compose -f "docker-compose.test.yml" down

.github/workflows/frontend.yaml

@@ -54,7 +54,7 @@ jobs:
         with:
           context: ./frontend
           target: clowder-build
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
@@ -64,7 +64,7 @@ jobs:
         with:
           context: ./frontend
           target: clowder-runtime
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          labels: ${{ steps.meta.outputs.labels }}

README.md

@@ -82,8 +82,9 @@ following steps.
 3. Run for development: `npm run start:dev`
 4. By default backend runs at `http://localhost:8000`. If running at different url/port, use:
 `CLOWDER_REMOTE_HOSTNAME=http://somewhere:9999 npm start`
-5. After modifying backend API, update autogenerated client function calls (backend must be running):
-`CLOWDER_REMOTE_HOSTNAME=http://localhost:8000 npm run codegen:v2`
+5. After modifying backend API, update autogenerated client function calls:
+   - Backend must be running
+   - Run codege: `npm run codegen:v2:dev`
 
 
 ### Configuring Keycloak

backend/app/routers/keycloak.py

@@ -188,27 +188,28 @@ async def refresh_token(
         )
 
 
-@router.get("/broker/{identity_provider}/token")
-def get_identity_provider_token(
-    identity_provider: str, access_token: str = Security(oauth2_scheme)
-) -> Json:
-    """Get identity provider JWT token from keyclok. Keycloak must be configured to store external tokens."""
-    if identity_provider in settings.keycloak_ipds:
-        idp_url = f"{settings.auth_base}/auth/realms/{settings.auth_realm}/broker/{identity_provider}/token"
-        idp_headers = {
-            "Content-Type": "application/x-www-form-urlencoded",
-            "Authorization": f"Bearer {access_token}",
-        }
-        idp_token = requests.request("GET", idp_url, headers=idp_headers)
-        # FIXME is there a better way to know if the token as expired and the above call did not go through?
-        idp_token.raise_for_status()
-        itp_token_body = json.loads(idp_token.content)
-        return itp_token_body
-    else:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail={
-                "error_msg": f"Identy provider [{identity_provider}] not recognized."
-            },
-            headers={"WWW-Authenticate": "Bearer"},
-        )
+# FIXME: we need to parse and return a consistent response
+# @router.get("/broker/{identity_provider}/token")
+# def get_idenity_provider_token(
+#     identity_provider: str, access_token: str = Security(oauth2_scheme)
+# ) -> Json:
+#     """Get identity provider JWT token from keyclok. Keycloak must be configured to store external tokens."""
+#     if identity_provider in settings.keycloak_ipds:
+#         idp_url = f"{settings.auth_base}/auth/realms/{settings.auth_realm}/broker/{identity_provider}/token"
+#         idp_headers = {
+#             "Content-Type": "application/x-www-form-urlencoded",
+#             "Authorization": f"Bearer {access_token}",
+#         }
+#         idp_token = requests.request("GET", idp_url, headers=idp_headers)
+#         # FIXME is there a better way to know if the token as expired and the above call did not go through?
+#         idp_token.raise_for_status()
+#         itp_token_body = json.loads(idp_token.content)
+#         return itp_token_body
+#     else:
+#         raise HTTPException(
+#             status_code=status.HTTP_400_BAD_REQUEST,
+#             detail={
+#                 "error_msg": f"Identy provider [{identity_provider}] not recognized."
+#             },
+#             headers={"WWW-Authenticate": "Bearer"},
+#         )

docker-compose.dev.yml

@@ -167,7 +167,6 @@ services:
       - RABBITMQ_HOST=rabbitmq
       - RABBITMQ_USER=${RABBITMQ_USER:-guest}
       - RABBITMQ_PASS=${RABBITMQ_PASS:-guest}
-    restart: unless-stopped
     networks:
       - clowder2
     restart: unless-stopped

docker-compose.test.yml

@@ -0,0 +1,198 @@
+version: '3.7'
+
+# Settings and configurations that are common for all minio containers
+x-minio-common: &minio-common
+  image: quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z
+  command: server --console-address ":9001" http://minio{1...4}/data
+  restart: unless-stopped
+  environment:
+    MINIO_ROOT_USER: minioadmin
+    MINIO_ROOT_PASSWORD: minioadmin
+  networks:
+    - clowder2
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    interval: 30s
+    timeout: 20s
+    retries: 3
+
+services:
+
+  reverse-proxy:
+    image: traefik:v2.5
+    restart: unless-stopped
+    command:
+      - --api.insecure=true
+      - --providers.docker
+#      - --entryPoints.web.address=:80
+    ports:
+      # The HTTP port
+      - "80:80"
+      # The Web UI (enabled by --api.insecure=true)
+      - "8080:8080"
+    networks:
+      - clowder2
+    volumes:
+      # So that Traefik can listen to the Docker events
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  backend:
+    image: 'clowder/clowder2-backend:github-action'
+    restart: unless-stopped
+    build:
+      context: ./backend
+    networks:
+      - clowder2
+    environment:
+      MONGODB_URL: mongodb://mongo:27017
+      MINIO_SERVER_URL: minio-nginx:9000
+      RABBITMQ_HOST: rabbitmq:15672
+      elasticsearch_url: http://elasticsearch:9200
+      auth_base: http://localhost
+      auth_url: http://localhost/keycloak/realms/clowder/protocol/openid-connect/auth?client_id=clowder2-backend&response_type=code
+      oauth2_scheme_auth_url: http://keycloak:8080/keycloak/realms/clowder/protocol/openid-connect/auth?client_id=clowder2-backend&response_type=code
+      auth_register_url: http://localhost/keycloak/realms/clowder/protocol/openid-connect/registrations?client_id=clowder2-backend&response_type=code
+      auth_token_url: http://keycloak:8080/keycloak/realms/clowder/protocol/openid-connect/token
+      auth_server_url: http://keycloak:8080/keycloak/
+      keycloak_base: http://localhost/api
+      frontend_url: http://localhost
+    depends_on:
+      - mongo
+      - minio-nginx
+      - keycloak
+      - elasticsearch
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.backend.rule=PathPrefix(`/api`)"
+      - "traefik.http.services.backend.loadbalancer.server.port=80"
+      - "traefik.http.routers.backend.priority=5"
+
+  mongo:
+    image: mongo:5.0
+    restart: unless-stopped
+    networks:
+      - clowder2
+    volumes:
+      - mongo:/data/db
+      - ./scripts/mongoviews/mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js:ro
+
+  minio1:
+    <<: *minio-common
+    hostname: minio1
+    volumes:
+      - data1:/data
+
+  minio2:
+    <<: *minio-common
+    hostname: minio2
+    volumes:
+      - data2:/data
+
+  minio3:
+    <<: *minio-common
+    hostname: minio3
+    volumes:
+      - data3:/data
+
+  minio4:
+    <<: *minio-common
+    hostname: minio4
+    volumes:
+      - data4:/data
+
+  minio-nginx:
+    image: nginx:1.19.2-alpine
+    restart: unless-stopped
+    hostname: nginx
+    networks:
+      - clowder2
+    volumes:
+      - ./docker/minio-nginx.conf:/etc/nginx/nginx.conf:ro
+    depends_on:
+      - minio1
+      - minio2
+      - minio3
+      - minio4
+
+  postgres:
+      image: postgres
+      restart: unless-stopped
+      networks:
+        - clowder2
+      volumes:
+        - postgres_data:/var/lib/postgresql/data
+      environment:
+        POSTGRES_DB: keycloak_prod
+        POSTGRES_USER: keycloak
+        POSTGRES_PASSWORD: password
+
+  keycloak:
+      image: quay.io/keycloak/keycloak:19.0
+      restart: unless-stopped
+      networks:
+        - clowder2
+      volumes:
+        - ./scripts/keycloak/clowder-realm-prod.json:/opt/keycloak/data/import/realm.json:ro
+        - ./scripts/keycloak/clowder-theme/:/opt/keycloak/themes/clowder-theme/:ro
+      command:
+        - start-dev
+        - --http-relative-path /keycloak
+        - --import-realm
+      environment:
+        KEYCLOAK_ADMIN: admin
+        KEYCLOAK_ADMIN_PASSWORD: admin
+        KC_DB: postgres
+        KC_DB_URL_HOST: postgres
+        KC_DB_URL_DATABASE: keycloak_prod
+        KC_DB_USERNAME: keycloak
+        KC_DB_PASSWORD: password
+      depends_on:
+        - postgres
+      labels:
+        - "traefik.enable=true"
+        - "traefik.http.routers.keycloak.rule=PathPrefix(`/keycloak`)"
+        - "traefik.http.services.keycloak.loadbalancer.server.port=8080"
+        - "traefik.http.routers.keycloak.priority=10"
+
+  # message broker
+  rabbitmq:
+    image: rabbitmq:3-management-alpine
+    environment:
+      - RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=-rabbitmq_management path_prefix "/rabbitmq"
+      - RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER:-guest}
+      - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS:-guest}
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+    networks:
+      - clowder2
+    volumes:
+      - rabbitmq:/var/lib/rabbitmq
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.3.3
+    restart: unless-stopped
+    networks:
+      - clowder2
+    environment:
+      - "cluster.name=clowder2"
+      - "discovery.type=single-node"
+      - "xpack.security.enabled=false"
+      - "xpack.security.http.ssl.enabled=false"
+    volumes:
+      - elasticsearch:/usr/share/elasticsearch/data
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  mongo:
+  data1:
+  data2:
+  data3:
+  data4:
+  postgres_data:
+  elasticsearch:
+  rabbitmq:
+
+networks:
+  clowder2:

frontend/package.json

@@ -27,6 +27,7 @@
 		"codegen:v1": "./node_modules/.bin/openapi -i https://clowder.ncsa.illinois.edu/clowder/swagger -o src/openapi/v1",
 		"codegen:v2": "./node_modules/.bin/openapi -i ${CLOWDER_REMOTE_HOSTNAME}/api/v2/openapi.json -o src/openapi/v2",
 		"codegen:v2:dev": "export CLOWDER_REMOTE_HOSTNAME=http://localhost:8000 && ./node_modules/.bin/openapi -i ${CLOWDER_REMOTE_HOSTNAME}/api/v2/openapi.json -o src/openapi/v2",
+		"codegen:v2:test": "export CLOWDER_REMOTE_HOSTNAME=http://localhost && ./node_modules/.bin/openapi -i ${CLOWDER_REMOTE_HOSTNAME}/api/v2/openapi.json -o src/openapi/v2",
 		"codegen": "npm run codegen:v1 && npm run codegen:v2",
 		"docs": "typedoc"
 	},