From ceba17af7e1b875d9026f1cc8a9fde751a981425 Mon Sep 17 00:00:00 2001 From: Elias Segundo Antonio Date: Wed, 28 Sep 2022 12:26:52 -0500 Subject: [PATCH 01/16] Feat: Adding composite and base job (#215) --- .../actions/common-rc-validation/action.yaml | 14 ++ .github/workflows/run_rc_validation.yml | 130 ++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 .github/actions/common-rc-validation/action.yaml create mode 100644 .github/workflows/run_rc_validation.yml diff --git a/.github/actions/common-rc-validation/action.yaml b/.github/actions/common-rc-validation/action.yaml new file mode 100644 index 000000000000..b1976b687ecd --- /dev/null +++ b/.github/actions/common-rc-validation/action.yaml @@ -0,0 +1,14 @@ +name: "Setup RC Variables" +description: Setup Common RC Variables and dependecies +runs: + using: composite + steps: + - id: set_common_vars + shell: bash + run: | + echo "RC_TAG=v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}" >> $GITHUB_ENV + echo "RELEASE_BRANCH=releasev${{github.event.inputs.RELEASE_VER}}" >> $GITHUB_ENV + echo "WORKING_BRANCH=v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}_validations" >> $GITHUB_ENV + echo "GIT_REPO_URL=https://github.com/elink21/beam.git" >> $GITHUB_ENV + echo "PYTHON_RC_DOWNLOAD_URL=https://dist.apache.org/repos/dist/dev/beam" >> $GITHUB_ENV + echo "RELEASE_VER=${{github.event.inputs.RELEASE_VER}}" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml new file mode 100644 index 000000000000..11fabaebda37 --- /dev/null +++ b/.github/workflows/run_rc_validation.yml @@ -0,0 +1,130 @@ +name: Run RC Validation +on: + workflow_dispatch: + inputs: + RELEASE_VER: + description: Beam version of current release + required: true + default: 2.42.0 # remove default before merging + USER_GCS_BUCKET: + description: Bucket to upload results + required: false + default: gs://rc-validation-migration-tests + RC_NUM: + description: RC number + required: true + default: "1" +env: # This should be a set of github secrets + USER_GCP_PROJECT: apache-beam-testing + USER_GCP_REGION: us-central1 + USER_GCP_ZONE: us-central1-a + REPO_URL: https://repository.apache.org/content/repositories/orgapachebeam-1284 +jobs: + python_cross_validation: + runs-on: [self-hosted,ubuntu-20.04] + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Common tasks + uses: ./.github/actions/common-rc-validation + - name: Verify ENV values + run: | + echo "" + echo "====================Checking Environment & Variables=================" + echo "" + echo "running validations on release ${{github.event.inputs.RELEASE_VER}} RC${{github.event.inputs.RC_NUM}}." + echo "repo URL for this RC: $GIT_REPO_URL" + - name: Install Kubectl + uses: azure/setup-kubectl@v3 + + - name: Setup Java JDK + uses: actions/setup-java@v3.5.1 + with: + distribution: 'temurin' + java-version: 11 + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + cache: 'pip' # caching pip dependencies + - name: Downloading Python Staging RC + run: | + echo "" + echo "====================Starting Python Cross-language Validations===============" + + + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then + { echo "Fail to download Python Staging RC files." ;exit 1; } + fi + + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + + `which pip` install --upgrade pip + `which pip` install --upgrade setuptools + + KAFKA_CLUSTER_NAME=xlang-kafka-cluster-$RANDOM + - name: Installing gcloud-auth-plugin + run: sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin + - name: Setting Kafka Cluster Name # TODO: change to random again once the tests are finished + run: | + echo "KAFKA_CLUSTER_NAME=xlang-kafka-cluster-12817">> $GITHUB_ENV +# - name: Creating Kafka Cluster +# run: | +# gcloud container clusters create --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --no-enable-ip-alias $KAFKA_CLUSTER_NAME +# kubectl apply -R -f .test-infra/kubernetes/kafka-cluster + - name: Getting cluster credentials + run: gcloud container clusters get-credentials ${KAFKA_CLUSTER_NAME} --region ${USER_GCP_REGION} --project ${USER_GCP_PROJECT} + - name: Setting Python Environment + run: | + echo "--------------------------Installing Python SDK-------------------------------" + pip install --upgrade pip setuptools wheel + pip install apache-beam-${RELEASE_VER}.zip[gcp] + - name: Waiting for Kafka cluster to be ready + run: kubectl wait --for=condition=Ready pod/kafka-0 --timeout=900s + - name: Start xlang Kafka Taxi with Dataflow Runner + run: | + echo "BOOTSTRAP_SERVERS=$(kubectl get svc outside-0 -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):32400" >> $GITHUB_ENV + echo "KAFKA_TAXI_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV + echo "KAFKA_EXPANSION_SERVICE_JAR=${REPO_URL}/org/apache/beam/beam-sdks-java-io-expansion-service/${RELEASE_VER}/beam-sdks-java-io-expansion-service-${RELEASE_VER}.jar" >> $GITHUB_ENV + - name: Checking variables + run: | + echo $BOOTSTRAP_SERVERS + echo $KAFKA_TAXI_DF_DATASET + echo $KAFKA_EXPANSION_SERVICE_JAR + - name: Creating BigQuery Dataset + run: bq mk --project_id=${USER_GCP_PROJECT} ${KAFKA_TAXI_DF_DATASET} + - name: Running Xlang Kafka Taxi with Dataflow Runner + run: | + echo '*****************************************************'; + echo '* Running Python XLang Kafka Taxi with DataflowRunner'; + echo '*****************************************************'; + python -m apache_beam.examples.kafkataxi.kafka_taxi \ + --project=${USER_GCP_PROJECT} \ + --region=${USER_GCP_REGION} \ + --topic beam-runnerv2 \ + --bootstrap_servers ${BOOTSTRAP_SERVERS} \ + --bq_dataset ${KAFKA_TAXI_DF_DATASET} \ + --runner DataflowRunner \ + --num_workers 5 \ + --temp_location=${USER_GCS_BUCKET}/temp/ \ + --with_metadata \ + --beam_services="{\"sdks:java:io:expansion-service:shadowJar\": \"${KAFKA_EXPANSION_SERVICE_JAR}\"}" \ + --sdk_location apache-beam-${RELEASE_VER}.zip; \ + - name: Sleep Warning + run: | + echo "***************************************************************" + echo "* Please wait for at least 20 mins to let Dataflow job be launched and results get populated." + echo "* Sleeping for 20 mins" + sleep 20m + - name: Remove BigQuery Dataset + run: | + bq rm -f ${KAFKA_TAXI_DF_DATASET} + #- name: Delete Kafka Cluster + # run: gcloud container clusters delete --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --async -q $KAFKA_CLUSTER_NAME + # #From this point, steps 7 to last can be implemented as independent jobs + # The ones that uses python_versions_to_validate should use matrix instead \ No newline at end of file From d6106854242c08068f1765dccc357fa42b6761eb Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Wed, 28 Sep 2022 15:27:20 -0500 Subject: [PATCH 02/16] Added python_release_candidate job. --- .github/workflows/run_rc_validation.yml | 238 +++++++++++++----------- 1 file changed, 132 insertions(+), 106 deletions(-) diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index 11fabaebda37..5a3ac80966dd 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -20,111 +20,137 @@ env: # This should be a set of github secrets USER_GCP_ZONE: us-central1-a REPO_URL: https://repository.apache.org/content/repositories/orgapachebeam-1284 jobs: - python_cross_validation: - runs-on: [self-hosted,ubuntu-20.04] + python_release_candidate: + #runs-on: [self-hosted, ubuntu-20.04] + runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Common tasks - uses: ./.github/actions/common-rc-validation - - name: Verify ENV values - run: | - echo "" - echo "====================Checking Environment & Variables=================" - echo "" - echo "running validations on release ${{github.event.inputs.RELEASE_VER}} RC${{github.event.inputs.RC_NUM}}." - echo "repo URL for this RC: $GIT_REPO_URL" - - name: Install Kubectl - uses: azure/setup-kubectl@v3 - - - name: Setup Java JDK - uses: actions/setup-java@v3.5.1 - with: - distribution: 'temurin' - java-version: 11 - - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - cache: 'pip' # caching pip dependencies - - name: Downloading Python Staging RC - run: | - echo "" - echo "====================Starting Python Cross-language Validations===============" - - - echo "---------------------Downloading Python Staging RC----------------------------" - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 - if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then - { echo "Fail to download Python Staging RC files." ;exit 1; } - fi - - echo "--------------------------Verifying Hashes------------------------------------" - sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 - - `which pip` install --upgrade pip - `which pip` install --upgrade setuptools - - KAFKA_CLUSTER_NAME=xlang-kafka-cluster-$RANDOM - - name: Installing gcloud-auth-plugin - run: sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin - - name: Setting Kafka Cluster Name # TODO: change to random again once the tests are finished - run: | - echo "KAFKA_CLUSTER_NAME=xlang-kafka-cluster-12817">> $GITHUB_ENV -# - name: Creating Kafka Cluster + - name: Setup Common + uses: ./.github/actions/common-rc-validation + - name: Check out code + uses: actions/checkout@v3 + with: + ref: ${{ env.RC_TAG }} + - name: Set git config + run: | + git config user.name $GITHUB_ACTOR + git config user.email actions@"$RUNNER_NAME".local + - name: Create Pull Request + run: | + git checkout -b ${{env.WORKING_BRANCH}} --quiet + touch empty_file.txt + git add empty_file.txt + git commit -m "Add empty file in order to create PR" --quiet + git push origin ${{env.WORKING_BRANCH}} --quiet + GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t"[DO NOT MERGE] Run Python RC Validation Tests" +#GITHUB_PR_URL=$(gh pr create -B ${RELEASE_BRANCH} -H ${RC_TAG} -t"[DO NOT MERGE] Run Python RC Validation Tests" + - name: Comment on PR to Trigger Python ReleaseCandidate Test + run: | + gh pr comment "$GITHUB_PR_URL" --body "Run Python ReleaseCandidate" +# python_cross_validation: +# runs-on: [self-hosted, ubuntu-20.04] +# steps: +# - name: Checkout code +# uses: actions/checkout@v3 +# - name: Common tasks +# uses: ./.github/actions/common-rc-validation +# - name: Verify ENV values # run: | -# gcloud container clusters create --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --no-enable-ip-alias $KAFKA_CLUSTER_NAME -# kubectl apply -R -f .test-infra/kubernetes/kafka-cluster - - name: Getting cluster credentials - run: gcloud container clusters get-credentials ${KAFKA_CLUSTER_NAME} --region ${USER_GCP_REGION} --project ${USER_GCP_PROJECT} - - name: Setting Python Environment - run: | - echo "--------------------------Installing Python SDK-------------------------------" - pip install --upgrade pip setuptools wheel - pip install apache-beam-${RELEASE_VER}.zip[gcp] - - name: Waiting for Kafka cluster to be ready - run: kubectl wait --for=condition=Ready pod/kafka-0 --timeout=900s - - name: Start xlang Kafka Taxi with Dataflow Runner - run: | - echo "BOOTSTRAP_SERVERS=$(kubectl get svc outside-0 -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):32400" >> $GITHUB_ENV - echo "KAFKA_TAXI_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV - echo "KAFKA_EXPANSION_SERVICE_JAR=${REPO_URL}/org/apache/beam/beam-sdks-java-io-expansion-service/${RELEASE_VER}/beam-sdks-java-io-expansion-service-${RELEASE_VER}.jar" >> $GITHUB_ENV - - name: Checking variables - run: | - echo $BOOTSTRAP_SERVERS - echo $KAFKA_TAXI_DF_DATASET - echo $KAFKA_EXPANSION_SERVICE_JAR - - name: Creating BigQuery Dataset - run: bq mk --project_id=${USER_GCP_PROJECT} ${KAFKA_TAXI_DF_DATASET} - - name: Running Xlang Kafka Taxi with Dataflow Runner - run: | - echo '*****************************************************'; - echo '* Running Python XLang Kafka Taxi with DataflowRunner'; - echo '*****************************************************'; - python -m apache_beam.examples.kafkataxi.kafka_taxi \ - --project=${USER_GCP_PROJECT} \ - --region=${USER_GCP_REGION} \ - --topic beam-runnerv2 \ - --bootstrap_servers ${BOOTSTRAP_SERVERS} \ - --bq_dataset ${KAFKA_TAXI_DF_DATASET} \ - --runner DataflowRunner \ - --num_workers 5 \ - --temp_location=${USER_GCS_BUCKET}/temp/ \ - --with_metadata \ - --beam_services="{\"sdks:java:io:expansion-service:shadowJar\": \"${KAFKA_EXPANSION_SERVICE_JAR}\"}" \ - --sdk_location apache-beam-${RELEASE_VER}.zip; \ - - name: Sleep Warning - run: | - echo "***************************************************************" - echo "* Please wait for at least 20 mins to let Dataflow job be launched and results get populated." - echo "* Sleeping for 20 mins" - sleep 20m - - name: Remove BigQuery Dataset - run: | - bq rm -f ${KAFKA_TAXI_DF_DATASET} - #- name: Delete Kafka Cluster - # run: gcloud container clusters delete --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --async -q $KAFKA_CLUSTER_NAME - # #From this point, steps 7 to last can be implemented as independent jobs - # The ones that uses python_versions_to_validate should use matrix instead \ No newline at end of file +# echo "" +# echo "====================Checking Environment & Variables=================" +# echo "" +# echo "running validations on release ${{github.event.inputs.RELEASE_VER}} RC${{github.event.inputs.RC_NUM}}." +# echo "repo URL for this RC: $GIT_REPO_URL" +# - name: Install Kubectl +# uses: azure/setup-kubectl@v3 +# +# - name: Setup Java JDK +# uses: actions/setup-java@v3.5.1 +# with: +# distribution: 'temurin' +# java-version: 11 +# +# - name: Install Python +# uses: actions/setup-python@v4 +# with: +# python-version: '3.8' +# cache: 'pip' # caching pip dependencies +# - name: Downloading Python Staging RC +# run: | +# echo "" +# echo "====================Starting Python Cross-language Validations===============" +# +# +# echo "---------------------Downloading Python Staging RC----------------------------" +# wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip +# wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 +# if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then +# { echo "Fail to download Python Staging RC files." ;exit 1; } +# fi +# +# echo "--------------------------Verifying Hashes------------------------------------" +# sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 +# +# `which pip` install --upgrade pip +# `which pip` install --upgrade setuptools +# +# KAFKA_CLUSTER_NAME=xlang-kafka-cluster-$RANDOM +# - name: Installing gcloud-auth-plugin +# run: sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin +# - name: Setting Kafka Cluster Name # TODO: change to random again once the tests are finished +# run: | +# echo "KAFKA_CLUSTER_NAME=xlang-kafka-cluster-12817">> $GITHUB_ENV +## - name: Creating Kafka Cluster +## run: | +## gcloud container clusters create --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --no-enable-ip-alias $KAFKA_CLUSTER_NAME +## kubectl apply -R -f .test-infra/kubernetes/kafka-cluster +# - name: Getting cluster credentials +# run: gcloud container clusters get-credentials ${KAFKA_CLUSTER_NAME} --region ${USER_GCP_REGION} --project ${USER_GCP_PROJECT} +# - name: Setting Python Environment +# run: | +# echo "--------------------------Installing Python SDK-------------------------------" +# pip install --upgrade pip setuptools wheel +# pip install apache-beam-${RELEASE_VER}.zip[gcp] +# - name: Waiting for Kafka cluster to be ready +# run: kubectl wait --for=condition=Ready pod/kafka-0 --timeout=900s +# - name: Start xlang Kafka Taxi with Dataflow Runner +# run: | +# echo "BOOTSTRAP_SERVERS=$(kubectl get svc outside-0 -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):32400" >> $GITHUB_ENV +# echo "KAFKA_TAXI_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV +# echo "KAFKA_EXPANSION_SERVICE_JAR=${REPO_URL}/org/apache/beam/beam-sdks-java-io-expansion-service/${RELEASE_VER}/beam-sdks-java-io-expansion-service-${RELEASE_VER}.jar" >> $GITHUB_ENV +# - name: Checking variables +# run: | +# echo $BOOTSTRAP_SERVERS +# echo $KAFKA_TAXI_DF_DATASET +# echo $KAFKA_EXPANSION_SERVICE_JAR +# - name: Creating BigQuery Dataset +# run: bq mk --project_id=${USER_GCP_PROJECT} ${KAFKA_TAXI_DF_DATASET} +# - name: Running Xlang Kafka Taxi with Dataflow Runner +# run: | +# echo '*****************************************************'; +# echo '* Running Python XLang Kafka Taxi with DataflowRunner'; +# echo '*****************************************************'; +# python -m apache_beam.examples.kafkataxi.kafka_taxi \ +# --project=${USER_GCP_PROJECT} \ +# --region=${USER_GCP_REGION} \ +# --topic beam-runnerv2 \ +# --bootstrap_servers ${BOOTSTRAP_SERVERS} \ +# --bq_dataset ${KAFKA_TAXI_DF_DATASET} \ +# --runner DataflowRunner \ +# --num_workers 5 \ +# --temp_location=${USER_GCS_BUCKET}/temp/ \ +# --with_metadata \ +# --beam_services="{\"sdks:java:io:expansion-service:shadowJar\": \"${KAFKA_EXPANSION_SERVICE_JAR}\"}" \ +# --sdk_location apache-beam-${RELEASE_VER}.zip; \ +# - name: Sleep Warning +# run: | +# echo "***************************************************************" +# echo "* Please wait for at least 20 mins to let Dataflow job be launched and results get populated." +# echo "* Sleeping for 20 mins" +# sleep 20m +# - name: Remove BigQuery Dataset +# run: | +# bq rm -f ${KAFKA_TAXI_DF_DATASET} +# #- name: Delete Kafka Cluster +# # run: gcloud container clusters delete --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --async -q $KAFKA_CLUSTER_NAME +# # #From this point, steps 7 to last can be implemented as independent jobs +# # The ones that uses python_versions_to_validate should use matrix instead \ No newline at end of file From 4cc0d5713ee3f36a4decf6a88083743798b085dc Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Wed, 28 Sep 2022 17:47:18 -0500 Subject: [PATCH 03/16] Testing pr from rc_tag. --- .github/workflows/run_rc_validation.yml | 38 ++++++++++++++++--------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index 5a3ac80966dd..f467694b0c13 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -5,43 +5,53 @@ on: RELEASE_VER: description: Beam version of current release required: true - default: 2.42.0 # remove default before merging - USER_GCS_BUCKET: - description: Bucket to upload results - required: false - default: gs://rc-validation-migration-tests + default: "02" # remove default before merging +# USER_GCS_BUCKET: +# description: Bucket to upload results +# required: false +# default: gs://rc-validation-migration-tests RC_NUM: description: RC number required: true default: "1" -env: # This should be a set of github secrets - USER_GCP_PROJECT: apache-beam-testing - USER_GCP_REGION: us-central1 - USER_GCP_ZONE: us-central1-a - REPO_URL: https://repository.apache.org/content/repositories/orgapachebeam-1284 +#env: # This should be a set of github secrets +# USER_GCP_PROJECT: apache-beam-testing +# USER_GCP_REGION: us-central1 +# USER_GCP_ZONE: us-central1-a +# REPO_URL: https://repository.apache.org/content/repositories/orgapachebeam-1284 jobs: python_release_candidate: #runs-on: [self-hosted, ubuntu-20.04] runs-on: ubuntu-latest + permissions: + pull-requests: write + contents: write steps: - - name: Setup Common - uses: ./.github/actions/common-rc-validation + - name: Set RC_TAG branch + run: | + echo "RC_TAG=v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}" >> $GITHUB_ENV - name: Check out code uses: actions/checkout@v3 with: ref: ${{ env.RC_TAG }} + #token: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Common + uses: ./.github/actions/common-rc-validation - name: Set git config run: | git config user.name $GITHUB_ACTOR git config user.email actions@"$RUNNER_NAME".local - name: Create Pull Request + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - git checkout -b ${{env.WORKING_BRANCH}} --quiet + git checkout -b ${{env.WORKING_BRANCH}} ${{env.RC_TAG}} --quiet touch empty_file.txt git add empty_file.txt git commit -m "Add empty file in order to create PR" --quiet git push origin ${{env.WORKING_BRANCH}} --quiet - GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t"[DO NOT MERGE] Run Python RC Validation Tests" + GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.RC_TAG}} -t"[DO NOT MERGE] Run Python RC Validation Tests") + #GITHUB_PR_URL=$(gh pr create -B ${RELEASE_BRANCH} -H ${RC_TAG} -t"[DO NOT MERGE] Run Python RC Validation Tests" - name: Comment on PR to Trigger Python ReleaseCandidate Test run: | From 331c329a5991dfae3fe8152d4ef565c10d180599 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Wed, 28 Sep 2022 18:21:11 -0500 Subject: [PATCH 04/16] Adding GH_TOKEN env var --- .github/workflows/run_rc_validation.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index f467694b0c13..99a4387b1875 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -5,7 +5,7 @@ on: RELEASE_VER: description: Beam version of current release required: true - default: "02" # remove default before merging + default: "05" # remove default before merging # USER_GCS_BUCKET: # description: Bucket to upload results # required: false @@ -26,6 +26,8 @@ jobs: permissions: pull-requests: write contents: write + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Set RC_TAG branch run: | @@ -42,16 +44,13 @@ jobs: git config user.name $GITHUB_ACTOR git config user.email actions@"$RUNNER_NAME".local - name: Create Pull Request - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - git checkout -b ${{env.WORKING_BRANCH}} ${{env.RC_TAG}} --quiet + git checkout -b ${{env.WORKING_BRANCH}} --quiet touch empty_file.txt git add empty_file.txt git commit -m "Add empty file in order to create PR" --quiet git push origin ${{env.WORKING_BRANCH}} --quiet - GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.RC_TAG}} -t"[DO NOT MERGE] Run Python RC Validation Tests") - + GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t"[DO NOT MERGE] Run Python RC Validation Tests" -b "PR to run Python ReleaseCandidate Jenkins Job.") #GITHUB_PR_URL=$(gh pr create -B ${RELEASE_BRANCH} -H ${RC_TAG} -t"[DO NOT MERGE] Run Python RC Validation Tests" - name: Comment on PR to Trigger Python ReleaseCandidate Test run: | From 88a68242bae5546b253b48d146bf33be582502da Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Thu, 29 Sep 2022 13:20:43 -0500 Subject: [PATCH 05/16] Set PR head as WORKING_BRANCH --- .github/workflows/run_rc_validation.yml | 34 +++++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index 99a4387b1875..52ced64b3917 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -1,3 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.m + name: Run RC Validation on: workflow_dispatch: @@ -29,28 +48,33 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - name: Set RC_TAG branch + - name: Set environment variables run: | echo "RC_TAG=v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}" >> $GITHUB_ENV + echo "RELEASE_BRANCH=release-${{github.event.inputs.RELEASE_VER}}" >> $GITHUB_ENV + echo "WORKING_BRANCH=v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}_validations" >> $GITHUB_ENV - name: Check out code uses: actions/checkout@v3 with: ref: ${{ env.RC_TAG }} - #token: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Common - uses: ./.github/actions/common-rc-validation +# - name: Setup Common +# uses: ./.github/actions/common-rc-validation + #- name: Setup GitHub CLI + # uses: ./.github/actions/setup-gh-cli-linux - name: Set git config run: | git config user.name $GITHUB_ACTOR git config user.email actions@"$RUNNER_NAME".local - name: Create Pull Request run: | - git checkout -b ${{env.WORKING_BRANCH}} --quiet + git checkout -b ${{env.WORKING_BRANCH}} ${{ env.RC_TAG }} --quiet touch empty_file.txt git add empty_file.txt git commit -m "Add empty file in order to create PR" --quiet git push origin ${{env.WORKING_BRANCH}} --quiet GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t"[DO NOT MERGE] Run Python RC Validation Tests" -b "PR to run Python ReleaseCandidate Jenkins Job.") +#git push origin ${{env.WORKING_BRANCH}} --quiet +#git checkout -b ${{env.WORKING_BRANCH}} --quiet #GITHUB_PR_URL=$(gh pr create -B ${RELEASE_BRANCH} -H ${RC_TAG} -t"[DO NOT MERGE] Run Python RC Validation Tests" - name: Comment on PR to Trigger Python ReleaseCandidate Test run: | From 3c238f4c96ec830376bd0df054595a18bd7ee57c Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Thu, 29 Sep 2022 16:36:35 -0500 Subject: [PATCH 06/16] Removed unused code. --- .github/workflows/run_rc_validation.yml | 129 +----------------------- 1 file changed, 3 insertions(+), 126 deletions(-) diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index 52ced64b3917..f2ee1167d440 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -25,23 +25,13 @@ on: description: Beam version of current release required: true default: "05" # remove default before merging -# USER_GCS_BUCKET: -# description: Bucket to upload results -# required: false -# default: gs://rc-validation-migration-tests RC_NUM: description: RC number required: true default: "1" -#env: # This should be a set of github secrets -# USER_GCP_PROJECT: apache-beam-testing -# USER_GCP_REGION: us-central1 -# USER_GCP_ZONE: us-central1-a -# REPO_URL: https://repository.apache.org/content/repositories/orgapachebeam-1284 jobs: python_release_candidate: - #runs-on: [self-hosted, ubuntu-20.04] - runs-on: ubuntu-latest + runs-on: [self-hosted, ubuntu-20.04] permissions: pull-requests: write contents: write @@ -57,10 +47,8 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ env.RC_TAG }} -# - name: Setup Common -# uses: ./.github/actions/common-rc-validation - #- name: Setup GitHub CLI - # uses: ./.github/actions/setup-gh-cli-linux + - name: Setup GitHub CLI + uses: ./.github/actions/setup-gh-cli-linux - name: Set git config run: | git config user.name $GITHUB_ACTOR @@ -73,117 +61,6 @@ jobs: git commit -m "Add empty file in order to create PR" --quiet git push origin ${{env.WORKING_BRANCH}} --quiet GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t"[DO NOT MERGE] Run Python RC Validation Tests" -b "PR to run Python ReleaseCandidate Jenkins Job.") -#git push origin ${{env.WORKING_BRANCH}} --quiet -#git checkout -b ${{env.WORKING_BRANCH}} --quiet -#GITHUB_PR_URL=$(gh pr create -B ${RELEASE_BRANCH} -H ${RC_TAG} -t"[DO NOT MERGE] Run Python RC Validation Tests" - name: Comment on PR to Trigger Python ReleaseCandidate Test run: | gh pr comment "$GITHUB_PR_URL" --body "Run Python ReleaseCandidate" -# python_cross_validation: -# runs-on: [self-hosted, ubuntu-20.04] -# steps: -# - name: Checkout code -# uses: actions/checkout@v3 -# - name: Common tasks -# uses: ./.github/actions/common-rc-validation -# - name: Verify ENV values -# run: | -# echo "" -# echo "====================Checking Environment & Variables=================" -# echo "" -# echo "running validations on release ${{github.event.inputs.RELEASE_VER}} RC${{github.event.inputs.RC_NUM}}." -# echo "repo URL for this RC: $GIT_REPO_URL" -# - name: Install Kubectl -# uses: azure/setup-kubectl@v3 -# -# - name: Setup Java JDK -# uses: actions/setup-java@v3.5.1 -# with: -# distribution: 'temurin' -# java-version: 11 -# -# - name: Install Python -# uses: actions/setup-python@v4 -# with: -# python-version: '3.8' -# cache: 'pip' # caching pip dependencies -# - name: Downloading Python Staging RC -# run: | -# echo "" -# echo "====================Starting Python Cross-language Validations===============" -# -# -# echo "---------------------Downloading Python Staging RC----------------------------" -# wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip -# wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 -# if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then -# { echo "Fail to download Python Staging RC files." ;exit 1; } -# fi -# -# echo "--------------------------Verifying Hashes------------------------------------" -# sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 -# -# `which pip` install --upgrade pip -# `which pip` install --upgrade setuptools -# -# KAFKA_CLUSTER_NAME=xlang-kafka-cluster-$RANDOM -# - name: Installing gcloud-auth-plugin -# run: sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin -# - name: Setting Kafka Cluster Name # TODO: change to random again once the tests are finished -# run: | -# echo "KAFKA_CLUSTER_NAME=xlang-kafka-cluster-12817">> $GITHUB_ENV -## - name: Creating Kafka Cluster -## run: | -## gcloud container clusters create --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --no-enable-ip-alias $KAFKA_CLUSTER_NAME -## kubectl apply -R -f .test-infra/kubernetes/kafka-cluster -# - name: Getting cluster credentials -# run: gcloud container clusters get-credentials ${KAFKA_CLUSTER_NAME} --region ${USER_GCP_REGION} --project ${USER_GCP_PROJECT} -# - name: Setting Python Environment -# run: | -# echo "--------------------------Installing Python SDK-------------------------------" -# pip install --upgrade pip setuptools wheel -# pip install apache-beam-${RELEASE_VER}.zip[gcp] -# - name: Waiting for Kafka cluster to be ready -# run: kubectl wait --for=condition=Ready pod/kafka-0 --timeout=900s -# - name: Start xlang Kafka Taxi with Dataflow Runner -# run: | -# echo "BOOTSTRAP_SERVERS=$(kubectl get svc outside-0 -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):32400" >> $GITHUB_ENV -# echo "KAFKA_TAXI_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV -# echo "KAFKA_EXPANSION_SERVICE_JAR=${REPO_URL}/org/apache/beam/beam-sdks-java-io-expansion-service/${RELEASE_VER}/beam-sdks-java-io-expansion-service-${RELEASE_VER}.jar" >> $GITHUB_ENV -# - name: Checking variables -# run: | -# echo $BOOTSTRAP_SERVERS -# echo $KAFKA_TAXI_DF_DATASET -# echo $KAFKA_EXPANSION_SERVICE_JAR -# - name: Creating BigQuery Dataset -# run: bq mk --project_id=${USER_GCP_PROJECT} ${KAFKA_TAXI_DF_DATASET} -# - name: Running Xlang Kafka Taxi with Dataflow Runner -# run: | -# echo '*****************************************************'; -# echo '* Running Python XLang Kafka Taxi with DataflowRunner'; -# echo '*****************************************************'; -# python -m apache_beam.examples.kafkataxi.kafka_taxi \ -# --project=${USER_GCP_PROJECT} \ -# --region=${USER_GCP_REGION} \ -# --topic beam-runnerv2 \ -# --bootstrap_servers ${BOOTSTRAP_SERVERS} \ -# --bq_dataset ${KAFKA_TAXI_DF_DATASET} \ -# --runner DataflowRunner \ -# --num_workers 5 \ -# --temp_location=${USER_GCS_BUCKET}/temp/ \ -# --with_metadata \ -# --beam_services="{\"sdks:java:io:expansion-service:shadowJar\": \"${KAFKA_EXPANSION_SERVICE_JAR}\"}" \ -# --sdk_location apache-beam-${RELEASE_VER}.zip; \ -# - name: Sleep Warning -# run: | -# echo "***************************************************************" -# echo "* Please wait for at least 20 mins to let Dataflow job be launched and results get populated." -# echo "* Sleeping for 20 mins" -# sleep 20m -# - name: Remove BigQuery Dataset -# run: | -# bq rm -f ${KAFKA_TAXI_DF_DATASET} -# #- name: Delete Kafka Cluster -# # run: gcloud container clusters delete --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --async -q $KAFKA_CLUSTER_NAME -# # #From this point, steps 7 to last can be implemented as independent jobs -# # The ones that uses python_versions_to_validate should use matrix instead \ No newline at end of file From 2b077db3267f8ae0f8535ea35d8b5e617989f651 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Thu, 29 Sep 2022 17:39:04 -0500 Subject: [PATCH 07/16] Adding Run RC Validation job in CI.md doc file. --- CI.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CI.md b/CI.md index ee911584f7f0..6c3fb6a2f22b 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,18 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### Release Preparation and Validation Workflows +#### Verify Release Build - [verify_release_build.yml](.github/workflows/verify_release_build.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|------------------------------|-----------------------------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Verify Release Build | Verifies full life cycle of Gradle Build and all PostCommit/PreCommit tests against Release Branch on Jenkins. | No | No | No | No | + +#### Run RC Validation - [run_rc_validation.yml](.github/workflows/run_rc_validation.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|------------------------------|-----------------------------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Python Release Candidate | Comment on PR to trigger Python ReleaseCandidate Jenkins job. | No | No | No | No | + + ### All migrated workflows run based on the following triggers | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Workflow Dispatch | From e3b0a78d355fe8c101ec696f9aa5ec08c25b79ae Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 4 Oct 2022 16:47:40 -0500 Subject: [PATCH 08/16] Removed Set Environment Variables step. Setting variables in env property. --- .github/workflows/run_rc_validation.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index f2ee1167d440..c3751a8cbb9b 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -37,12 +37,9 @@ jobs: contents: write env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_BRANCH: "release-${{github.event.inputs.RELEASE_VER}}" + WORKING_BRANCH: "v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}_validations" steps: - - name: Set environment variables - run: | - echo "RC_TAG=v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}" >> $GITHUB_ENV - echo "RELEASE_BRANCH=release-${{github.event.inputs.RELEASE_VER}}" >> $GITHUB_ENV - echo "WORKING_BRANCH=v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}_validations" >> $GITHUB_ENV - name: Check out code uses: actions/checkout@v3 with: From c3759f3addda34622c5ac979e35476196e98252e Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 4 Oct 2022 17:13:21 -0500 Subject: [PATCH 09/16] Removed composite rc-validation action. --- .github/actions/common-rc-validation/action.yaml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 .github/actions/common-rc-validation/action.yaml diff --git a/.github/actions/common-rc-validation/action.yaml b/.github/actions/common-rc-validation/action.yaml deleted file mode 100644 index b1976b687ecd..000000000000 --- a/.github/actions/common-rc-validation/action.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: "Setup RC Variables" -description: Setup Common RC Variables and dependecies -runs: - using: composite - steps: - - id: set_common_vars - shell: bash - run: | - echo "RC_TAG=v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}" >> $GITHUB_ENV - echo "RELEASE_BRANCH=releasev${{github.event.inputs.RELEASE_VER}}" >> $GITHUB_ENV - echo "WORKING_BRANCH=v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}_validations" >> $GITHUB_ENV - echo "GIT_REPO_URL=https://github.com/elink21/beam.git" >> $GITHUB_ENV - echo "PYTHON_RC_DOWNLOAD_URL=https://dist.apache.org/repos/dist/dev/beam" >> $GITHUB_ENV - echo "RELEASE_VER=${{github.event.inputs.RELEASE_VER}}" >> $GITHUB_ENV \ No newline at end of file From 2ab0affd26a2816841f5de93db09da5b83f72aac Mon Sep 17 00:00:00 2001 From: Elias Segundo Antonio Date: Tue, 4 Oct 2022 17:18:12 -0500 Subject: [PATCH 10/16] Rc validation workflow for Dataflow Taxi, Python Cross Validation and Runners (#227) --- .../actions/common-rc-validation/action.yaml | 27 + .github/workflows/run_rc_validation.yml | 707 +++++++++++++++++- 2 files changed, 732 insertions(+), 2 deletions(-) create mode 100644 .github/actions/common-rc-validation/action.yaml diff --git a/.github/actions/common-rc-validation/action.yaml b/.github/actions/common-rc-validation/action.yaml new file mode 100644 index 000000000000..1ce7c73962eb --- /dev/null +++ b/.github/actions/common-rc-validation/action.yaml @@ -0,0 +1,27 @@ +name: "Setup Python Env" +description: Setup Common RC Variables and dependencies +runs: + using: composite + steps: + - name: install-wheel + shell: bash + run: pip install --upgrade pip setuptools wheel + - name: Downloading Python Staging RC + shell: bash + run: | + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then + { echo "Fail to download Python Staging RC files." ;exit 1; } + fi + + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + + `which pip` install --upgrade pip + `which pip` install --upgrade setuptools + + - name: Installing python SDK + shell: bash + run: pip install apache-beam-${RELEASE_VER}.zip[gcp] diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index c3751a8cbb9b..f5b2e8fabae5 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -22,13 +22,50 @@ on: workflow_dispatch: inputs: RELEASE_VER: - description: Beam version of current release + description: Beam current Release Version required: true - default: "05" # remove default before merging + default: 2.42.0 + USER_GCS_BUCKET: + description: Bucket to upload results + required: true + default: gs://rc-validation-migration-tests RC_NUM: description: RC number required: true default: "1" + APACHE_CONTENTS_REPO: + description: Apache Contents Repository + required: true + default: https://repository.apache.org/content/repositories/orgapachebeam-1285 + RUN_SQL_TAXI_WITH_DATAFLOW: + description: Include SQL Taxi with Dataflow? + required: true + type: boolean + default: true + RUN_PYTHON_CROSS_VALIDATION: + description: Include Python Cross Validation? + required: true + type: boolean + default: true + RUN_DIRECT_RUNNER_TESTS: + description: Include Direct Runner Leaderboard & Gamestates? + required: true + type: boolean + default: true + RUN_DATAFLOW_RUNNER_TESTS: + description: Include Dataflow Runner Leaderboard & Gamestates? + type: boolean + required: true + default: true +env: + RC_TAG: "v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}" + RELEASE_VER: ${{github.event.inputs.RELEASE_VER}} + USER_GCP_PROJECT: apache-beam-testing + PYTHON_RC_DOWNLOAD_URL: https://dist.apache.org/repos/dist/dev/beam + USER_GCP_REGION: us-central1 + USER_GCP_ZONE: us-central1-a + APACHE_CONTENTS_REPO: ${{github.event.inputs.APACHE_CONTENTS_REPO}} + FIXED_WINDOW_DURATION: 20 jobs: python_release_candidate: runs-on: [self-hosted, ubuntu-20.04] @@ -61,3 +98,669 @@ jobs: - name: Comment on PR to Trigger Python ReleaseCandidate Test run: | gh pr comment "$GITHUB_PR_URL" --body "Run Python ReleaseCandidate" + sql_taxi_with_dataflow: + runs-on: [self-hosted,ubuntu-20.04] + if: ${{github.event.inputs.RUN_SQL_TAXI_WITH_DATAFLOW == 'true'}} + strategy: + matrix: + py_version: [3.8] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + repository: apache/beam + ref: ${{env.RC_TAG}} + + - name: Install Kubectl + uses: azure/setup-kubectl@v3 + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: ${{matrix.py_version}} + cache: 'pip' # caching pip dependencies + + - name: Setup Java JDK + uses: actions/setup-java@v3.5.1 + with: + distribution: 'temurin' + java-version: 11 + + - name: Downloading Python Staging RC + run: | + echo "" + echo "====================Starting Python Cross-language Validations===============" + + + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then + { echo "Fail to download Python Staging RC files." ;exit 1; } + fi + + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + + `which pip` install --upgrade pip + `which pip` install --upgrade setuptools + + - name: Setting Python Environment + run: | + echo "--------------------------Installing Python SDK-------------------------------" + pip install --upgrade pip setuptools wheel + pip install apache-beam-${RELEASE_VER}.zip[gcp] + + - name: Setting Taxi Variables + run: | + echo "SQL_TAXI_TOPIC=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV + echo "SQL_TAXI_SUBSCRIPTION=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV + echo "SQL_EXPANSION_SERVICE_JAR=${APACHE_CONTENTS_REPO}/org/apache/beam/beam-sdks-java-extensions-sql-expansion-service/${RELEASE_VER}/beam-sdks-java-extensions-sql-expansion-service-${RELEASE_VER}.jar" >> $GITHUB_ENV + - name: Creating Pub/Sub Topics + run: | + gcloud pubsub topics create --project=${USER_GCP_PROJECT} ${SQL_TAXI_TOPIC} + gcloud pubsub subscriptions create --project=${USER_GCP_PROJECT} --topic=${SQL_TAXI_TOPIC} ${SQL_TAXI_SUBSCRIPTION} + - name: XLang SQL Taxi with DataflowRunner + run: | + echo '***************************************************'; + echo '* Running Python XLang SQL Taxi with DataflowRunner'; + echo '***************************************************'; + timeout --preserve-status 20m python -m apache_beam.examples.sql_taxi \ + --project=${USER_GCP_PROJECT} \ + --region=${USER_GCP_REGION} \ + --runner DataflowRunner \ + --num_workers 5 \ + --output_topic projects/${USER_GCP_PROJECT}/topics/${SQL_TAXI_TOPIC} \ + --beam_services="{\":sdks:java:extensions:sql:expansion-service:shadowJar\": \"${SQL_EXPANSION_SERVICE_JAR}\"}" \ + --sdk_location apache-beam-${RELEASE_VER}.zip || true + - name: Checking Results + run: | + gcloud pubsub subscriptions pull --project=${USER_GCP_PROJECT} --limit=5 ${SQL_TAXI_SUBSCRIPTION} + gcloud pubsub subscriptions pull --project=${USER_GCP_PROJECT} --limit=5 ${SQL_TAXI_SUBSCRIPTION} + - name: Removing Pub Sub Topic + if: always() + run: | + gcloud pubsub topics delete --project=${USER_GCP_PROJECT} ${SQL_TAXI_TOPIC} + gcloud pubsub subscriptions delete --project=${USER_GCP_PROJECT} ${SQL_TAXI_SUBSCRIPTION} + + + python_cross_validation: + runs-on: [self-hosted,ubuntu-20.04] + if: ${{github.event.inputs.RUN_PYTHON_CROSS_VALIDATION == 'true'}} + strategy: + matrix: + py_version: [3.8] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + repository: apache/beam + ref: ${{env.RC_TAG}} + - name: Verify ENV values + run: | + echo "" + echo "====================Checking Environment & Variables=================" + echo "" + echo "running validations on release ${{github.event.inputs.RELEASE_VER}} RC${{github.event.inputs.RC_NUM}}." + - name: Install Kubectl + uses: azure/setup-kubectl@v3 + + - name: Setup Java JDK + uses: actions/setup-java@v3.5.1 + with: + distribution: 'temurin' + java-version: 11 + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: ${{matrix.py_version}} + cache: 'pip' + ######## + - name: install-wheel + shell: bash + run: pip install --upgrade pip setuptools wheel + - name: Downloading Python Staging RC + shell: bash + run: | + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then + { echo "Fail to download Python Staging RC files." ;exit 1; } + fi + + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + + `which pip` install --upgrade pip + `which pip` install --upgrade setuptools + - name: Installing python SDK + shell: bash + run: pip install apache-beam-${RELEASE_VER}.zip[gcp] + + - name: Donwloading Python RC + shell: bash + run: | + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + + - name: Verifying Hashes + shell: bash + run: | + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + ######## + + - name: Installing gcloud-auth-plugin + run: sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin + - name: Setting Kafka Cluster Name + run: | + echo "KAFKA_CLUSTER_NAME=xlang-kafka-cluster-$RANDOM">> $GITHUB_ENV + + + - name: Creating Kafka Cluster + run: | + gcloud container clusters create --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --no-enable-ip-alias $KAFKA_CLUSTER_NAME + kubectl apply -R -f .test-infra/kubernetes/kafka-cluster + + - name: Setting Python Environment + run: | + echo "--------------------------Installing Python SDK-------------------------------" + pip install --upgrade pip setuptools wheel + pip install apache-beam-${RELEASE_VER}.zip[gcp] + - name: Waiting for Kafka cluster to be ready + run: kubectl wait --for=condition=Ready pod/kafka-0 --timeout=1200s + - name: Start xlang Kafka Taxi with Dataflow Runner + run: | + echo "BOOTSTRAP_SERVERS=$(kubectl get svc outside-0 -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):32400" >> $GITHUB_ENV + echo "KAFKA_TAXI_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV + echo "KAFKA_EXPANSION_SERVICE_JAR=${APACHE_CONTENTS}/org/apache/beam/beam-sdks-java-io-expansion-service/${RELEASE_VER}/beam-sdks-java-io-expansion-service-${RELEASE_VER}.jar" >> $GITHUB_ENV + - name: Checking variables + run: | + echo $BOOTSTRAP_SERVERS + echo $KAFKA_TAXI_DF_DATASET + echo $KAFKA_EXPANSION_SERVICE_JAR + - name: Creating BigQuery Dataset + run: bq mk --project_id=${USER_GCP_PROJECT} ${KAFKA_TAXI_DF_DATASET} + - name: Running Xlang Kafka Taxi with Dataflow Runner + run: | + echo '*****************************************************'; + echo '* Running Python XLang Kafka Taxi with DataflowRunner'; + echo '*****************************************************'; + echo "***************************************************************"; + timeout --preserve-status 20m python -m apache_beam.examples.kafkataxi.kafka_taxi \ + --project=${USER_GCP_PROJECT} \ + --region=${USER_GCP_REGION} \ + --topic beam-runnerv2 \ + --bootstrap_servers ${BOOTSTRAP_SERVERS} \ + --bq_dataset ${KAFKA_TAXI_DF_DATASET} \ + --runner DataflowRunner \ + --num_workers 5 \ + --temp_location=${USER_GCS_BUCKET}/temp/ \ + --with_metadata \ + --beam_services="{\"sdks:java:io:expansion-service:shadowJar\": \"${KAFKA_EXPANSION_SERVICE_JAR}\"}" \ + --sdk_location apache-beam-${RELEASE_VER}.zip || true + - name: Checking executions results + run: | + bq head -n 10 ${KAFKA_TAXI_DF_DATASET}.xlang_kafka_taxi + - name: Remove BigQuery Dataset + if: always() + run: | + bq rm -f ${KAFKA_TAXI_DF_DATASET}.xlang_kafka_taxi + bq rm -f ${KAFKA_TAXI_DF_DATASET} + + - name: Delete Kafka Cluster + if: always() + run: gcloud container clusters delete --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --async -q $KAFKA_CLUSTER_NAME + + + generate_shared_pubsub: + outputs: + name: ${{ steps.generate_pubsub_name.outputs.pubsub }} + runs-on: [self-hosted,ubuntu-20.04] + if: ${{github.event.inputs.RUN_DIRECT_RUNNER_TESTS == 'true' || github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS== 'true'}} + steps: + - name: Sending PubSub name to env + run: | + echo "SHARED_PUBSUB_TOPIC=leader_board-${GITHUB_ACTOR}-python-topic-$(date +%m%d)_$RANDOM" >> $GITHUB_ENV + - id: generate_pubsub_name + run: | + echo "::set-output name=pubsub::$SHARED_PUBSUB_TOPIC" + - name: Creating Pub Sub Topics + run: gcloud pubsub topics create --project=${USER_GCP_PROJECT} ${SHARED_PUBSUB_TOPIC} + + + java_injector: + runs-on: [self-hosted,ubuntu-20.04] + if: ${{github.event.inputs.RUN_DIRECT_RUNNER_TESTS == 'true' || github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS== 'true'}} + needs: generate_shared_pubsub + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + repository: apache/beam + ref: ${{env.RC_TAG}} + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + cache: 'pip' # caching pip dependencies + ######## + - name: install-wheel + shell: bash + run: pip install --upgrade pip setuptools wheel + - name: Downloading Python Staging RC + shell: bash + run: | + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then + { echo "Fail to download Python Staging RC files." ;exit 1; } + fi + + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + + `which pip` install --upgrade pip + `which pip` install --upgrade setuptools + - name: Installing python SDK + shell: bash + run: pip install apache-beam-${RELEASE_VER}.zip[gcp] + + - name: Donwloading Python RC + shell: bash + run: | + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + + - name: Verifying Hashes + shell: bash + run: | + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + ######## + - name: Setup Maven Action + uses: s4u/setup-maven-action@v1.2.1 + with: + java-version: 11 + - name: Updating Settings + run: | + cd ~ + if [[ ! -d .m2 ]]; then + mkdir .m2 + fi + cd .m2 + if [[ -f ~/.m2/settings.xml ]]; then + mv settings.xml /tmp/ + fi + touch settings.xml + echo "" >> settings.xml + echo " " >> settings.xml + echo " " >> settings.xml + echo " release-repo" >> settings.xml + echo " " >> settings.xml + echo " true" >> settings.xml + echo " " >> settings.xml + echo " " >> settings.xml + echo " " >> settings.xml + echo " Release ${RELEASE_VER} RC${{github.event.inputs.RC_NUM}}" >> settings.xml + echo " Release ${RELEASE_VER} RC${{github.event.inputs.RC_NUM}}" >> settings.xml + echo " ${APACHE_CONTENTS_REPO}" >> settings.xml + echo " " >> settings.xml + echo " " >> settings.xml + echo " " >> settings.xml + echo " " >> settings.xml + echo "" >> settings.xml + cat settings.xml + - name: Generate Maven archetype + run: | + mvn archetype:generate \ + -DarchetypeGroupId=org.apache.beam \ + -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \ + -DarchetypeVersion=2.41.0 \ + -DgroupId=org.example \ + -DartifactId=word-count-beam \ + -Dversion="0.1" \ + -Dpackage=org.apache.beam.examples \ + -DinteractiveMode=false \ + -DarchetypeCatalog=internal + - name: Running Pub/Sub Java injector + run: | + ls + cd word-count-beam + timeout --preserve-status 55m mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector -Dexec.args="${USER_GCP_PROJECT} ${{needs.generate_shared_pubsub.outputs.name}} none" || true + + + direct_runner_leaderboard: + runs-on: [self-hosted, ubuntu-20.04] + if: ${{github.event.inputs.RUN_DIRECT_RUNNER_TESTS == 'true' }} + strategy: + matrix: + py_version: [3.8] + needs: generate_shared_pubsub + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + repository: apache/beam + ref: ${{env.RC_TAG}} + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: ${{matrix.py_version}} + ######## + - name: install-wheel + shell: bash + run: pip install --upgrade pip setuptools wheel + - name: Downloading Python Staging RC + shell: bash + run: | + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then + { echo "Fail to download Python Staging RC files." ;exit 1; } + fi + + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + + `which pip` install --upgrade pip + `which pip` install --upgrade setuptools + - name: Installing python SDK + shell: bash + run: pip install apache-beam-${RELEASE_VER}.zip[gcp] + + - name: Donwloading Python RC + shell: bash + run: | + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + + - name: Verifying Hashes + shell: bash + run: | + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + ######## + - name: Exporting leaderboard Dataset + run: echo "LEADERBOARD_DIRECT_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV + - name: Creating Dataset + run: bq mk --project_id=${USER_GCP_PROJECT} ${LEADERBOARD_DIRECT_DATASET} + - name: Starting Leaderboard with DirectRunner + run: | + echo '*****************************************************'; + echo '* Running Python Leaderboard with DirectRunner'; + echo '*****************************************************'; + timeout --preserve-status 5m python -m apache_beam.examples.complete.game.leader_board \ + --project=${USER_GCP_PROJECT} \ + --topic projects/${USER_GCP_PROJECT}/topics/${{needs.generate_shared_pubsub.outputs.name}} \ + --dataset ${LEADERBOARD_DIRECT_DATASET} || true + - name: Checking results + run: | + bq head -n 10 ${LEADERBOARD_DIRECT_DATASET}.leader_board_users + bq head -n 10 ${LEADERBOARD_DIRECT_DATASET}.leader_board_teams + - name: Removing BigQuery Dataset + if: always() + run: | + bq rm -f ${LEADERBOARD_DIRECT_DATASET}.leader_board_users + bq rm -f ${LEADERBOARD_DIRECT_DATASET}.leader_board_teams + bq rm -f $LEADERBOARD_DIRECT_DATASET + dataflow_runner_leaderboard: + runs-on: [self-hosted,ubuntu-20.04] + if: ${{github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS=='true'}} + strategy: + matrix: + py_version: [3.8] + needs: [generate_shared_pubsub] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + repository: apache/beam + ref: ${{env.RC_TAG}} + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: ${{matrix.py_version}} + + ######## + - name: install-wheel + shell: bash + run: pip install --upgrade pip setuptools wheel + - name: Downloading Python Staging RC + shell: bash + run: | + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then + { echo "Fail to download Python Staging RC files." ;exit 1; } + fi + + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + + `which pip` install --upgrade pip + `which pip` install --upgrade setuptools + - name: Installing python SDK + shell: bash + run: pip install apache-beam-${RELEASE_VER}.zip[gcp] + + - name: Donwloading Python RC + shell: bash + run: | + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + + - name: Verifying Hashes + shell: bash + run: | + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + ######## + - name: Exporting Dataflow Dataset + run: echo "LEADERBOARD_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV + - name: Creating Dataset + run: bq mk --project_id=${USER_GCP_PROJECT} ${LEADERBOARD_DF_DATASET} + - name: Starting Leaderboard with Dataflow + run: | + echo '*****************************************************'; + echo '* Running Python Leaderboard with DataflowRunner'; + echo '*****************************************************'; + timeout --preserve-status 20m python -m apache_beam.examples.complete.game.leader_board \ + --project=${USER_GCP_PROJECT} \ + --region=${USER_GCP_REGION} \ + --topic projects/${USER_GCP_PROJECT}/topics/${{needs.generate_shared_pubsub.outputs.name}} \ + --dataset ${LEADERBOARD_DF_DATASET} \ + --runner DataflowRunner \ + --temp_location=${USER_GCS_BUCKET}/temp/ \ + --sdk_location apache-beam-${RELEASE_VER}.zip || true + - name: Checking results + run: | + bq head -n 10 ${LEADERBOARD_DF_DATASET}.leader_board_users + bq head -n 10 ${LEADERBOARD_DF_DATASET}.leader_board_teams + - name: Removing BigQuery Dataset + if: always() + run: | + bq rm -f ${LEADERBOARD_DF_DATASET}.leader_board_users + bq rm -f ${LEADERBOARD_DF_DATASET}.leader_board_teams + bq rm -f $LEADERBOARD_DF_DATASET + + + direct_runner_gamestats: + runs-on: [self-hosted,ubuntu-20.04] + if: ${{github.event.inputs.RUN_DIRECT_RUNNER_TESTS == 'true' }} + strategy: + matrix: + py_version: [3.8] + needs: [generate_shared_pubsub] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + repository: apache/beam + ref: ${{env.RC_TAG}} + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: ${{matrix.py_version}} + cache: 'pip' # caching pip dependencies + ######## + - name: install-wheel + shell: bash + run: pip install --upgrade pip setuptools wheel + - name: Downloading Python Staging RC + shell: bash + run: | + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then + { echo "Fail to download Python Staging RC files." ;exit 1; } + fi + + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + + `which pip` install --upgrade pip + `which pip` install --upgrade setuptools + - name: Installing python SDK + shell: bash + run: pip install apache-beam-${RELEASE_VER}.zip[gcp] + + - name: Donwloading Python RC + shell: bash + run: | + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + + - name: Verifying Hashes + shell: bash + run: | + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + ######## + - name: Exporting Gamestates Direct Dataset + run: echo "GAMESTATS_DIRECT_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV + - name: Creating Dataset + run: bq mk --project_id=${USER_GCP_PROJECT} ${GAMESTATS_DIRECT_DATASET} + - name: Starting Gamestats with DirectRunner + run: | + echo '*****************************************************'; + echo '* Running GameStats with DirectRunner'; + echo '*****************************************************'; + timeout --preserve-status 40m python -m apache_beam.examples.complete.game.game_stats \ + --project=${USER_GCP_PROJECT} \ + --topic projects/${USER_GCP_PROJECT}/topics/${{needs.generate_shared_pubsub.outputs.name}} \ + --dataset ${GAMESTATS_DIRECT_DATASET} \ + --fixed_window_duration ${FIXED_WINDOW_DURATION} || true + - name: Checking Results + run: | + bq head -n 10 ${GAMESTATS_DIRECT_DATASET}.game_stats_teams + bq head -n 10 ${GAMESTATS_DIRECT_DATASET}.game_stats_sessions + - name: Removing BigQuery Dataset + if: always() + run: | + bq rm -f ${GAMESTATS_DIRECT_DATASET}.game_stats_sessions + bq rm -f ${GAMESTATS_DIRECT_DATASET}.game_stats_teams + bq rm -f $GAMESTATS_DIRECT_DATASET + + + dataflow_runner_gamestats: + runs-on: [self-hosted,ubuntu-20.04] + if: ${{github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS=='true'}} + strategy: + matrix: + py_version: [3.8] + needs: [generate_shared_pubsub] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + repository: apache/beam + ref: ${{env.RC_TAG}} + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: ${{matrix.py_version}} + ######## + - name: install-wheel + shell: bash + run: pip install --upgrade pip setuptools wheel + - name: Downloading Python Staging RC + shell: bash + run: | + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then + { echo "Fail to download Python Staging RC files." ;exit 1; } + fi + + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + + `which pip` install --upgrade pip + `which pip` install --upgrade setuptools + - name: Installing python SDK + shell: bash + run: pip install apache-beam-${RELEASE_VER}.zip[gcp] + + - name: Donwloading Python RC + shell: bash + run: | + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + + - name: Verifying Hashes + shell: bash + run: | + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 + ######## + - name: Exporting Gamestates Direct Dataset + run: echo "GAMESTATS_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV + - name: Creating Dataset + run: bq mk --project_id=${USER_GCP_PROJECT} ${GAMESTATS_DF_DATASET} + - name: Starting Gamestats with DirectRunner + run: | + echo '*****************************************************'; + echo '* Running GameStats with DataflowRunner'; + echo '*****************************************************'; + timeout --preserve-status 50m python -m apache_beam.examples.complete.game.game_stats \ + --project=${USER_GCP_PROJECT} \ + --region=${USER_GCP_REGION} \ + --topic projects/${USER_GCP_PROJECT}/topics/${{needs.generate_shared_pubsub.outputs.name}} \ + --dataset ${GAMESTATS_DF_DATASET} \ + --runner DataflowRunner \ + --temp_location=${USER_GCS_BUCKET}/temp/ \ + --sdk_location apache-beam-${RELEASE_VER}.zip \ + --fixed_window_duration ${FIXED_WINDOW_DURATION} || true + - name: Checking Results + run: | + bq head -n 10 ${GAMESTATS_DF_DATASET}.game_stats_teams + bq head -n 10 ${GAMESTATS_DF_DATASET}.game_stats_sessions + - name: Removing BigQuery Dataset + if: always() + run: | + bq rm -f ${GAMESTATS_DF_DATASET}.game_stats_teams + bq rm -f ${GAMESTATS_DF_DATASET}.game_stats_sessions + bq rm -f $GAMESTATS_DF_DATASET + remove_shared_pubsub: + runs-on: [self-hosted,ubuntu-20.04] + needs: [java_injector] + if: ${{github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS=='true' || github.event.inputs.RUN_DIRECT_RUNNER_TESTS == 'true'}} + steps: + - name: Deleting Shared Pub Sub + run: gcloud pubsub topics delete --project=${USER_GCP_PROJECT} ${{needs.generate_shared_pubsub.outputs.name}} From d3e2a9ac59769b751bfb1cea272adcaae206bee1 Mon Sep 17 00:00:00 2001 From: Elias Segundo Antonio Date: Wed, 5 Oct 2022 17:52:41 -0500 Subject: [PATCH 11/16] RC Validation Workflow (#228) Adding the following jobs: * sql_taxi_with_dataflow * python_cross_validation * generate_shared_pubsub * java_injector * direct_runner_leaderboard * dataflow_runner_leaderboard * direct_runner_gamestats * dataflow_runner_gamestats * remove_shared_pubsub --- .github/workflows/run_rc_validation.yml | 316 ++++-------------------- 1 file changed, 42 insertions(+), 274 deletions(-) diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index f5b2e8fabae5..0c30e18696ae 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -82,6 +82,7 @@ jobs: with: ref: ${{ env.RC_TAG }} - name: Setup GitHub CLI + - id: setup-gh-cli-linux uses: ./.github/actions/setup-gh-cli-linux - name: Set git config run: | @@ -93,11 +94,12 @@ jobs: touch empty_file.txt git add empty_file.txt git commit -m "Add empty file in order to create PR" --quiet - git push origin ${{env.WORKING_BRANCH}} --quiet + git push origin ${{env.WORKING_BRANCH}} --quiet GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t"[DO NOT MERGE] Run Python RC Validation Tests" -b "PR to run Python ReleaseCandidate Jenkins Job.") - name: Comment on PR to Trigger Python ReleaseCandidate Test run: | gh pr comment "$GITHUB_PR_URL" --body "Run Python ReleaseCandidate" + sql_taxi_with_dataflow: runs-on: [self-hosted,ubuntu-20.04] if: ${{github.event.inputs.RUN_SQL_TAXI_WITH_DATAFLOW == 'true'}} @@ -107,18 +109,14 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 - with: - repository: apache/beam - ref: ${{env.RC_TAG}} - - - name: Install Kubectl - uses: azure/setup-kubectl@v3 + with: + ref: ${{env.RC_TAG}} - name: Install Python uses: actions/setup-python@v4 with: python-version: ${{matrix.py_version}} - cache: 'pip' # caching pip dependencies + - name: Setup Java JDK uses: actions/setup-java@v3.5.1 @@ -126,30 +124,9 @@ jobs: distribution: 'temurin' java-version: 11 - - name: Downloading Python Staging RC - run: | - echo "" - echo "====================Starting Python Cross-language Validations===============" - - - echo "---------------------Downloading Python Staging RC----------------------------" - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 - if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then - { echo "Fail to download Python Staging RC files." ;exit 1; } - fi - - echo "--------------------------Verifying Hashes------------------------------------" - sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 - - `which pip` install --upgrade pip - `which pip` install --upgrade setuptools - - - name: Setting Python Environment - run: | - echo "--------------------------Installing Python SDK-------------------------------" - pip install --upgrade pip setuptools wheel - pip install apache-beam-${RELEASE_VER}.zip[gcp] + - name: Setting python env + uses: ./.github/actions/common-rc-validation + - name: Setting Taxi Variables run: | @@ -194,7 +171,6 @@ jobs: - name: Checkout code uses: actions/checkout@v3 with: - repository: apache/beam ref: ${{env.RC_TAG}} - name: Verify ENV values run: | @@ -215,43 +191,10 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{matrix.py_version}} - cache: 'pip' - ######## - - name: install-wheel - shell: bash - run: pip install --upgrade pip setuptools wheel - - name: Downloading Python Staging RC - shell: bash - run: | - echo "---------------------Downloading Python Staging RC----------------------------" - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 - if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then - { echo "Fail to download Python Staging RC files." ;exit 1; } - fi + - echo "--------------------------Verifying Hashes------------------------------------" - sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 - - `which pip` install --upgrade pip - `which pip` install --upgrade setuptools - - name: Installing python SDK - shell: bash - run: pip install apache-beam-${RELEASE_VER}.zip[gcp] - - - name: Donwloading Python RC - shell: bash - run: | - echo "---------------------Downloading Python Staging RC----------------------------" - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 - - - name: Verifying Hashes - shell: bash - run: | - echo "--------------------------Verifying Hashes------------------------------------" - sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 - ######## + - name: Setting python env + uses: ./.github/actions/common-rc-validation - name: Installing gcloud-auth-plugin run: sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin @@ -265,23 +208,14 @@ jobs: gcloud container clusters create --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --no-enable-ip-alias $KAFKA_CLUSTER_NAME kubectl apply -R -f .test-infra/kubernetes/kafka-cluster - - name: Setting Python Environment - run: | - echo "--------------------------Installing Python SDK-------------------------------" - pip install --upgrade pip setuptools wheel - pip install apache-beam-${RELEASE_VER}.zip[gcp] - name: Waiting for Kafka cluster to be ready run: kubectl wait --for=condition=Ready pod/kafka-0 --timeout=1200s - name: Start xlang Kafka Taxi with Dataflow Runner run: | echo "BOOTSTRAP_SERVERS=$(kubectl get svc outside-0 -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):32400" >> $GITHUB_ENV echo "KAFKA_TAXI_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV - echo "KAFKA_EXPANSION_SERVICE_JAR=${APACHE_CONTENTS}/org/apache/beam/beam-sdks-java-io-expansion-service/${RELEASE_VER}/beam-sdks-java-io-expansion-service-${RELEASE_VER}.jar" >> $GITHUB_ENV - - name: Checking variables - run: | - echo $BOOTSTRAP_SERVERS - echo $KAFKA_TAXI_DF_DATASET - echo $KAFKA_EXPANSION_SERVICE_JAR + echo "KAFKA_EXPANSION_SERVICE_JAR=${APACHE_CONTENTS_REPO}/org/apache/beam/beam-sdks-java-io-expansion-service/${RELEASE_VER}/beam-sdks-java-io-expansion-service-${RELEASE_VER}.jar" >> $GITHUB_ENV + - name: Creating BigQuery Dataset run: bq mk --project_id=${USER_GCP_PROJECT} ${KAFKA_TAXI_DF_DATASET} - name: Running Xlang Kafka Taxi with Dataflow Runner @@ -340,49 +274,15 @@ jobs: - name: Checkout code uses: actions/checkout@v3 with: - repository: apache/beam ref: ${{env.RC_TAG}} - name: Install Python uses: actions/setup-python@v4 with: python-version: '3.8' - cache: 'pip' # caching pip dependencies - ######## - - name: install-wheel - shell: bash - run: pip install --upgrade pip setuptools wheel - - name: Downloading Python Staging RC - shell: bash - run: | - echo "---------------------Downloading Python Staging RC----------------------------" - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 - if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then - { echo "Fail to download Python Staging RC files." ;exit 1; } - fi - - echo "--------------------------Verifying Hashes------------------------------------" - sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 - - `which pip` install --upgrade pip - `which pip` install --upgrade setuptools - - name: Installing python SDK - shell: bash - run: pip install apache-beam-${RELEASE_VER}.zip[gcp] - - - name: Donwloading Python RC - shell: bash - run: | - echo "---------------------Downloading Python Staging RC----------------------------" - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 + + - name: Setting python env + uses: ./.github/actions/common-rc-validation - - name: Verifying Hashes - shell: bash - run: | - echo "--------------------------Verifying Hashes------------------------------------" - sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 - ######## - name: Setup Maven Action uses: s4u/setup-maven-action@v1.2.1 with: @@ -432,7 +332,7 @@ jobs: run: | ls cd word-count-beam - timeout --preserve-status 55m mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector -Dexec.args="${USER_GCP_PROJECT} ${{needs.generate_shared_pubsub.outputs.name}} none" || true + timeout --preserve-status 50m mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector -Dexec.args="${USER_GCP_PROJECT} ${{needs.generate_shared_pubsub.outputs.name}} none" || true direct_runner_leaderboard: @@ -446,50 +346,17 @@ jobs: - name: Checkout code uses: actions/checkout@v3 with: - repository: apache/beam ref: ${{env.RC_TAG}} - name: Install Python uses: actions/setup-python@v4 with: python-version: ${{matrix.py_version}} - ######## - - name: install-wheel - shell: bash - run: pip install --upgrade pip setuptools wheel - - name: Downloading Python Staging RC - shell: bash - run: | - echo "---------------------Downloading Python Staging RC----------------------------" - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 - if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then - { echo "Fail to download Python Staging RC files." ;exit 1; } - fi - - echo "--------------------------Verifying Hashes------------------------------------" - sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 - - `which pip` install --upgrade pip - `which pip` install --upgrade setuptools - - name: Installing python SDK - shell: bash - run: pip install apache-beam-${RELEASE_VER}.zip[gcp] - - - name: Donwloading Python RC - shell: bash - run: | - echo "---------------------Downloading Python Staging RC----------------------------" - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 - - - name: Verifying Hashes - shell: bash - run: | - echo "--------------------------Verifying Hashes------------------------------------" - sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 - ######## - - name: Exporting leaderboard Dataset + + - name: Setting python env + uses: ./.github/actions/common-rc-validation + + - name: Exporting leaderboard Dataset Name run: echo "LEADERBOARD_DIRECT_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV - name: Creating Dataset run: bq mk --project_id=${USER_GCP_PROJECT} ${LEADERBOARD_DIRECT_DATASET} @@ -512,6 +379,7 @@ jobs: bq rm -f ${LEADERBOARD_DIRECT_DATASET}.leader_board_users bq rm -f ${LEADERBOARD_DIRECT_DATASET}.leader_board_teams bq rm -f $LEADERBOARD_DIRECT_DATASET + dataflow_runner_leaderboard: runs-on: [self-hosted,ubuntu-20.04] if: ${{github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS=='true'}} @@ -523,7 +391,6 @@ jobs: - name: Checkout code uses: actions/checkout@v3 with: - repository: apache/beam ref: ${{env.RC_TAG}} - name: Install Python @@ -531,43 +398,10 @@ jobs: with: python-version: ${{matrix.py_version}} - ######## - - name: install-wheel - shell: bash - run: pip install --upgrade pip setuptools wheel - - name: Downloading Python Staging RC - shell: bash - run: | - echo "---------------------Downloading Python Staging RC----------------------------" - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 - if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then - { echo "Fail to download Python Staging RC files." ;exit 1; } - fi - - echo "--------------------------Verifying Hashes------------------------------------" - sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 - - `which pip` install --upgrade pip - `which pip` install --upgrade setuptools - - name: Installing python SDK - shell: bash - run: pip install apache-beam-${RELEASE_VER}.zip[gcp] - - - name: Donwloading Python RC - shell: bash - run: | - echo "---------------------Downloading Python Staging RC----------------------------" - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 - - - name: Verifying Hashes - shell: bash - run: | - echo "--------------------------Verifying Hashes------------------------------------" - sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 - ######## - - name: Exporting Dataflow Dataset + - name: Setting python env + uses: ./.github/actions/common-rc-validation + + - name: Exporting Dataflow Dataset Name run: echo "LEADERBOARD_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV - name: Creating Dataset run: bq mk --project_id=${USER_GCP_PROJECT} ${LEADERBOARD_DF_DATASET} @@ -607,50 +441,16 @@ jobs: - name: Checkout code uses: actions/checkout@v3 with: - repository: apache/beam ref: ${{env.RC_TAG}} - name: Install Python uses: actions/setup-python@v4 with: python-version: ${{matrix.py_version}} - cache: 'pip' # caching pip dependencies - ######## - - name: install-wheel - shell: bash - run: pip install --upgrade pip setuptools wheel - - name: Downloading Python Staging RC - shell: bash - run: | - echo "---------------------Downloading Python Staging RC----------------------------" - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 - if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then - { echo "Fail to download Python Staging RC files." ;exit 1; } - fi - - echo "--------------------------Verifying Hashes------------------------------------" - sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 - - `which pip` install --upgrade pip - `which pip` install --upgrade setuptools - - name: Installing python SDK - shell: bash - run: pip install apache-beam-${RELEASE_VER}.zip[gcp] - - - name: Donwloading Python RC - shell: bash - run: | - echo "---------------------Downloading Python Staging RC----------------------------" - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 - - - name: Verifying Hashes - shell: bash - run: | - echo "--------------------------Verifying Hashes------------------------------------" - sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 - ######## - - name: Exporting Gamestates Direct Dataset + + - name: Setting python env + uses: ./.github/actions/common-rc-validation + + - name: Exporting Gamestates Direct Dataset Name run: echo "GAMESTATS_DIRECT_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV - name: Creating Dataset run: bq mk --project_id=${USER_GCP_PROJECT} ${GAMESTATS_DIRECT_DATASET} @@ -687,49 +487,16 @@ jobs: - name: Checkout code uses: actions/checkout@v3 with: - repository: apache/beam ref: ${{env.RC_TAG}} - name: Install Python uses: actions/setup-python@v4 with: python-version: ${{matrix.py_version}} - ######## - - name: install-wheel - shell: bash - run: pip install --upgrade pip setuptools wheel - - name: Downloading Python Staging RC - shell: bash - run: | - echo "---------------------Downloading Python Staging RC----------------------------" - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 - if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then - { echo "Fail to download Python Staging RC files." ;exit 1; } - fi - - echo "--------------------------Verifying Hashes------------------------------------" - sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 - - `which pip` install --upgrade pip - `which pip` install --upgrade setuptools - - name: Installing python SDK - shell: bash - run: pip install apache-beam-${RELEASE_VER}.zip[gcp] - - - name: Donwloading Python RC - shell: bash - run: | - echo "---------------------Downloading Python Staging RC----------------------------" - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip - wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512 - - - name: Verifying Hashes - shell: bash - run: | - echo "--------------------------Verifying Hashes------------------------------------" - sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512 - ######## - - name: Exporting Gamestates Direct Dataset + + - name: Setting python env + uses: ./.github/actions/common-rc-validation + + - name: Exporting Gamestates Direct Dataset Name run: echo "GAMESTATS_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV - name: Creating Dataset run: bq mk --project_id=${USER_GCP_PROJECT} ${GAMESTATS_DF_DATASET} @@ -738,7 +505,7 @@ jobs: echo '*****************************************************'; echo '* Running GameStats with DataflowRunner'; echo '*****************************************************'; - timeout --preserve-status 50m python -m apache_beam.examples.complete.game.game_stats \ + timeout --preserve-status 45m python -m apache_beam.examples.complete.game.game_stats \ --project=${USER_GCP_PROJECT} \ --region=${USER_GCP_REGION} \ --topic projects/${USER_GCP_PROJECT}/topics/${{needs.generate_shared_pubsub.outputs.name}} \ @@ -757,10 +524,11 @@ jobs: bq rm -f ${GAMESTATS_DF_DATASET}.game_stats_teams bq rm -f ${GAMESTATS_DF_DATASET}.game_stats_sessions bq rm -f $GAMESTATS_DF_DATASET + remove_shared_pubsub: runs-on: [self-hosted,ubuntu-20.04] - needs: [java_injector] - if: ${{github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS=='true' || github.event.inputs.RUN_DIRECT_RUNNER_TESTS == 'true'}} + needs: [java_injector, generate_shared_pubsub] + if: always() steps: - name: Deleting Shared Pub Sub run: gcloud pubsub topics delete --project=${USER_GCP_PROJECT} ${{needs.generate_shared_pubsub.outputs.name}} From 61a19940a510ab2c640b7f2861d4230c4b3605d8 Mon Sep 17 00:00:00 2001 From: Elias Segundo Antonio Date: Wed, 5 Oct 2022 18:23:27 -0500 Subject: [PATCH 12/16] Adding extras jobs to CI.MD (#230) --- CI.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CI.md b/CI.md index 6c3fb6a2f22b..3f7e5642aae9 100644 --- a/CI.md +++ b/CI.md @@ -135,7 +135,12 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | |------------------------------|-----------------------------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| | Python Release Candidate | Comment on PR to trigger Python ReleaseCandidate Jenkins job. | No | No | No | No | - +| Python XLang SQL Taxi | Runs Python XLang SQL Taxi with DataflowRunner | No | No | No | Yes | +| Python XLang Kafka | Runs Python XLang Kafka Taxi with DataflowRunner | No | No | No | Yes | +| Direct Runner Leaderboard | Runs Python Leaderboard with DirectRunner | No | No | No | Yes | +| Direct Runner GameStats | Runs Python GameStats with DirectRunner. | No | No | No | Yes | +| Dataflow Runner Leaderboard | Runs Python Leaderboard with DataflowRunner | No | No | No | Yes | +| Dataflow Runner GameStats | Runs Python GameStats with DataflowRunner | No | No | No | Yes | ### All migrated workflows run based on the following triggers From 8aa85f5cdef5c0e9cc04f69a99eeff7fc839ca78 Mon Sep 17 00:00:00 2001 From: Elias Segundo Antonio Date: Thu, 6 Oct 2022 10:28:38 -0500 Subject: [PATCH 13/16] Fixing workflow linter error (#231) --- .github/workflows/run_rc_validation.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index 0c30e18696ae..c7aff30054b5 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -17,7 +17,7 @@ # To learn more about GitHub Actions in Apache Beam check the CI.m -name: Run RC Validation +name: Run RC Validation on: workflow_dispatch: inputs: @@ -82,7 +82,6 @@ jobs: with: ref: ${{ env.RC_TAG }} - name: Setup GitHub CLI - - id: setup-gh-cli-linux uses: ./.github/actions/setup-gh-cli-linux - name: Set git config run: | From 203b3a116ca70307198e7d65ad57c7dffeab05d6 Mon Sep 17 00:00:00 2001 From: Elias Segundo Antonio Date: Mon, 10 Oct 2022 18:07:38 -0500 Subject: [PATCH 14/16] Using inputs in composite action (#235) * Using inputs in composite action Co-authored-by: Elias Segundo --- .../actions/common-rc-validation/action.yaml | 27 +++++++++++++++++++ .github/workflows/run_rc_validation.yml | 23 ++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/.github/actions/common-rc-validation/action.yaml b/.github/actions/common-rc-validation/action.yaml index 1ce7c73962eb..23efa93d1533 100644 --- a/.github/actions/common-rc-validation/action.yaml +++ b/.github/actions/common-rc-validation/action.yaml @@ -1,5 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + name: "Setup Python Env" description: Setup Common RC Variables and dependencies +inputs: + PYTHON_RC_DOWNLOAD_URL: + description: Python RC package URL + required: true + default: https://dist.apache.org/repos/dist/dev/beam + RELEASE_VER: + description: Release Version + required: true + runs: using: composite steps: @@ -25,3 +51,4 @@ runs: - name: Installing python SDK shell: bash run: pip install apache-beam-${RELEASE_VER}.zip[gcp] + \ No newline at end of file diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index c7aff30054b5..bc5dd44087e8 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -125,8 +125,12 @@ jobs: - name: Setting python env uses: ./.github/actions/common-rc-validation + with: + RELEASE_VER: ${{env.RELEASE_VER}} + PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}} + - name: Setting Taxi Variables run: | echo "SQL_TAXI_TOPIC=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV @@ -194,6 +198,9 @@ jobs: - name: Setting python env uses: ./.github/actions/common-rc-validation + with: + RELEASE_VER: ${{env.RELEASE_VER}} + PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}} - name: Installing gcloud-auth-plugin run: sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin @@ -281,6 +288,9 @@ jobs: - name: Setting python env uses: ./.github/actions/common-rc-validation + with: + RELEASE_VER: ${{env.RELEASE_VER}} + PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}} - name: Setup Maven Action uses: s4u/setup-maven-action@v1.2.1 @@ -354,6 +364,9 @@ jobs: - name: Setting python env uses: ./.github/actions/common-rc-validation + with: + RELEASE_VER: ${{env.RELEASE_VER}} + PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}} - name: Exporting leaderboard Dataset Name run: echo "LEADERBOARD_DIRECT_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV @@ -399,6 +412,9 @@ jobs: - name: Setting python env uses: ./.github/actions/common-rc-validation + with: + RELEASE_VER: ${{env.RELEASE_VER}} + PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}} - name: Exporting Dataflow Dataset Name run: echo "LEADERBOARD_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV @@ -448,6 +464,9 @@ jobs: - name: Setting python env uses: ./.github/actions/common-rc-validation + with: + RELEASE_VER: ${{env.RELEASE_VER}} + PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}} - name: Exporting Gamestates Direct Dataset Name run: echo "GAMESTATS_DIRECT_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV @@ -494,6 +513,9 @@ jobs: - name: Setting python env uses: ./.github/actions/common-rc-validation + with: + RELEASE_VER: ${{env.RELEASE_VER}} + PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}} - name: Exporting Gamestates Direct Dataset Name run: echo "GAMESTATS_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV @@ -531,3 +553,4 @@ jobs: steps: - name: Deleting Shared Pub Sub run: gcloud pubsub topics delete --project=${USER_GCP_PROJECT} ${{needs.generate_shared_pubsub.outputs.name}} + \ No newline at end of file From 1d76433c52a743226945696086efa9c366af62a2 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Mon, 17 Oct 2022 17:33:40 -0500 Subject: [PATCH 15/16] Added Verify Working Branch step. --- .github/workflows/run_rc_validation.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index bc5dd44087e8..2407a0406168 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -87,6 +87,9 @@ jobs: run: | git config user.name $GITHUB_ACTOR git config user.email actions@"$RUNNER_NAME".local + - name: Verify working branch name + run: + - sh ./ci_check_git_branch.sh $WORKING_BRANCH - name: Create Pull Request run: | git checkout -b ${{env.WORKING_BRANCH}} ${{ env.RC_TAG }} --quiet From 907b5d7fbdb28a11535bc6330e0a8d6903494ff5 Mon Sep 17 00:00:00 2001 From: Elias Segundo Antonio Date: Wed, 5 Oct 2022 18:23:27 -0500 Subject: [PATCH 16/16] Adding extras jobs to CI.MD (#230) --- CI.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CI.md b/CI.md index 22c6e71a7ca3..cb26171654b2 100644 --- a/CI.md +++ b/CI.md @@ -133,6 +133,18 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog |------------------------------|-----------------------------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| | Verify Release Build | Verifies full life cycle of Gradle Build and all PostCommit/PreCommit tests against Release Branch on CI. | No | No | No | No | +#### Run RC Validation - [run_rc_validation.yml](.github/workflows/run_rc_validation.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|------------------------------|-----------------------------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Python Release Candidate | Comment on PR to trigger Python ReleaseCandidate Jenkins job. | No | No | No | No | +| Python XLang SQL Taxi | Runs Python XLang SQL Taxi with DataflowRunner | No | No | No | Yes | +| Python XLang Kafka | Runs Python XLang Kafka Taxi with DataflowRunner | No | No | No | Yes | +| Direct Runner Leaderboard | Runs Python Leaderboard with DirectRunner | No | No | No | Yes | +| Direct Runner GameStats | Runs Python GameStats with DirectRunner. | No | No | No | Yes | +| Dataflow Runner Leaderboard | Runs Python Leaderboard with DataflowRunner | No | No | No | Yes | +| Dataflow Runner GameStats | Runs Python GameStats with DataflowRunner | No | No | No | Yes | + ### All migrated workflows run based on the following triggers | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Workflow Dispatch |