diff --git a/.github/workflows/job-precommit-typescript.yml b/.github/workflows/job-precommit-typescript.yml new file mode 100644 index 000000000000..578515825b5b --- /dev/null +++ b/.github/workflows/job-precommit-typescript.yml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run Typescript PreCommit. + +name: Typescript + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/python/apache_beam/runners/interactive/extensions/**'] +permissions: read-all + +jobs: + typescript: + name: Typescript + runs-on: [self-hosted, ubuntu-20.04] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run :typescriptPreCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :typescriptPreCommit diff --git a/.github/workflows/job-precommit-website-stage-gcs.yml b/.github/workflows/job-precommit-website-stage-gcs.yml new file mode 100644 index 000000000000..64035a36ea4a --- /dev/null +++ b/.github/workflows/job-precommit-website-stage-gcs.yml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the Website in Stage GCS. + +name: Website Stage GCS + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['website/**'] +permissions: read-all + +jobs: + website-stage-gcs: + name: Run Website Stage in GCS + runs-on: [self-hosted, ubuntu-20.04] + steps: + - name: Checking out composite to setup service account + uses: actions/checkout@v3 + with: + ref: master + - name: Setup service account + uses: ./.github/actions/activate-service-account + with: + GCP_SERVICE_ACCOUNT: ${{secrets.GCP_SELF_HOSTED_SA_KEY}} + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run :website:stageWebsite + uses: ./.github/actions/gradle-command-self-hosted-action + env: # TODO(https://github.com/apache/beam/issues/23580) Rename ghprb env vars to a more GHA concept. + ghprbPullId: ${{github.event.pull_request.number}} + ghprbPullAuthorLogin: ${{github.actor}} + ghprbSourceBranch: ${{github.ref_name}} + ghBranch: ${{github.head_ref}} + repositoryUrl: ${{github.repository}} + with: + gradle-command: :website:stageWebsite diff --git a/.github/workflows/job-precommit-website.yml b/.github/workflows/job-precommit-website.yml new file mode 100644 index 000000000000..bdcfffe98e95 --- /dev/null +++ b/.github/workflows/job-precommit-website.yml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run Website PreCommit. + +name: Website + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['website/**'] +permissions: read-all + +jobs: + website: + name: website + runs-on: [self-hosted, ubuntu-20.04] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + requires-py-37: false + requires-py-38: false + requires-py-39: false + - name: Run :websitePreCommit + uses: ./.github/actions/gradle-command-self-hosted-action + env: # TODO(https://github.com/apache/beam/issues/23580) Rename ghprb env vars to a more GHA concept. + ghprbPullId: ${{github.event.pull_request.number}} + ghprbPullAuthorLogin: ${{github.actor}} + ghprbSourceBranch: ${{github.ref_name}} + ghBranch: ${{github.head_ref}} + repositoryUrl: ${{github.repository}} + with: + gradle-command: :websitePreCommit diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index e53dae86c458..ea0c58b7c54f 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -52,10 +52,7 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PreCommit_Spotless | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Phrase/) | `Run Spotless PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron) | | beam_PreCommit_SQL | [commit](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Phrase/) | `Run SQL PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/) | | beam_PreCommit_SQL_Java11 | [commit](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Phrase/) | `Run SQL_Java11 PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/) | -| beam_PreCommit_Website | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Website_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Website_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Website_Phrase/) | `Run Website PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Website_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Website_Cron) | -| beam_PreCommit_Website_Stage_GCS | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Phrase/) | `Run Website_Stage_GCS PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Cron) | | beam_PreCommit_Whitespace | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Phrase/) | `Run Whitespace PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Cron) | -| beam_PreCommit_Typescript | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Phrase/) | `Run Typescript PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Cron) | ### PostCommit Jobs diff --git a/.test-infra/jenkins/job_PreCommit_Typescript.groovy b/.test-infra/jenkins/job_PreCommit_Typescript.groovy deleted file mode 100644 index 29cb93ede8b6..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Typescript.groovy +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Typescript', - gradleTask: ':typescriptPreCommit', - triggerPathPatterns: [ - '^sdks/python/apache_beam/runners/interactive/extensions/.*$', - ] - ) -builder.build() diff --git a/.test-infra/jenkins/job_PreCommit_Website.groovy b/.test-infra/jenkins/job_PreCommit_Website.groovy deleted file mode 100644 index 73014819ed00..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Website.groovy +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Website', - gradleTask: ':websitePreCommit', - triggerPathPatterns: ['^website/.*$']) -builder.build() - diff --git a/.test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy b/.test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy deleted file mode 100644 index e2f7202d14eb..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Website_Stage_GCS', - gradleTask: ':website:stageWebsite', - triggerPathPatterns: ['^website/.*$']) -builder.build { - publishers { - buildDescription(/Website published to (http:\/\/.+\/index.html)/) - } -} - diff --git a/CI.md b/CI.md index 2234af307ad4..8750f09b5506 100644 --- a/CI.md +++ b/CI.md @@ -166,9 +166,11 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog ### PreCommit Workflows -| Workflow | Description | Requires GCP Credentials | -|----------------------------------------------------------------------------------|-------------------------|---------------------------| -| [job-precommit-placeholder.yml](.github/workflows/job-precommit-placeholder.yml) | Description placeholder | Yes/No | +| Workflow | Description | Requires GCP Credentials | +|----------------------------------------------------------------------------------------------|---------------------------|--------------------------| +| [job-precommit-typescript.yml](.github/workflows/job-precommit-typescript.yml) | Runs Typescript PreCommit | No | +| [job-precommit-website.yml](.github/workflows/job-precommit-website.yml) | Runs Website PreCommit | No | +| [job-precommit-website-stage-gcs.yml](.github/workflows/job-precommit-website-stage-gcs.yml) | Runs Website in Stage GCS | Yes | ### PostCommit Workflows diff --git a/website/Dockerfile b/website/Dockerfile index 4c4847481045..fb9f8f25887a 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -69,4 +69,29 @@ RUN HUGOHOME="$(mktemp -d)" \ && chmod +x /usr/local/bin/hugo \ && rm -r "${HUGOHOME}" + +# Install gcloud +# Make sure that your operating system meets the requirements + +# Add the gcloud CLI distribution URI as a package source +RUN apt-get install apt-transport-https ca-certificates gnupg +RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list +# Import the Google Cloud public key +RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - +# Update and install the gcloud CLI +RUN apt-get update && apt-get install google-cloud-cli +# #Installing and replacing default python version +RUN apt install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev \ + && curl -O https://www.python.org/ftp/python/3.8.2/Python-3.8.2.tar.xz \ + && tar -xf Python-3.8.2.tar.xz \ + && cd Python-3.8.2 && ./configure --enable-optimizations \ + && make -j 4 && make altinstall + +RUN update-alternatives --install /usr/bin/python python /usr/local/bin/python3.8 1\ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.5 2 \ + && update-alternatives --set python $(update-alternatives --list python | grep python3.8) + +# Add dedicated user to have permissions on opt dir +RUN useradd -m actions && usermod -aG sudo actions && echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +RUN chown -R actions:actions /opt/ WORKDIR /opt/ diff --git a/website/build.gradle b/website/build.gradle index 6aa88a2c7feb..8eab3a05d031 100644 --- a/website/build.gradle +++ b/website/build.gradle @@ -80,10 +80,10 @@ task createDockerContainer(type: Exec) { } else { // Otherwise: run as current user, so as to not require sudo to clean up // build/ directory. - extraOptions += " -u \$(id -u):\$(id -g)" + extraOptions += " -u actions" } commandLine '/bin/bash', '-c', - "docker create -v $project.rootDir:$dockerWorkDir $extraOptions $dockerImageTag sh -c 'trap \"exit 0\" INT; while true; do sleep 30; done;'" + "docker create $extraOptions $dockerImageTag sh -c 'trap \"exit 0\" INT; while true; do sleep 30; done;'" } } @@ -115,9 +115,39 @@ task buildCodeSamples(type: Exec) { "${->startDockerContainer.containerId()}", 'yarn', 'build_code_samples' } +task cloneRepository(type: Exec) { + def branch = System.getenv('ghBranch') ? System.getenv('ghBranch'): + grgit ? grgit.branch.current().getName(): 'master' + def repositoryUrl = System.getenv('repositoryUrl') ? "https://github.com/${System.getenv('repositoryUrl')}" : + "https://github.com/apache/beam" + + commandLine 'docker', 'exec', + "${->startDockerContainer.containerId()}", "git", "clone", "-b", branch, repositoryUrl, "." +} + +task setupVirtualenvWithDependenciesInDocker { + doLast { + exec { + commandLine 'docker', 'exec', + "${->startDockerContainer.containerId()}", "python${project.ext.pythonVersion}", + '-m', 'venv', "${envdir}" + } + exec { + commandLine 'docker', 'exec', + "${->startDockerContainer.containerId()}", '/bin/bash', '-c', + """. ${envdir}/bin/activate && pip install beautifulsoup4""" + } + } +} + +initGitSubmodules.mustRunAfter cloneRepository +installDependencies.mustRunAfter cloneRepository +buildCodeSamples.mustRunAfter cloneRepository + task setupDockerContainer(type: Exec) { dependsOn startDockerContainer - finalizedBy initGitSubmodules, installDependencies, buildCodeSamples + finalizedBy cloneRepository, initGitSubmodules, installDependencies, buildCodeSamples, + setupVirtualenvWithDependenciesInDocker ext.containerId = { return startDockerContainer.containerId() } @@ -147,7 +177,6 @@ def createBuildTask = { BuildTaskConfiguration config = it as BuildTaskConfiguration task "build${config.name}Website" (type:Exec) { dependsOn setupDockerContainer - finalizedBy stopAndRemoveDockerContainer def configs = "$dockerSourceDir/site/config.toml" def baseUrlFlag = config.baseUrl ? "--baseURL /${config.baseUrl}" : "" @@ -230,13 +259,12 @@ def getBranchRepo() { return "apache/beam/blob/master" } -def buildContentDir(name) { - "${project.rootDir}/build/website/generated-${name.toLowerCase()}-content" +def buildContentDir(name, dir = project.rootDir) { + "${dir}/build/website/generated-${name.toLowerCase()}-content" } task serveWebsite(type: Exec) { dependsOn setupDockerContainer - finalizedBy stopAndRemoveDockerContainer commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', """cd $dockerSourceDir && \ @@ -263,6 +291,7 @@ task preCommit { // Creates a new commit on asf-site branch task commitWebsite { + finalizedBy stopAndRemoveDockerContainer doLast { assert grgit : "Cannot commit website outside of git repository" assert file("${buildContentDir('apache')}/index.html").exists() @@ -346,6 +375,7 @@ publishWebsite.dependsOn commitWebsite * ./gradlew :website:stageWebsite -PwebsiteBucket=foo */ task stageWebsite { + finalizedBy stopAndRemoveDockerContainer doLast { def baseUrl = getBaseUrl() assert baseUrl : 'Website staging requires a valid baseUrl' @@ -353,10 +383,18 @@ task stageWebsite { def gcs_path = "gs://${gcs_bucket}/${baseUrl}" // Fixup the links to index.html files - shell ". ${envdir}/bin/activate && python append_index_html_to_internal_links.py ${buildContentDir('gcs')}" + exec { + commandLine 'docker', 'exec', '--workdir', "$dockerWorkDir/website", + "${->startDockerContainer.containerId()}", '/bin/bash', '-c', + """. ${envdir}/bin/activate && python append_index_html_to_internal_links.py ${buildContentDir('gcs', ".")} """ + } // Copy the build website to GCS - shell "gsutil -m rsync -r -d ${buildContentDir('gcs')} ${gcs_path}" + exec { + commandLine 'docker', 'exec', '--workdir', "$dockerWorkDir", + "${->startDockerContainer.containerId()}", '/bin/bash', '-c', + """gsutil -m rsync -r -d ${buildContentDir('gcs', ".")} ${gcs_path}""" + } println "Website published to http://${gcs_bucket}." + "storage.googleapis.com/${baseUrl}/index.html"