diff --git a/.github/actions/maven-settings/action.yaml b/.github/actions/maven-settings/action.yaml new file mode 100644 index 00000000000..4ac50984226 --- /dev/null +++ b/.github/actions/maven-settings/action.yaml @@ -0,0 +1,13 @@ +# Installs .github/maven/settings.xml as ~/.m2/settings.xml (Central mirror on Artifactory, global-artifacts for the Talend artifacts). +# Run it after vault-secrets (which exports the credentials) and without `server-id` on actions/setup-java, +# which would otherwise generate its own settings.xml. +name: Maven settings +description: Install the Maven settings of the Syncope fork builds + +runs: + using: composite + steps: + - shell: bash + run: | + mkdir -p ~/.m2 + cp "$GITHUB_WORKSPACE/.github/maven/settings.xml" ~/.m2/settings.xml diff --git a/.github/actions/select-gavs/action.yml b/.github/actions/select-gavs/action.yml new file mode 100644 index 00000000000..6643f730966 --- /dev/null +++ b/.github/actions/select-gavs/action.yml @@ -0,0 +1,75 @@ +# Compares the GAVs of the modules in scope with what is already published in global-artifacts. +# Release versions are immutable there: a GAV that already exists is skipped by the deploy job +# (re-deploying it fails with 409), so a changed module that keeps its old version would never ship. +# Used by the build job (before the install: only the new GAVs are built, also on pull requests) and by the deploy job (selects what to deploy). +name: Select GAVs to deploy +description: Lists the reactor GAVs in scope and splits them into already published / to deploy. +inputs: + modules: + description: "-pl value, or ALL for the whole reactor" + required: true + url: + description: Maven repository URL + required: true + username: + required: true + token: + description: "Token with read:packages (empty = check skipped, e.g. pull requests from forks)" + required: false + default: '' + maven_args: + description: "Extra Maven arguments, word-split (profiles such as -Ppatch, -Drevision=...): the GAVs depend on them" + required: false + default: '' +outputs: + checked: + description: "true when the check ran; false when it was skipped (no token) or the GAVs could not be listed: then `modules` is empty and means nothing" + value: ${{ steps.select.outputs.checked }} + modules: + description: Comma-separated groupId:artifactId of the modules not yet published (for -pl) + value: ${{ steps.select.outputs.modules }} +runs: + using: composite + steps: + - id: select + shell: bash + env: + MVN_MODULES: ${{ inputs.modules }} + URL: ${{ inputs.url }} + GH_USER: ${{ inputs.username }} + GH_TOKEN: ${{ inputs.token }} + MVN_ARGS: ${{ inputs.maven_args }} + run: | + if [ -z "$GH_TOKEN" ]; then + echo "::notice::no token available, published-GAV check skipped" + echo "checked=false" >> "$GITHUB_OUTPUT"; echo "modules=" >> "$GITHUB_OUTPUT"; exit 0 + fi + W="$RUNNER_TEMP/select-gavs"; mkdir -p "$W"; cd "$W"; : > todo.txt; : > todo-gav.txt; : > skipped.txt + cd "$GITHUB_WORKSPACE" + PL=(); [ "$MVN_MODULES" = "ALL" ] || PL=(-pl "$MVN_MODULES") + mvn -q -B $MVN_ARGS "${PL[@]}" org.codehaus.mojo:exec-maven-plugin:3.5.0:exec \ + -Dexec.executable=echo -Dexec.args='GAV ${project.groupId}:${project.artifactId}:${project.version}' \ + | sed -n 's/^GAV //p' > "$W/gavs.txt" || true + if [ ! -s "$W/gavs.txt" ]; then + echo "::warning::could not list the GAVs of MVN_MODULES=$MVN_MODULES, published-GAV check skipped" + echo "checked=false" >> "$GITHUB_OUTPUT"; echo "modules=" >> "$GITHUB_OUTPUT"; exit 0 + fi + while IFS=: read -r g a v; do + url="$URL/${g//.//}/$a/$v/$a-$v.pom" + code=$(curl -s -L -o /dev/null -w '%{http_code}' -u "$GH_USER:$GH_TOKEN" "$url") + case "$code" in + 200) echo "$g:$a:$v" >> "$W/skipped.txt" ;; + 404) echo "$g:$a" >> "$W/todo.txt"; echo "$g:$a:$v" >> "$W/todo-gav.txt" ;; + *) echo "::error::unexpected HTTP $code checking $url"; exit 1 ;; + esac + done < "$W/gavs.txt" + echo "checked=true" >> "$GITHUB_OUTPUT" + echo "modules=$(paste -sd, "$W/todo.txt")" >> "$GITHUB_OUTPUT" + { + echo "### global-artifacts: GAVs of the modules in scope" + echo "**New, will be deployed ($(wc -l < "$W/todo.txt" | tr -d ' '))**"; echo '```'; cat "$W/todo-gav.txt"; echo '```' + echo "**Already published, will be skipped ($(wc -l < "$W/skipped.txt" | tr -d ' '))**"; echo '```'; cat "$W/skipped.txt"; echo '```' + } >> "$GITHUB_STEP_SUMMARY" + if [ ! -s "$W/todo.txt" ]; then + echo "::warning::every module in scope is already published in global-artifacts: nothing would be deployed. Bump the version (fork dated version) if the content changed." + fi diff --git a/.github/actions/vault-secrets/action.yaml b/.github/actions/vault-secrets/action.yaml new file mode 100644 index 00000000000..c6625cbb864 --- /dev/null +++ b/.github/actions/vault-secrets/action.yaml @@ -0,0 +1,44 @@ +# Imports the CI secrets from Vault (JWT auth of the GitHub OIDC token: the job needs `permissions: id-token: write`). +# Copied from Talend/tesb-rt-ee (.github/actions/vault-secrets). +# CI_BUILD_DPE_PAT (read/write:packages on Talend/global-artifacts) and the Artifactory credentials (Maven Central mirror) +# are exported as GLOBAL_ARTIFACTS_USER / GLOBAL_ARTIFACTS_TOKEN / ARTIFACTORY_USER / ARTIFACTORY_PASSWORD for the following steps +# (see .github/maven/settings.xml). +# TODO(DPE-4514): confirm the role, the paths and the key names below with the Vault admins. +name: Vault Secrets +description: Import the Vault secrets of the Syncope fork builds + +inputs: + role: + description: Vault JWT role granted to this repository + required: false + default: github-actions-runner-dp-engine + +runs: + using: composite + steps: + - name: Import secrets + id: vault + uses: hashicorp/vault-action@v3 + with: + url: "https://vault-vaas.rnd-cd.cloud.talend.com" + role: ${{ inputs.role }} + method: jwt + path: jwt-github + exportEnv: false + secrets: | + secret/data/dp-engine/ci github-ci-build-dpe-token | github-ci-build-dpe-token; + secret/data/dp-engine/ci artifactory-credentials-login | artifactory-credentials-login; + secret/data/dp-engine/ci artifactory-credentials-password | artifactory-credentials-password + - name: Export the Maven credentials + shell: bash + env: + CI_BUILD_DPE_PAT: ${{ steps.vault.outputs.github-ci-build-dpe-token }} + ARTIFACTORY_LOGIN: ${{ steps.vault.outputs.artifactory-credentials-login }} + ARTIFACTORY_PWD: ${{ steps.vault.outputs.artifactory-credentials-password }} + run: | + { + echo "GLOBAL_ARTIFACTS_USER=$GITHUB_ACTOR" + echo "GLOBAL_ARTIFACTS_TOKEN=$CI_BUILD_DPE_PAT" + echo "ARTIFACTORY_USER=$ARTIFACTORY_LOGIN" + echo "ARTIFACTORY_PASSWORD=$ARTIFACTORY_PWD" + } >> "$GITHUB_ENV" diff --git a/.github/maven/settings.xml b/.github/maven/settings.xml new file mode 100644 index 00000000000..f779d0402d2 --- /dev/null +++ b/.github/maven/settings.xml @@ -0,0 +1,49 @@ + + + + + + artifactory-maven-central-remote + ${env.ARTIFACTORY_USER} + ${env.ARTIFACTORY_PASSWORD} + + + + global-artifacts + ${env.GLOBAL_ARTIFACTS_USER} + ${env.GLOBAL_ARTIFACTS_TOKEN} + + + + + + artifactory-maven-central-remote + central + https://artifactory.datapwn.com/artifactory/maven-central-remote + + + + + + + global-artifacts + + + global-artifacts + https://maven.pkg.github.com/Talend/global-artifacts + true + true + + + + + + global-artifacts + + diff --git a/.github/workflows/global-artifacts.yml b/.github/workflows/global-artifacts.yml new file mode 100644 index 00000000000..59a06e68ac9 --- /dev/null +++ b/.github/workflows/global-artifacts.yml @@ -0,0 +1,176 @@ +# Publishes the Talend fork modules (former Jenkinsfile.talend, shared-lib step `Fork`) to GitHub Packages (Talend/global-artifacts). +# Build -> Deploy (manual workflow_dispatch with `deploy` ticked, approval on the `global-artifacts-deploy` environment). +# Runs on the internal runners (the repository is private). Credentials come from Vault, see .github/actions/vault-secrets. +# MVN_MODULES (-pl value, upstream modules included through -am): input `mvn_modules` > repository variable MVN_MODULES > default below. +# Deploy only publishes the GAVs missing from global-artifacts (release versions are immutable: re-deploying one fails with 409), +# so bump the version in the poms when the content of a module changes. +name: Publish to global-artifacts + +on: + push: + branches: ['tesb/**'] + workflow_dispatch: + inputs: + mvn_modules: + description: 'Modules to build, -pl value (default: MVN_MODULES below)' + required: false + type: string + deploy: + description: 'Deploy to global-artifacts after the build (needs approval on the environment)' + required: false + default: false + type: boolean + deploy_goal: + description: 'Maven deploy goal' + required: false + default: deploy + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + packages: read + id-token: write + +env: + MVN_MODULES: ${{ vars.MVN_MODULES || ':syncope-common-idrepo,:syncope-client-idrepo-lib' }} + GITHUB_ACTOR: ci-build-dpe + # GLOBAL_ARTIFACTS_USER / GLOBAL_ARTIFACTS_TOKEN / ARTIFACTORY_* are exported by the vault-secrets action + GLOBAL_ARTIFACTS_URL: https://maven.pkg.github.com/Talend/global-artifacts + # ${GLOBAL_ARTIFACTS_URL} is expanded by the `Maven options` step (the env context is not available in a workflow-level env) + MAVEN_CUSTOM_OPTIONS: >- + -U -B + -DaltReleaseDeploymentRepository=global-artifacts::default::${GLOBAL_ARTIFACTS_URL} + -DaltSnapshotDeploymentRepository=global-artifacts::default::${GLOBAL_ARTIFACTS_URL} + -Drat.skip=true + +jobs: + build: + name: Build + runs-on: + group: organization/default + labels: runner-medium + timeout-minutes: 120 + outputs: + modules: ${{ steps.modules.outputs.modules }} + steps: + - uses: actions/checkout@v7 + + - name: Vault secrets + uses: ./.github/actions/vault-secrets + + - uses: actions/setup-java@v6 + with: + distribution: temurin + java-version: '17' + # after setup-java, which would otherwise leave its own settings.xml + - name: Maven settings + uses: ./.github/actions/maven-settings + + - name: Maven dependencies cache + uses: actions/cache@v6 + with: + path: ~/.m2/repository + key: m2-${{ hashFiles('**/pom.xml') }}-build + restore-keys: | + m2-${{ hashFiles('**/pom.xml') }} + m2 + - name: Maven options + run: echo "MAVEN_CUSTOM_OPTIONS=${MAVEN_CUSTOM_OPTIONS//'${GLOBAL_ARTIFACTS_URL}'/$GLOBAL_ARTIFACTS_URL}" >> "$GITHUB_ENV" + + - name: Resolve MVN_MODULES + id: modules + env: + INPUT_MODULES: ${{ inputs.mvn_modules }} + run: | + echo "modules=${INPUT_MODULES:-$MVN_MODULES}" >> "$GITHUB_OUTPUT" + echo "::notice title=Modules to build::${INPUT_MODULES:-$MVN_MODULES}" + + # lists the GAVs of the modules + their upstream modules (-am) and keeps the ones missing from global-artifacts + - name: Check GAVs against global-artifacts + id: check + uses: ./.github/actions/select-gavs + with: + modules: ${{ steps.modules.outputs.modules }} + maven_args: -am + url: ${{ env.GLOBAL_ARTIFACTS_URL }} + username: ${{ env.GLOBAL_ARTIFACTS_USER }} + token: ${{ env.GLOBAL_ARTIFACTS_TOKEN }} + + - name: mvn install + env: + CHECKED: ${{ steps.check.outputs.checked }} + NEW_MODULES: ${{ steps.check.outputs.modules }} + MVN_MODULES_IN: ${{ steps.modules.outputs.modules }} + run: | + mvn --version + if [ "$CHECKED" = "true" ]; then + # GAVs already published are immutable and never redeployed: build only the new ones + if [ -z "$NEW_MODULES" ]; then echo "::notice::every module in scope is already published, nothing to build"; exit 0; fi + mvn install $MAVEN_CUSTOM_OPTIONS -pl "$NEW_MODULES" -DskipTests + else + mvn install $MAVEN_CUSTOM_OPTIONS -pl "$MVN_MODULES_IN" -am -DskipTests + fi + + deploy: + name: Deploy + needs: build + # Jenkins: `input 'Do you want to proceed to the Deployment?'`. Here: required reviewers on the + # `global-artifacts-deploy` environment (Settings > Environments). + if: github.event_name == 'workflow_dispatch' && inputs.deploy + runs-on: + group: organization/default + labels: runner-small + timeout-minutes: 120 + environment: global-artifacts-deploy + permissions: + contents: read + packages: write + id-token: write + steps: + - uses: actions/checkout@v7 + + - name: Vault secrets + uses: ./.github/actions/vault-secrets + + - uses: actions/setup-java@v6 + with: + distribution: temurin + java-version: '17' + - name: Maven settings + uses: ./.github/actions/maven-settings + + - name: Maven dependencies cache + uses: actions/cache@v6 + with: + path: ~/.m2/repository + key: m2-${{ hashFiles('**/pom.xml') }}-build + restore-keys: | + m2-${{ hashFiles('**/pom.xml') }} + m2 + - name: Maven options + run: echo "MAVEN_CUSTOM_OPTIONS=${MAVEN_CUSTOM_OPTIONS//'${GLOBAL_ARTIFACTS_URL}'/$GLOBAL_ARTIFACTS_URL}" >> "$GITHUB_ENV" + + - name: Select GAVs not yet published + id: select + uses: ./.github/actions/select-gavs + with: + modules: ${{ needs.build.outputs.modules }} + maven_args: -am + url: ${{ env.GLOBAL_ARTIFACTS_URL }} + username: ${{ env.GLOBAL_ARTIFACTS_USER }} + token: ${{ env.GLOBAL_ARTIFACTS_TOKEN }} + + - name: Fail when the GAV check could not run + if: steps.select.outputs.checked != 'true' + run: echo "::error::GAV check did not run, refusing to deploy"; exit 1 + + - name: mvn deploy + if: steps.select.outputs.modules != '' + env: + MVN_DEPLOY_MODULES: ${{ steps.select.outputs.modules }} + MVN_DEPLOY_GOAL: ${{ inputs.deploy_goal }} + run: mvn clean "$MVN_DEPLOY_GOAL" $MAVEN_CUSTOM_OPTIONS -pl "$MVN_DEPLOY_MODULES" -DskipTests diff --git a/Jenkinsfile.talend b/Jenkinsfile.talend deleted file mode 100644 index 8263c1cc2eb..00000000000 --- a/Jenkinsfile.talend +++ /dev/null @@ -1,5 +0,0 @@ -@Library('esb') _ - -Fork( - MVN_MODULES: ':syncope-common-idrepo,:syncope-client-idrepo-lib -am' -) \ No newline at end of file