Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .github/actions/maven-settings/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Installs .github/maven/settings.xml as ~/.m2/settings.xml (Central mirror on Artifactory, global-artifacts for the Talend artifacts).
# Run it after vault-secrets (which exports the credentials) and without `server-id` on actions/setup-java,
# which would otherwise generate its own settings.xml.
name: Maven settings
description: Install the Maven settings of the Syncope fork builds

runs:
using: composite
steps:
- shell: bash
run: |
mkdir -p ~/.m2
cp "$GITHUB_WORKSPACE/.github/maven/settings.xml" ~/.m2/settings.xml
75 changes: 75 additions & 0 deletions .github/actions/select-gavs/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# Compares the GAVs of the modules in scope with what is already published in global-artifacts.
# Release versions are immutable there: a GAV that already exists is skipped by the deploy job
# (re-deploying it fails with 409), so a changed module that keeps its old version would never ship.
# Used by the build job (before the install: only the new GAVs are built, also on pull requests) and by the deploy job (selects what to deploy).
name: Select GAVs to deploy
description: Lists the reactor GAVs in scope and splits them into already published / to deploy.
inputs:
modules:
description: "-pl value, or ALL for the whole reactor"
required: true
url:
description: Maven repository URL
required: true
username:
required: true
token:
description: "Token with read:packages (empty = check skipped, e.g. pull requests from forks)"
required: false
default: ''
maven_args:
description: "Extra Maven arguments, word-split (profiles such as -Ppatch, -Drevision=...): the GAVs depend on them"
required: false
default: ''
outputs:
checked:
description: "true when the check ran; false when it was skipped (no token) or the GAVs could not be listed: then `modules` is empty and means nothing"
value: ${{ steps.select.outputs.checked }}
modules:
description: Comma-separated groupId:artifactId of the modules not yet published (for -pl)
value: ${{ steps.select.outputs.modules }}
runs:
using: composite
steps:
- id: select
shell: bash
env:
MVN_MODULES: ${{ inputs.modules }}
URL: ${{ inputs.url }}
GH_USER: ${{ inputs.username }}
GH_TOKEN: ${{ inputs.token }}
MVN_ARGS: ${{ inputs.maven_args }}
run: |
if [ -z "$GH_TOKEN" ]; then
echo "::notice::no token available, published-GAV check skipped"
echo "checked=false" >> "$GITHUB_OUTPUT"; echo "modules=" >> "$GITHUB_OUTPUT"; exit 0
fi
W="$RUNNER_TEMP/select-gavs"; mkdir -p "$W"; cd "$W"; : > todo.txt; : > todo-gav.txt; : > skipped.txt
cd "$GITHUB_WORKSPACE"
PL=(); [ "$MVN_MODULES" = "ALL" ] || PL=(-pl "$MVN_MODULES")
mvn -q -B $MVN_ARGS "${PL[@]}" org.codehaus.mojo:exec-maven-plugin:3.5.0:exec \
-Dexec.executable=echo -Dexec.args='GAV ${project.groupId}:${project.artifactId}:${project.version}' \
| sed -n 's/^GAV //p' > "$W/gavs.txt" || true
if [ ! -s "$W/gavs.txt" ]; then
echo "::warning::could not list the GAVs of MVN_MODULES=$MVN_MODULES, published-GAV check skipped"
echo "checked=false" >> "$GITHUB_OUTPUT"; echo "modules=" >> "$GITHUB_OUTPUT"; exit 0
fi
while IFS=: read -r g a v; do
url="$URL/${g//.//}/$a/$v/$a-$v.pom"
code=$(curl -s -L -o /dev/null -w '%{http_code}' -u "$GH_USER:$GH_TOKEN" "$url")
case "$code" in
200) echo "$g:$a:$v" >> "$W/skipped.txt" ;;
404) echo "$g:$a" >> "$W/todo.txt"; echo "$g:$a:$v" >> "$W/todo-gav.txt" ;;
*) echo "::error::unexpected HTTP $code checking $url"; exit 1 ;;
esac
done < "$W/gavs.txt"
echo "checked=true" >> "$GITHUB_OUTPUT"
echo "modules=$(paste -sd, "$W/todo.txt")" >> "$GITHUB_OUTPUT"
{
echo "### global-artifacts: GAVs of the modules in scope"
echo "**New, will be deployed ($(wc -l < "$W/todo.txt" | tr -d ' '))**"; echo '```'; cat "$W/todo-gav.txt"; echo '```'
echo "**Already published, will be skipped ($(wc -l < "$W/skipped.txt" | tr -d ' '))**"; echo '```'; cat "$W/skipped.txt"; echo '```'
} >> "$GITHUB_STEP_SUMMARY"
if [ ! -s "$W/todo.txt" ]; then
echo "::warning::every module in scope is already published in global-artifacts: nothing would be deployed. Bump the version (fork dated version) if the content changed."
fi
44 changes: 44 additions & 0 deletions .github/actions/vault-secrets/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# Imports the CI secrets from Vault (JWT auth of the GitHub OIDC token: the job needs `permissions: id-token: write`).
# Copied from Talend/tesb-rt-ee (.github/actions/vault-secrets).
# CI_BUILD_DPE_PAT (read/write:packages on Talend/global-artifacts) and the Artifactory credentials (Maven Central mirror)
# are exported as GLOBAL_ARTIFACTS_USER / GLOBAL_ARTIFACTS_TOKEN / ARTIFACTORY_USER / ARTIFACTORY_PASSWORD for the following steps
# (see .github/maven/settings.xml).
# TODO(DPE-4514): confirm the role, the paths and the key names below with the Vault admins.
name: Vault Secrets
description: Import the Vault secrets of the Syncope fork builds

inputs:
role:
description: Vault JWT role granted to this repository
required: false
default: github-actions-runner-dp-engine

runs:
using: composite
steps:
- name: Import secrets
id: vault
uses: hashicorp/vault-action@v3
with:
url: "https://vault-vaas.rnd-cd.cloud.talend.com"
role: ${{ inputs.role }}
method: jwt
path: jwt-github
exportEnv: false
secrets: |
secret/data/dp-engine/ci github-ci-build-dpe-token | github-ci-build-dpe-token;
secret/data/dp-engine/ci artifactory-credentials-login | artifactory-credentials-login;
secret/data/dp-engine/ci artifactory-credentials-password | artifactory-credentials-password
- name: Export the Maven credentials
shell: bash
env:
CI_BUILD_DPE_PAT: ${{ steps.vault.outputs.github-ci-build-dpe-token }}
ARTIFACTORY_LOGIN: ${{ steps.vault.outputs.artifactory-credentials-login }}
ARTIFACTORY_PWD: ${{ steps.vault.outputs.artifactory-credentials-password }}
run: |
{
echo "GLOBAL_ARTIFACTS_USER=$GITHUB_ACTOR"
echo "GLOBAL_ARTIFACTS_TOKEN=$CI_BUILD_DPE_PAT"
echo "ARTIFACTORY_USER=$ARTIFACTORY_LOGIN"
echo "ARTIFACTORY_PASSWORD=$ARTIFACTORY_PWD"
} >> "$GITHUB_ENV"
49 changes: 49 additions & 0 deletions .github/maven/settings.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Maven settings of the GitHub Actions builds, installed as ~/.m2/settings.xml by .github/actions/maven-settings.
The credentials are environment variables (Maven resolves ${env.X} itself), nothing secret is stored here:
GLOBAL_ARTIFACTS_USER / GLOBAL_ARTIFACTS_TOKEN Talend/global-artifacts (GitHub Packages)
ARTIFACTORY_USER / ARTIFACTORY_PASSWORD artifactory.datapwn.com, the mirror of Maven Central
Both are set by .github/actions/vault-secrets. -->
<settings xmlns="http://maven.apache.org/SETTINGS/1.2.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.2.0 https://maven.apache.org/xsd/settings-1.2.0.xsd">
<servers>
<server>
<id>artifactory-maven-central-remote</id>
<username>${env.ARTIFACTORY_USER}</username>
<password>${env.ARTIFACTORY_PASSWORD}</password>
</server>
<!-- same id as the repository below and in -DaltDeploymentRepository -->
<server>
<id>global-artifacts</id>
<username>${env.GLOBAL_ARTIFACTS_USER}</username>
<password>${env.GLOBAL_ARTIFACTS_TOKEN}</password>
</server>
</servers>

<mirrors>
<mirror>
<id>artifactory-maven-central-remote</id>
<mirrorOf>central</mirrorOf>
<url>https://artifactory.datapwn.com/artifactory/maven-central-remote</url>
</mirror>
</mirrors>

<!-- Already published Syncope GAVs (the modules not rebuilt are resolved from there) -->
<profiles>
<profile>
<id>global-artifacts</id>
<repositories>
<repository>
<id>global-artifacts</id>
<url>https://maven.pkg.github.com/Talend/global-artifacts</url>
<releases><enabled>true</enabled></releases>
<snapshots><enabled>true</enabled></snapshots>
</repository>
</repositories>
</profile>
</profiles>
<activeProfiles>
<activeProfile>global-artifacts</activeProfile>
</activeProfiles>
</settings>
176 changes: 176 additions & 0 deletions .github/workflows/global-artifacts.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
# Publishes the Talend fork modules (former Jenkinsfile.talend, shared-lib step `Fork`) to GitHub Packages (Talend/global-artifacts).
# Build -> Deploy (manual workflow_dispatch with `deploy` ticked, approval on the `global-artifacts-deploy` environment).
# Runs on the internal runners (the repository is private). Credentials come from Vault, see .github/actions/vault-secrets.
# MVN_MODULES (-pl value, upstream modules included through -am): input `mvn_modules` > repository variable MVN_MODULES > default below.
# Deploy only publishes the GAVs missing from global-artifacts (release versions are immutable: re-deploying one fails with 409),
# so bump the version in the poms when the content of a module changes.
name: Publish to global-artifacts

on:
push:
branches: ['tesb/**']
workflow_dispatch:
inputs:
mvn_modules:
description: 'Modules to build, -pl value (default: MVN_MODULES below)'
required: false
type: string
deploy:
description: 'Deploy to global-artifacts after the build (needs approval on the environment)'
required: false
default: false
type: boolean
deploy_goal:
description: 'Maven deploy goal'
required: false
default: deploy
type: string

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

permissions:
contents: read
packages: read
id-token: write

env:
MVN_MODULES: ${{ vars.MVN_MODULES || ':syncope-common-idrepo,:syncope-client-idrepo-lib' }}
GITHUB_ACTOR: ci-build-dpe
# GLOBAL_ARTIFACTS_USER / GLOBAL_ARTIFACTS_TOKEN / ARTIFACTORY_* are exported by the vault-secrets action
GLOBAL_ARTIFACTS_URL: https://maven.pkg.github.com/Talend/global-artifacts
# ${GLOBAL_ARTIFACTS_URL} is expanded by the `Maven options` step (the env context is not available in a workflow-level env)
MAVEN_CUSTOM_OPTIONS: >-
-U -B
-DaltReleaseDeploymentRepository=global-artifacts::default::${GLOBAL_ARTIFACTS_URL}
-DaltSnapshotDeploymentRepository=global-artifacts::default::${GLOBAL_ARTIFACTS_URL}
-Drat.skip=true

jobs:
build:
name: Build
runs-on:
group: organization/default
labels: runner-medium
timeout-minutes: 120
outputs:
modules: ${{ steps.modules.outputs.modules }}
steps:
- uses: actions/checkout@v7

- name: Vault secrets
uses: ./.github/actions/vault-secrets

- uses: actions/setup-java@v6
with:
distribution: temurin
java-version: '17'
# after setup-java, which would otherwise leave its own settings.xml
- name: Maven settings
uses: ./.github/actions/maven-settings

- name: Maven dependencies cache
uses: actions/cache@v6
with:
path: ~/.m2/repository
key: m2-${{ hashFiles('**/pom.xml') }}-build
restore-keys: |
m2-${{ hashFiles('**/pom.xml') }}
m2
- name: Maven options
run: echo "MAVEN_CUSTOM_OPTIONS=${MAVEN_CUSTOM_OPTIONS//'${GLOBAL_ARTIFACTS_URL}'/$GLOBAL_ARTIFACTS_URL}" >> "$GITHUB_ENV"

- name: Resolve MVN_MODULES
id: modules
env:
INPUT_MODULES: ${{ inputs.mvn_modules }}
run: |
echo "modules=${INPUT_MODULES:-$MVN_MODULES}" >> "$GITHUB_OUTPUT"
echo "::notice title=Modules to build::${INPUT_MODULES:-$MVN_MODULES}"

# lists the GAVs of the modules + their upstream modules (-am) and keeps the ones missing from global-artifacts
- name: Check GAVs against global-artifacts
id: check
uses: ./.github/actions/select-gavs
with:
modules: ${{ steps.modules.outputs.modules }}
maven_args: -am
url: ${{ env.GLOBAL_ARTIFACTS_URL }}
username: ${{ env.GLOBAL_ARTIFACTS_USER }}
token: ${{ env.GLOBAL_ARTIFACTS_TOKEN }}

- name: mvn install
env:
CHECKED: ${{ steps.check.outputs.checked }}
NEW_MODULES: ${{ steps.check.outputs.modules }}
MVN_MODULES_IN: ${{ steps.modules.outputs.modules }}
run: |
mvn --version
if [ "$CHECKED" = "true" ]; then
# GAVs already published are immutable and never redeployed: build only the new ones
if [ -z "$NEW_MODULES" ]; then echo "::notice::every module in scope is already published, nothing to build"; exit 0; fi
mvn install $MAVEN_CUSTOM_OPTIONS -pl "$NEW_MODULES" -DskipTests
else
mvn install $MAVEN_CUSTOM_OPTIONS -pl "$MVN_MODULES_IN" -am -DskipTests
fi

deploy:
name: Deploy
needs: build
# Jenkins: `input 'Do you want to proceed to the Deployment?'`. Here: required reviewers on the
# `global-artifacts-deploy` environment (Settings > Environments).
if: github.event_name == 'workflow_dispatch' && inputs.deploy
runs-on:
group: organization/default
labels: runner-small
timeout-minutes: 120
environment: global-artifacts-deploy
permissions:
contents: read
packages: write
id-token: write
steps:
- uses: actions/checkout@v7

- name: Vault secrets
uses: ./.github/actions/vault-secrets

- uses: actions/setup-java@v6
with:
distribution: temurin
java-version: '17'
- name: Maven settings
uses: ./.github/actions/maven-settings

- name: Maven dependencies cache
uses: actions/cache@v6
with:
path: ~/.m2/repository
key: m2-${{ hashFiles('**/pom.xml') }}-build
restore-keys: |
m2-${{ hashFiles('**/pom.xml') }}
m2
- name: Maven options
run: echo "MAVEN_CUSTOM_OPTIONS=${MAVEN_CUSTOM_OPTIONS//'${GLOBAL_ARTIFACTS_URL}'/$GLOBAL_ARTIFACTS_URL}" >> "$GITHUB_ENV"

- name: Select GAVs not yet published
id: select
uses: ./.github/actions/select-gavs
with:
modules: ${{ needs.build.outputs.modules }}
maven_args: -am
url: ${{ env.GLOBAL_ARTIFACTS_URL }}
username: ${{ env.GLOBAL_ARTIFACTS_USER }}
token: ${{ env.GLOBAL_ARTIFACTS_TOKEN }}

- name: Fail when the GAV check could not run
if: steps.select.outputs.checked != 'true'
run: echo "::error::GAV check did not run, refusing to deploy"; exit 1

- name: mvn deploy
if: steps.select.outputs.modules != ''
env:
MVN_DEPLOY_MODULES: ${{ steps.select.outputs.modules }}
MVN_DEPLOY_GOAL: ${{ inputs.deploy_goal }}
run: mvn clean "$MVN_DEPLOY_GOAL" $MAVEN_CUSTOM_OPTIONS -pl "$MVN_DEPLOY_MODULES" -DskipTests
5 changes: 0 additions & 5 deletions Jenkinsfile.talend

This file was deleted.