diff --git a/.github/actions/README.md b/.github/actions/README.md index 5c4d833..1d50fe5 100644 --- a/.github/actions/README.md +++ b/.github/actions/README.md @@ -42,6 +42,23 @@ Every `pluginval validate` option can also be set as an environment variable on the step (e.g. `env: { TIMEOUT_MS: 60000 }`), which is the way to pass values containing spaces. +To skip tests, set `DISABLED_TESTS` to their names, comma-separated (or to an +absolute path to a file listing one name per line). Test names contain spaces, +so they can't be given as `--disabled-tests` in `args`. For example, at strictness +8 and above "Discrete parameter state restoration" fails for plugins using +`juce::AudioParameterBool`, until [JUCE snaps its value to 0 or 1](https://forum.juce.com/t/pluginval-can-set-an-audioparameterbools-to-odd-values/50971): + +```yaml +- uses: Tracktion/pluginval/validate@v2 + env: + DISABLED_TESTS: Discrete parameter state restoration + with: + plugins: build/MyPlugin_artefacts/Release/VST3/MyPlugin.vst3 + strictness-level: 10 +``` + +Run `pluginval strictness-help ` to list the test names. + ## acceptance Runs the [acceptance tests](../../docs/Acceptance%20testing.md) for every config diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 353fc77..e71f37f 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -81,6 +81,14 @@ jobs: *) echo "pluginval=build/pluginval_artefacts/Release/pluginval" >> "$GITHUB_OUTPUT" ;; esac + # Every dogfood plugin is validated, so count them rather than hard-coding + # the number the check step expects. + shopt -s nullglob + glob='build/tests/test_plugins/*/*_artefacts/Release/VST3/*.vst3' + plugins=( $glob ) + echo "plugin-glob=$glob" >> "$GITHUB_OUTPUT" + echo "plugin-count=${#plugins[@]}" >> "$GITHUB_OUTPUT" + # Tone generator configs with the plugin removed, so it has to come from # the action's "plugin" input. One is nested and a reference manifest is # added, to check the recursion and that manifests aren't run as configs. @@ -104,7 +112,7 @@ jobs: with: pluginval-path: ${{ steps.fixtures.outputs.pluginval }} args: --skip-gui-tests - plugins: build/tests/test_plugins/*/*_artefacts/Release/VST3/*.vst3 + plugins: ${{ steps.fixtures.outputs.plugin-glob }} - name: "Action: validate (missing plugin, expected to fail)" id: validate-missing @@ -156,7 +164,8 @@ jobs: shell: bash run: | check() { echo "$1: got '$2', expected '$3'"; [ "$2" = "$3" ] || exit 1; } - check "validate passed" "${{ steps.validate.outputs.passed }}" 4 + [ "${{ steps.fixtures.outputs.plugin-count }}" -gt 0 ] || { echo "No dogfood plugins found"; exit 1; } + check "validate passed" "${{ steps.validate.outputs.passed }}" "${{ steps.fixtures.outputs.plugin-count }}" check "validate failed" "${{ steps.validate.outputs.failed }}" 0 check "validate-missing outcome" "${{ steps.validate-missing.outcome }}" failure check "validate-missing passed" "${{ steps.validate-missing.outputs.passed }}" 1