diff --git a/.github/workflows/process.yml b/.github/workflows/process.yml index bbe664f884..d2e840ddba 100644 --- a/.github/workflows/process.yml +++ b/.github/workflows/process.yml @@ -17,93 +17,93 @@ env: python-version: '3.10' jobs: - unit-test: - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - name: Setup python and hatch - uses: Fusion-Power-Plant-Framework/fppf-actions/setup-hatch@c6af22fc8d00be67cca53a61177301462f499fb2 - - name: Run unit tests with coverage report - run: | - hatch run test:tests-cov tests/unit - - name: Upload coverage to Codecov - uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - fail_ci_if_error: ${{ github.ref_name == 'main' }} + # unit-test: + # strategy: + # matrix: + # os: [ubuntu-latest, macos-latest] + # runs-on: ${{ matrix.os }} + # steps: + # - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + # - name: Setup python and hatch + # uses: Fusion-Power-Plant-Framework/fppf-actions/setup-hatch@c6af22fc8d00be67cca53a61177301462f499fb2 + # - name: Run unit tests with coverage report + # run: | + # hatch run test:tests-cov tests/unit + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7 + # env: + # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + # with: + # fail_ci_if_error: ${{ github.ref_name == 'main' }} - integration-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - name: Setup python and hatch - uses: Fusion-Power-Plant-Framework/fppf-actions/setup-hatch@c6af22fc8d00be67cca53a61177301462f499fb2 - with: - python-version: ${{ env.python-version }} - - name: Run integration tests - run: hatch run test.py${{ env.python-version }}:tests-integration + # integration-test: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + # - name: Setup python and hatch + # uses: Fusion-Power-Plant-Framework/fppf-actions/setup-hatch@c6af22fc8d00be67cca53a61177301462f499fb2 + # with: + # python-version: ${{ env.python-version }} + # - name: Run integration tests + # run: hatch run test.py${{ env.python-version }}:tests-integration - example-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - name: Setup python and hatch - uses: Fusion-Power-Plant-Framework/fppf-actions/setup-hatch@c6af22fc8d00be67cca53a61177301462f499fb2 - with: - python-version: ${{ env.python-version }} - - name: Run example tests - run: hatch run test.py${{ env.python-version }}:tests-examples + # example-test: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + # - name: Setup python and hatch + # uses: Fusion-Power-Plant-Framework/fppf-actions/setup-hatch@c6af22fc8d00be67cca53a61177301462f499fb2 + # with: + # python-version: ${{ env.python-version }} + # - name: Run example tests + # run: hatch run test.py${{ env.python-version }}:tests-examples - regression-test: - runs-on: ubuntu-latest - needs: [tracking] - # depend on make and tracking job but don't skip if tracking was skipped - # run PROCESS on the regression test input files and compare results to the tracked files - if: always() - continue-on-error: true - strategy: - matrix: - tolerance: [0.2, 5] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - fetch-depth: 0 - - name: Setup python and hatch - uses: Fusion-Power-Plant-Framework/fppf-actions/setup-hatch@c6af22fc8d00be67cca53a61177301462f499fb2 - with: - python-version: ${{ env.python-version }} - - name: Run regression tests - run: hatch run test.py${{ env.python-version }}:tests-regression --reg-tolerance=${{ matrix.tolerance }} + # regression-test: + # runs-on: ubuntu-latest + # needs: [tracking] + # # depend on make and tracking job but don't skip if tracking was skipped + # # run PROCESS on the regression test input files and compare results to the tracked files + # if: always() + # continue-on-error: true + # strategy: + # matrix: + # tolerance: [0.2, 5] + # steps: + # - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + # with: + # fetch-depth: 0 + # - name: Setup python and hatch + # uses: Fusion-Power-Plant-Framework/fppf-actions/setup-hatch@c6af22fc8d00be67cca53a61177301462f499fb2 + # with: + # python-version: ${{ env.python-version }} + # - name: Run regression tests + # run: hatch run test.py${{ env.python-version }}:tests-regression --reg-tolerance=${{ matrix.tolerance }} - run-tracking-inputs: - runs-on: ubuntu-latest - # run PROCESS on the regression test input files and archive output MFILEs - # only runs upon merge to main - if: github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - name: Setup python and hatch - uses: Fusion-Power-Plant-Framework/fppf-actions/setup-hatch@c6af22fc8d00be67cca53a61177301462f499fb2 - with: - python-version: ${{ env.python-version }} - - name: Run regression input files - run: hatch run track:create-tracking-inputs - - name: Move other files - run: mv tests/regression/input_files/*.json tracking/ - - name: Archive tracked MFILEs - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: tracked-mfiles - path: tracking/*_MFILE.DAT - - name: Archive tracked SIG_TF.json - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: sig-tfs - path: tracking/*.json + # run-tracking-inputs: + # runs-on: ubuntu-latest + # # run PROCESS on the regression test input files and archive output MFILEs + # # only runs upon merge to main + # if: github.ref == 'refs/heads/main' + # steps: + # - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + # - name: Setup python and hatch + # uses: Fusion-Power-Plant-Framework/fppf-actions/setup-hatch@c6af22fc8d00be67cca53a61177301462f499fb2 + # with: + # python-version: ${{ env.python-version }} + # - name: Run regression input files + # run: hatch run track:create-tracking-inputs + # - name: Move other files + # run: mv tests/regression/input_files/*.json tracking/ + # - name: Archive tracked MFILEs + # uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + # with: + # name: tracked-mfiles + # path: tracking/*_MFILE.DAT + # - name: Archive tracked SIG_TF.json + # uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + # with: + # name: sig-tfs + # path: tracking/*.json pre-commit-quality-check: runs-on: ubuntu-latest @@ -117,6 +117,8 @@ jobs: uses: Fusion-Power-Plant-Framework/fppf-actions/hatch-lint@c6af22fc8d00be67cca53a61177301462f499fb2 improve_quality_check: + permissions: + pull-requests: write runs-on: ubuntu-latest if: github.event_name == 'pull_request' needs: [pre-commit-quality-check] @@ -155,101 +157,154 @@ jobs: hatch run lint:fmt_extra "${files[@]}" + # need number of errors being flagged in these files + num_pr_errors=$((hatch run lint:fmt_extra "${files[@]}" | tail -1) | tr -dc '0-9') + # default to 0 if no errors found + num_pr_errors=${num_pr_errors:-0} - tracking: - # update tracker to include the MFILEs created by the run-tracking-inputs job - concurrency: - group: tracking-jobs - cancel-in-progress: false - runs-on: ubuntu-latest - # depends on run-tracking-inputs; only runs upon merge to main - needs: run-tracking-inputs - if: github.ref == 'refs/heads/main' - env: - COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - name: Setup python and hatch - uses: Fusion-Power-Plant-Framework/fppf-actions/setup-hatch@c6af22fc8d00be67cca53a61177301462f499fb2 - with: - python-version: ${{ env.python-version }} - - name: Setup SSH identity - uses: webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555 - with: - ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - - name: Download tracking data - run: git clone git@github.com:timothy-nunn/process-tracking-data.git process-tracking-data - - name: Download MFILEs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: tracked-mfiles - path: tracking/ - - name: Create new tracking entries - shell: bash - run: | - MSG=$(printf "%q " $COMMIT_MESSAGE) - git config --global --add safe.directory '*' - hatch run track:run-tracker "${MSG}" ${{ github.sha }} - - name: Create the tracking dashboard - run: hatch run python tracking/tracking_data.py plot process-tracking-data --out tracking.html - - name: Archive tracking dashboard - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + echo "num pr errors" + echo "${num_pr_errors} + + id: improvement_lint + + - name: Get information about the current run + if: failure() + uses: Fusion-Power-Plant-Framework/fppf-actions/get-workflow-origin@f94a8de5f658cf65c79e1f143a7a31fb94a05793 + id: workflow-run-info + - name: Checkout and lint on main + # check out main + if: failure() && steps.workflow-run-info.outputs.pullRequestNumber != '' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - name: tracking-html - path: tracking.html - - name: Setup Git identity - run: | - git config --global user.email "${{ github.triggering_actor }}@github.runner" - git config --global user.name "${{ github.job }}" - - name: Commit and push tracking data + ref: ${{ steps.workflow-run-info.outputs.targetBranch }} + - name: Numbers of errors + if: failure() run: | - cd process-tracking-data - git add . - git commit -m "${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" - git push - - docs: - concurrency: - group: docs-jobs - cancel-in-progress: false - runs-on: ubuntu-latest - # depends on tracking; only runs upon merge to main - needs: tracking - if: github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - fetch-depth: 0 - - name: Setup python and hatch - uses: Fusion-Power-Plant-Framework/fppf-actions/setup-hatch@c6af22fc8d00be67cca53a61177301462f499fb2 - with: - python-version: ${{ env.python-version }} - - run: git config --global --add safe.directory '*' - - name: Download STF_TF.json files - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: sig-tfs - path: tracking/ - - name: Download MFILEs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: tracked-mfiles - path: tracking/ - - run: mv tracking/large_tokamak_nof.SIG_TF.json tracking/large_tokamak_nof_SIG_TF.json - - name: Create an example plot summary - run: hatch run process plot summary -f tracking/large_tokamak_nof_MFILE.DAT - - name: Move plot summary file to docs images - run: mv tracking/large_tokamak_nof_MFILE.DATSUMMARY.pdf documentation/source/images/plot_summary.pdf - - run: hatch run docs:build - - name: Download tracking html - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + num_main_errors=$((hatch run lint:fmt_extra "${files[@]}" | tail -1) | tr -dc '0-9') + num_main_errors=${num_main_errors:-0} + + echo $num_main_errors + + compared_errors=$(( ${{ steps.improvement_lint.outputs.num_pr_errors }} - ${{ steps.num_errors.outputs.num_main_errors }})) + echo "comp errors = $compared_errors" >> GITHUB_OUTPUT + echo "pr errors ${{ steps.improvement_lint.outputs.num_pr_errors }}" >> GITHUB_OUTPUT + id: num_errors + + - name: Find warning report comment + if: failure() # && steps.num_errors.outputs.compared_errors > 0 + uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 + id: find-warning-report-comment with: - name: tracking-html - - run: mv tracking.html site || cp site/404.html site/tracking.html - - name: Upload documentation page - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 + issue-number: ${{ steps.workflow-run-info.outputs.pullRequestNumber }} + comment-author: "github-actions[bot]" + body-includes: ⚠️ improve_quality_job warning! + direction: last + - name: Warning comment + if: failure() # && steps.num_errors.outputs.compared_errors > 0 + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 with: - path: site/ - - name: Deploy GitHub pages - id: deployment - uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 + comment-id: ${{ steps.find-warning-report-comment.outputs.comment-id }} + issue-number: ${{ steps.workflow-run-info.outputs.pullRequestNumber }} + body: | + Files changed by this PR have introduced $"{{ steps.num_errors.outputs.compared_errors }}" more linting errors when compared to main. + Please fix these + edit-mode: replace + + + # tracking: + # # update tracker to include the MFILEs created by the run-tracking-inputs job + # concurrency: + # group: tracking-jobs + # cancel-in-progress: false + # runs-on: ubuntu-latest + # # depends on run-tracking-inputs; only runs upon merge to main + # needs: run-tracking-inputs + # if: github.ref == 'refs/heads/main' + # env: + # COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + # steps: + # - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + # - name: Setup python and hatch + # uses: Fusion-Power-Plant-Framework/fppf-actions/setup-hatch@c6af22fc8d00be67cca53a61177301462f499fb2 + # with: + # python-version: ${{ env.python-version }} + # - name: Setup SSH identity + # uses: webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555 + # with: + # ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + # - name: Download tracking data + # run: git clone git@github.com:timothy-nunn/process-tracking-data.git process-tracking-data + # - name: Download MFILEs + # uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + # with: + # name: tracked-mfiles + # path: tracking/ + # - name: Create new tracking entries + # shell: bash + # run: | + # MSG=$(printf "%q " $COMMIT_MESSAGE) + # git config --global --add safe.directory '*' + # hatch run track:run-tracker "${MSG}" ${{ github.sha }} + # - name: Create the tracking dashboard + # run: hatch run python tracking/tracking_data.py plot process-tracking-data --out tracking.html + # - name: Archive tracking dashboard + # uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + # with: + # name: tracking-html + # path: tracking.html + # - name: Setup Git identity + # run: | + # git config --global user.email "${{ github.triggering_actor }}@github.runner" + # git config --global user.name "${{ github.job }}" + # - name: Commit and push tracking data + # run: | + # cd process-tracking-data + # git add . + # git commit -m "${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + # git push + + # docs: + # concurrency: + # group: docs-jobs + # cancel-in-progress: false + # runs-on: ubuntu-latest + # # depends on tracking; only runs upon merge to main + # needs: tracking + # if: github.ref == 'refs/heads/main' + # steps: + # - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + # with: + # fetch-depth: 0 + # - name: Setup python and hatch + # uses: Fusion-Power-Plant-Framework/fppf-actions/setup-hatch@c6af22fc8d00be67cca53a61177301462f499fb2 + # with: + # python-version: ${{ env.python-version }} + # - run: git config --global --add safe.directory '*' + # - name: Download STF_TF.json files + # uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + # with: + # name: sig-tfs + # path: tracking/ + # - name: Download MFILEs + # uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + # with: + # name: tracked-mfiles + # path: tracking/ + # - run: mv tracking/large_tokamak_nof.SIG_TF.json tracking/large_tokamak_nof_SIG_TF.json + # - name: Create an example plot summary + # run: hatch run process plot summary -f tracking/large_tokamak_nof_MFILE.DAT + # - name: Move plot summary file to docs images + # run: mv tracking/large_tokamak_nof_MFILE.DATSUMMARY.pdf documentation/source/images/plot_summary.pdf + # - run: hatch run docs:build + # - name: Download tracking html + # uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + # with: + # name: tracking-html + # - run: mv tracking.html site || cp site/404.html site/tracking.html + # - name: Upload documentation page + # uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 + # with: + # path: site/ + # - name: Deploy GitHub pages + # id: deployment + # uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 diff --git a/process/main.py b/process/main.py index ad22d34ad5..23d846b44c 100644 --- a/process/main.py +++ b/process/main.py @@ -30,6 +30,7 @@ Box file F/RS/CIRE5523/PWF (up to 15/01/96) Box file F/MI/PJK/PROCESS and F/PL/PJK/PROCESS (15/01/96 to 24/01/12) Box file T&M/PKNIGHT/PROCESS (from 24/01/12) + """ import logging @@ -608,6 +609,124 @@ def validate_input(self, replace_obsolete: bool = False): else: print("The IN.DAT file does not contain any obsolete variables.") + def test_another_lint_error(self, replace_obsolete: bool = False): + """Checks the input IN.DAT file for any obsolete variables in the OBS_VARS dict + contained within obsolete_variables.py. + If obsolete variables are found, and if `replace_obsolete` is set to True, + they are either removed or replaced by their updated names as specified + in the OBS_VARS dictionary. + + Raises + ------ + ValueError + If obsolete variables are present in the input file. + """ + obsolete_variables = ov.OBS_VARS + obsolete_vars_help_message = ov.OBS_VARS_HELP + + filename = self.input_file + variables_in_in_dat = [] + modified_lines = [] + changes_made = [] # To store details of the changes + + with open(filename) as file: + for line in file: + # Skip comment lines or lines without an assignment + if line.startswith("*") or "=" not in line: + modified_lines.append(line) + continue + + # Extract the variable name before the separator + raw_variable_name = line.split("=", 1)[0].strip() + # handle cases where the variable name might have parentheses + variable_name = ( + raw_variable_name.split("(", 1)[0] + if "(" in raw_variable_name + else raw_variable_name + ) + + # Check if the variable is obsolete and needs replacing + if variable_name in obsolete_variables: + replacement = obsolete_variables.get(variable_name) + if replace_obsolete: + # Prepare replacement or removal + if replacement is None: + # If no replacement is defined, comment out the line + modified_lines.append(f"* Obsolete: {line}") + changes_made.append( + f"Commented out obsolete variable: {variable_name}" + ) + else: + if isinstance(replacement, list): + # Raise an error if replacement is a list + replacement_str = ", ".join(replacement) + raise ValueError( + f"The variable '{variable_name}' is obsolete and " + "should be replaced by the following variables: " + f"{replacement_str}. " + "Please set their values accordingly." + ) + # Replace obsolete variable + modified_line = line.replace(variable_name, replacement, 1) + modified_lines.append( + f"* Replaced '{variable_name}' with " + f"'{replacement}'\n{modified_line}" + ) + changes_made.append( + f"Replaced '{variable_name}' with '{replacement}'" + ) + variables_in_in_dat.append(variable_name) + else: + # If replacement is False, add the line as-is + modified_lines.append(line) + variables_in_in_dat.append(variable_name) + else: + modified_lines.append(line) + + obs_vars_in_in_dat = [ + var for var in variables_in_in_dat if var in obsolete_variables + ] + + if obs_vars_in_in_dat: + if replace_obsolete: + # If replace_obsolete is True, write the modified content to the file + with open(filename, "w") as file: + file.writelines(modified_lines) + print( + "The IN.DAT file has been updated to replace or " + "comment out obsolete variables." + ) + print("Summary of changes made:") + for change in changes_made: + print(f" - {change}") + else: + # Only print the report if replace_obsolete is False + message = ( + "The IN.DAT file contains obsolete variables " + "from the OBS_VARS dictionary. " + "The obsolete variables in your IN.DAT file are: " + f"{obs_vars_in_in_dat}. " + "Either remove these or replace them with " + "their updated variable names. " + ) + for obs_var in obs_vars_in_in_dat: + replacement = obsolete_variables.get(obs_var) + if replacement is None: + message += ( + f"\n\n{obs_var} is an obsolete variable " + "and needs to be removed." + ) + else: + message += ( + f"\n\n{obs_var} is an obsolete variable " + f"and needs to be replaced by {replacement}." + ) + message += f" {obsolete_vars_help_message.get(obs_var, '')}" + raise ValueError(message) + + else: + print("The IN.DAT file does not contain any obsolete variables.") + def validate_user_model(self): """Checks that a user-created model has been injected correctly