Fix async life cycle management to prevent lost references to tasks. - #157

.github/workflows/check-python.yml

@@ -0,0 +1,82 @@
+name: Check Python
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        required: true
+        type: string
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python-version }}
+          cache: pip
+          cache-dependency-path: dev-requirements.txt
+      - name: Cache development dependencies
+        run: python -m pip install -r dev-requirements.txt
+
+  build:
+    needs: prepare
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with: {fetch-depth: 0}  # deep clone for setuptools-scm
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python-version }}
+          cache: pip
+          cache-dependency-path: dev-requirements.txt
+      - name: Install development dependencies
+        run: python -m pip install -r dev-requirements.txt
+      - name: Check package builds
+        run: python -m tox -e build
+
+  test:
+    needs: prepare
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python-version }}
+          cache: pip
+          cache-dependency-path: dev-requirements.txt
+      - name: Install development dependencies
+        run: python -m pip install -r dev-requirements.txt
+      - name: Run tests
+        run: python -m tox --develop
+
+  lint:
+    needs: prepare
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python-version }}
+          cache: pip
+          cache-dependency-path: dev-requirements.txt
+      - name: Install development dependencies
+        run: python -m pip install -r dev-requirements.txt
+      - name: Run lint
+        run: python -m tox -e lint --develop
+
+  typecheck:
+    needs: prepare
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python-version }}
+          cache: pip
+          cache-dependency-path: dev-requirements.txt
+      - name: Install development dependencies
+        run: python -m pip install -r dev-requirements.txt
+      - name: Run typecheck
+        run: python -m tox -e typecheck --develop

.github/workflows/ci.yml

@@ -24,138 +24,16 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  prepare:
-    runs-on: ubuntu-latest
-    outputs:
-      wheel-distribution: ${{ steps.wheel-distribution.outputs.path }}
-    steps:
-      - uses: actions/checkout@v4
-        with: {fetch-depth: 0}  # deep clone for setuptools-scm
-      - uses: actions/setup-python@v4
-        id: setup-python
-        with: {python-version: "3.11"}
-      # - name: Run static analysis and format checkers
-      #   run: pipx run pre-commit run --all-files --show-diff-on-failure
-      - name: Build package distribution files
-        run: >-
-          pipx run --python '${{ steps.setup-python.outputs.python-path }}'
-          tox -e clean,build
-      - name: Record the path of wheel distribution
-        id: wheel-distribution
-        run: echo "path=$(ls dist/*.whl)" >> $GITHUB_OUTPUT
-      - name: Store the distribution files for use in other stages
-        # `tests` and `publish` will use the same pre-built distributions,
-        # so we make sure to release the exact same package that was tested
-        uses: actions/upload-artifact@v4
-        with:
-          name: python-distribution-files
-          path: dist/
-          retention-days: 1
-
-  test:
-    needs: prepare
+  checks:
     strategy:
       matrix:
-        python:
-          - "3.9"
-          - "3.10"
-          - "3.11"
-          - "3.12"
-          - "3.13"  # newest Python that is stable          
-        platform:
-        - ubuntu-latest
-        # - macos-latest
-        # - windows-latest
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        id: setup-python
-        with:
-          python-version: ${{ matrix.python }}
-      - name: Retrieve pre-built distribution files
-        uses: actions/download-artifact@v4
-        with: {name: python-distribution-files, path: dist/}
-      - name: Run tests
-        run: >-
-          pipx run --python '${{ steps.setup-python.outputs.python-path }}'
-          tox --develop
-      # - name: Generate coverage report
-      #   run: pipx run coverage lcov -o coverage.lcov
-      # - name: Upload partial coverage report
-      #   uses: coverallsapp/github-action@master
-      #   with:
-      #     path-to-lcov: coverage.lcov
-      #     github-token: ${{ secrets.GITHUB_TOKEN }}
-      #     flag-name: ${{ matrix.platform }} - py${{ matrix.python }}
-      #     parallel: true
-
-  lint:
-    needs: prepare
-    strategy:
-      matrix:
-        python:
-          - "3.9"
-          - "3.10"
-          - "3.11"
-          - "3.12"
-          - "3.13"  # newest Python that is stable
-        platform:
-        - ubuntu-latest
-        # - macos-latest
-        # - windows-latest
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        id: setup-python
-        with:
-          python-version: ${{ matrix.python }}
-      - name: Retrieve pre-built distribution files
-        uses: actions/download-artifact@v4
-        with: {name: python-distribution-files, path: dist/}
-      - name: Run tests
-        run: >-
-          pipx run --python '${{ steps.setup-python.outputs.python-path }}'
-          tox -e lint --develop
-
-  typecheck:
-    needs: prepare
-    strategy:
-      matrix:
-        python:
-          - "3.9"
-          - "3.10"
-          - "3.11"
-          - "3.12"
-          - "3.13"  # newest Python that is stable          
-        platform:
-        - ubuntu-latest
-        # - macos-latest
-        # - windows-latest
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        id: setup-python
-        with:
-          python-version: ${{ matrix.python }}
-      - name: Retrieve pre-built distribution files
-        uses: actions/download-artifact@v4
-        with: {name: python-distribution-files, path: dist/}
-      - name: Run tests
-        run: >-
-          pipx run --python '${{ steps.setup-python.outputs.python-path }}'
-          tox -e typecheck --develop
-
-  finalize:
-    needs: [test, lint, typecheck]
-    runs-on: ubuntu-latest
-    steps:
-      - run: echo "Finished checks"
+        python: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+    uses: ./.github/workflows/check-python.yml
+    with:
+      python-version: ${{ matrix.python }}
 
   publish:
-    needs: finalize
+    needs: checks
     if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/v') }}
     runs-on: ubuntu-latest
     environment:
@@ -165,17 +43,21 @@ jobs:
       id-token: write
     steps:
       - uses: actions/checkout@v4
+        with: {fetch-depth: 0}  # deep clone for setuptools-scm
       - uses: actions/setup-python@v4
-        with: {python-version: "3.11"}
-      - name: Retrieve pre-built distribution files
-        uses: actions/download-artifact@v4
-        with: {name: python-distribution-files, path: dist/}
+        with:
+          python-version: "3.9"
+          cache: pip
+          cache-dependency-path: dev-requirements.txt
+      - name: Install development dependencies
+        run: python -m pip install -r dev-requirements.txt
+      - name: Build package distribution files
+        run: python -m tox -e build
       - name: Publish Package
         uses: pypa/gh-action-pypi-publish@release/v1
-        # run: pipx run tox -e publish
 
   test-publish:
-    needs: finalize
+    needs: checks
     if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/test') }}
     runs-on: ubuntu-latest
     environment:
@@ -185,13 +67,17 @@ jobs:
       id-token: write
     steps:
       - uses: actions/checkout@v4
+        with: {fetch-depth: 0}  # deep clone for setuptools-scm
       - uses: actions/setup-python@v4
-        with: {python-version: "3.11"}
-      - name: Retrieve pre-built distribution files
-        uses: actions/download-artifact@v4
-        with: {name: python-distribution-files, path: dist/}
+        with:
+          python-version: "3.9"
+          cache: pip
+          cache-dependency-path: dev-requirements.txt
+      - name: Install development dependencies
+        run: python -m pip install -r dev-requirements.txt
+      - name: Build package distribution files
+        run: python -m tox -e build
       - name: Publish Package
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           repository-url: https://test.pypi.org/legacy/
-        # run: pipx run tox -e publish

.pylintrc

@@ -11,3 +11,5 @@ ignore-paths=src/s2python/generated/
 jobs=1
 
 disable=missing-class-docstring,missing-module-docstring,too-few-public-methods,missing-function-docstring,no-member,unsubscriptable-object,line-too-long,duplicate-code
+# Lint entry points disable these separately for tests; keep them enabled elsewhere.
+enable=protected-access,invalid-overridden-method

AGENTS.md

@@ -0,0 +1,48 @@
+# Project
+- This project is a library, not an application. It is intended to be used as a dependency in other projects.
+- This project defines a python version for the S2 network protocol.
+- We use semantic versioning for this project. The version number is defined in `pyproject.toml` and should be updated with each release.
+- Our interface to users is:
+  - Message definitions for the S2 network protocol that live in `s2python/{common,ddbc,frbc,ombc,pebc,ppbc}/.`
+    - We use an openapi specification to generate the message definitions. The generated code is in `s2python/generated`.
+      - We do not alter generated code manually. If we need to change the generated code, we change the openapi specification and regenerate the code.
+      - The openapi specifications lives in `specification/` and is based on the official `s2-json` repository.
+  - Connection classes that run on top of some Medium (e.g. TCP, UDP, WebSocket, etc.) and handle the S2 network protocol that live in `s2python/connection/`.
+  - Standard medium definitions for websocket and other connections that live in `s2python/connection/{async_,sync}/medium`.
+  - Example scripts under the `examples/` folder.
+
+# Python
+- We use Python 3.9 as the earliest python to support.
+- We use Python 3.14 as the latest python to support.
+
+# Poetry
+- This is a library, do not pin any dependencies in pyproject.toml that break dependency management for downstream users.
+  - Feel free to use a pin that defines a minimum version of a dependency, but do not pin to a specific version.
+- `poetry.lock` is checked in to the repository to ensure reproducible builds for development and testing but this is NOT the same environment users use.
+
+# Development
+- You have development tooling available under `ci/` that you can use to run linters, formatters, type checkers, and unit tests:
+  - `lint.sh`: Run the linters and formatters.
+  - `test_unit.sh`: Run the unit tests.
+  - `typecheck.sh`: Run the type checkers.
+  - `update_dependencies.sh`: Update dependencies in poetry.lock based on pyproject.toml.
+  - `generate_s2.sh`: Generate the S2 message definitions from the openapi specification.
+  - `clean.sh`: Remove all generated files and build artifacts that are not in the git tree but locally available.
+  - `setup_dev_environment.sh`: Set up the development environment with an initial virtual environment. Dependencies still need to be installed with `install_dependencies.sh`.
+  - `install_dependencies.sh`: Install dev dependencies in the virtual environment.
+- Library source lives under `src/s2python`.
+
+# Unit Testing
+- Use `ci/test_unit.sh` directly for unit testing instead of `python -m pytest`.
+- Use #Arrange, # Act, #Assert comments to separate the three phases of a unit test.
+- Test only one function per test file. If a function has multiple behaviors, create multiple test files for that function.
+- Use `pytest.mark.parametrize` to test multiple inputs and outputs for a single function if it simplifies code and reduces duplication.
+  - Do not overcomplicate. If parametrize leads to a test that is hard to read, break it into multiple tests instead.
+- Unit tests live under `tests/unit/`.
+  - Mimic the folder structure under `src.`
+- Unit test functions should be named according to `test__<function under test>__<behavior being tested>`.
+- Unit tests should use type hints as well.
+
+# Typehinting
+- Use `ci/typecheck.sh` for type checking.
+- We want to satisfy both pyright and mypy.

ci/lint.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env sh
 
+set -e
 . .venv/bin/activate
-pylint src/ tests/unit/ examples/
+pylint src/ examples/
+PYTHONPATH="src:$PYTHONPATH" pylint --disable=protected-access,invalid-overridden-method tests/unit/

ci/setup_dev_environment.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-python3.8 -m venv ./.venv/
+python3.9 -m venv ./.venv/
 . ./.venv/bin/activate
 pip install pip-tools

ci/test_unit.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env sh
 
 . .venv/bin/activate
-PYTHONPATH="$PYTHONPATH:src/" pytest --cov=s2python --cov-report=html:./unit_test_coverage/ -v tests/unit/
+PYTHONPATH="$PYTHONPATH:src/" pytest --cov=s2python --cov-report=html:./unit_test_coverage/ -v tests/unit/ $@

ci/typecheck.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env sh
 
 . .venv/bin/activate
-mypy --config-file mypy.ini src/ ./tests/unit/ examples/
-pyright
+status=0
+mypy --config-file mypy.ini src/ ./tests/unit/ examples/ || status=1
+pyright || status=1
+exit "$status"