diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..78d0f1a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git/ +tests/ +*.egg-info +build/ +dist/ diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml index 203ba25..3e61598 100644 --- a/.github/workflows/license_tests.yml +++ b/.github/workflows/license_tests.yml @@ -5,6 +5,4 @@ on: jobs: license_tests: - uses: neongeckocom/.github/.github/workflows/license_tests.yml@master - with: - packages-exclude: '^(precise-runner|fann2|tqdm|bs4|ovos-phal-plugin|ovos-skill|neon-core|nvidia|neon-phal-plugin|bitstruct|audioread).*' \ No newline at end of file + uses: neongeckocom/.github/.github/workflows/license_tests.yml@master \ No newline at end of file diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index ff73f4a..8ccf757 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -10,7 +10,18 @@ jobs: build_and_publish_pypi_and_release: uses: neongeckocom/.github/.github/workflows/publish_stable_release.yml@master secrets: inherit - build_and_publish_docker: + build_and_publish_docker_gradio: needs: build_and_publish_pypi_and_release uses: neongeckocom/.github/.github/workflows/publish_docker.yml@master - secrets: inherit \ No newline at end of file + secrets: inherit + with: + build_args: EXTRAS=gradio + platforms: linux/amd64,linux/arm64 + build_and_publish_docker_websat: + needs: build_and_publish_pypi_and_release + uses: neongeckocom/.github/.github/workflows/publish_docker.yml@master + secrets: inherit + with: + build_args: EXTRAS=web_sat + image_name: ${{ github.repository }}-websat + platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/publish_test_build.yml b/.github/workflows/publish_test_build.yml index 5d9d572..39925f3 100644 --- a/.github/workflows/publish_test_build.yml +++ b/.github/workflows/publish_test_build.yml @@ -16,7 +16,18 @@ jobs: version_file: "neon_iris/version.py" setup_py: "setup.py" publish_prerelease: true - build_and_publish_docker: + build_and_publish_docker_gradio: needs: publish_alpha_release uses: neongeckocom/.github/.github/workflows/publish_docker.yml@master - secrets: inherit \ No newline at end of file + secrets: inherit + with: + build_args: EXTRAS=gradio + platforms: linux/amd64,linux/arm64 + build_and_publish_docker_websat: + needs: publish_alpha_release + uses: neongeckocom/.github/.github/workflows/publish_docker.yml@master + secrets: inherit + with: + build_args: EXTRAS=web_sat + image_name: ${{ github.repository }}-websat + platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 69e957a..d3641f6 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -9,26 +9,32 @@ jobs: py_build_tests: uses: neongeckocom/.github/.github/workflows/python_build_tests.yml@master unit_tests: + timeout-minutes: 5 strategy: matrix: - python-version: [3.7, 3.8, 3.9, '3.10'] + python-version: ['3.10', '3.11', '3.12'] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install apt dependencies + run: | + sudo apt update + sudo apt install -y rabbitmq-server - name: Install dependencies run: | python -m pip install --upgrade pip - pip install . -r requirements/test_requirements.txt + pip install .[test] - - name: Test Client + - name: Run Tests run: | - pytest tests/test_client.py --doctest-modules --junitxml=tests/client-test-results.xml - - name: Upload client test results - uses: actions/upload-artifact@v2 + pytest tests \ + --junitxml=tests/test-results-${{ matrix.python-version }}.xml + - name: Upload test results + uses: actions/upload-artifact@v4 with: - name: client-test-results - path: tests/client-test-results.xml + name: test-results-${{ matrix.python-version }} + path: tests/test-results-${{ matrix.python-version }}.xml diff --git a/.gitignore b/.gitignore index b6e4761..90ec82f 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ htmlcov/ .cache nosetests.xml coverage.xml +tests/test-results-*.xml *.cover *.py,cover .hypothesis/ @@ -127,3 +128,5 @@ dmypy.json # Pyre type checker .pyre/ + +.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index bbbe28b..8cf4e34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,66 +1,200 @@ # Changelog -## [0.0.5a21](https://github.com/NeonGeckoCom/neon-iris/tree/0.0.5a21) (2023-12-13) +## [0.1.1a23](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a23) (2026-07-29) -[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.0.5a20...0.0.5a21) +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a22...0.1.1a23) **Merged pull requests:** -- Update Documentation and Logging [\#48](https://github.com/NeonGeckoCom/neon-iris/pull/48) ([NeonDaniel](https://github.com/NeonDaniel)) +- Update dependencies [\#85](https://github.com/NeonGeckoCom/neon-iris/pull/85) ([NeonDaniel](https://github.com/NeonDaniel)) -## [0.0.5a20](https://github.com/NeonGeckoCom/neon-iris/tree/0.0.5a20) (2023-12-13) +## [0.1.1a22](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a22) (2026-05-15) -[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.0.5a19...0.0.5a20) +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a21...0.1.1a22) + +**Closed issues:** + +- Add linux/arm64 \(aarch64\) Docker image [\#81](https://github.com/NeonGeckoCom/neon-iris/issues/81) + +**Merged pull requests:** + +- Fix onnxruntime executable stack crash on ARM64/Rosetta [\#82](https://github.com/NeonGeckoCom/neon-iris/pull/82) ([mikejgray](https://github.com/mikejgray)) + +## [0.1.1a21](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a21) (2026-05-15) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a20...0.1.1a21) + +**Merged pull requests:** + +- Update test automation [\#83](https://github.com/NeonGeckoCom/neon-iris/pull/83) ([NeonDaniel](https://github.com/NeonDaniel)) + +## [0.1.1a20](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a20) (2025-11-17) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a19...0.1.1a20) + +**Fixed bugs:** + +- \[BUG\] Exception logged during service init [\#71](https://github.com/NeonGeckoCom/neon-iris/issues/71) + +**Merged pull requests:** + +- Refactor `_wait_for_connection` to resolve startup errors [\#80](https://github.com/NeonGeckoCom/neon-iris/pull/80) ([NeonDaniel](https://github.com/NeonDaniel)) + +## [0.1.1a19](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a19) (2025-11-14) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a18...0.1.1a19) + +**Fixed bugs:** + +- \[BUG\] Support self-signed certificates [\#75](https://github.com/NeonGeckoCom/neon-iris/issues/75) + +**Merged pull requests:** + +- Resolve dead MQ connector that leaves service in a broken state [\#79](https://github.com/NeonGeckoCom/neon-iris/pull/79) ([NeonDaniel](https://github.com/NeonDaniel)) + +## [0.1.1a18](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a18) (2025-07-31) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a17...0.1.1a18) + +**Merged pull requests:** + +- Update to Gradio 5.x with Session Handling Fix [\#78](https://github.com/NeonGeckoCom/neon-iris/pull/78) ([NeonDaniel](https://github.com/NeonDaniel)) + +## [0.1.1a17](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a17) (2025-06-25) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a16...0.1.1a17) + +**Merged pull requests:** + +- Implement Docker health checks [\#76](https://github.com/NeonGeckoCom/neon-iris/pull/76) ([NeonDaniel](https://github.com/NeonDaniel)) + +## [0.1.1a16](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a16) (2025-04-15) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a15...0.1.1a16) + +**Merged pull requests:** + +- Refactor BCP-47 Language codes to ISO 639-1 [\#73](https://github.com/NeonGeckoCom/neon-iris/pull/73) ([NeonDaniel](https://github.com/NeonDaniel)) + +## [0.1.1a15](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a15) (2025-04-09) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a14...0.1.1a15) + +**Merged pull requests:** + +- Implement models from `neon-data-models` [\#74](https://github.com/NeonGeckoCom/neon-iris/pull/74) ([NeonDaniel](https://github.com/NeonDaniel)) + +## [0.1.1a14](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a14) (2025-03-11) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a13...0.1.1a14) + +**Merged pull requests:** + +- Update dependencies [\#72](https://github.com/NeonGeckoCom/neon-iris/pull/72) ([NeonDaniel](https://github.com/NeonDaniel)) + +## [0.1.1a13](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a13) (2025-02-18) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a12...0.1.1a13) + +**Merged pull requests:** + +- Handle reconnection to RMQ Server [\#70](https://github.com/NeonGeckoCom/neon-iris/pull/70) ([NeonDaniel](https://github.com/NeonDaniel)) + +## [0.1.1a12](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a12) (2025-02-06) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a11...0.1.1a12) **Merged pull requests:** -- Improved config handling [\#46](https://github.com/NeonGeckoCom/neon-iris/pull/46) ([NeonDaniel](https://github.com/NeonDaniel)) +- Update dependency and license notices to prep for stable release [\#68](https://github.com/NeonGeckoCom/neon-iris/pull/68) ([NeonDaniel](https://github.com/NeonDaniel)) -## [0.0.5a19](https://github.com/NeonGeckoCom/neon-iris/tree/0.0.5a19) (2023-12-11) +## [0.1.1a11](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a11) (2025-02-03) -[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.0.5a18...0.0.5a19) +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a10...0.1.1a11) **Merged pull requests:** -- Update release automation [\#47](https://github.com/NeonGeckoCom/neon-iris/pull/47) ([NeonDaniel](https://github.com/NeonDaniel)) +- Refactor to support Async consumers [\#67](https://github.com/NeonGeckoCom/neon-iris/pull/67) ([NeonDaniel](https://github.com/NeonDaniel)) + +## [0.1.1a10](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a10) (2025-01-20) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a9...0.1.1a10) + +**Merged pull requests:** + +- fix: require numpy\<2 for websat [\#66](https://github.com/NeonGeckoCom/neon-iris/pull/66) ([mikejgray](https://github.com/mikejgray)) + +## [0.1.1a9](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a9) (2025-01-14) -## [0.0.5a18](https://github.com/NeonGeckoCom/neon-iris/tree/0.0.5a18) (2023-12-11) +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a8...0.1.1a9) -[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.0.4...0.0.5a18) +**Merged pull requests:** + +- Refactor Docker build to override default config instead of system [\#63](https://github.com/NeonGeckoCom/neon-iris/pull/63) ([NeonDaniel](https://github.com/NeonDaniel)) -**Implemented enhancements:** +## [0.1.1a8](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a8) (2025-01-14) -- \[FEAT\] Gradio User Settings Intents [\#35](https://github.com/NeonGeckoCom/neon-iris/issues/35) -- \[FEAT\] After submitting recorded input audio the UI element should reset [\#30](https://github.com/NeonGeckoCom/neon-iris/issues/30) -- \[FEAT\] Speech input should print STT results to the chat UI in addition to response [\#29](https://github.com/NeonGeckoCom/neon-iris/issues/29) +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a7...0.1.1a8) **Fixed bugs:** -- \[BUG\] Input audio resampling is not efficiently implemented [\#28](https://github.com/NeonGeckoCom/neon-iris/issues/28) -- \[BUG\] Implement per-user settings in web UI [\#27](https://github.com/NeonGeckoCom/neon-iris/issues/27) - -**Merged pull requests:** - -- Add GitHub pre-release automation [\#45](https://github.com/NeonGeckoCom/neon-iris/pull/45) ([NeonDaniel](https://github.com/NeonDaniel)) -- Update neon-mq-connector dependency to stable spec [\#43](https://github.com/NeonGeckoCom/neon-iris/pull/43) ([NeonDaniel](https://github.com/NeonDaniel)) -- Minor fixes to audio input handling [\#42](https://github.com/NeonGeckoCom/neon-iris/pull/42) ([NeonDaniel](https://github.com/NeonDaniel)) -- Refactor audio responses to utilize Chatbot UI [\#40](https://github.com/NeonGeckoCom/neon-iris/pull/40) ([NeonDaniel](https://github.com/NeonDaniel)) -- Get Language Support from Core [\#37](https://github.com/NeonGeckoCom/neon-iris/pull/37) ([NeonDaniel](https://github.com/NeonDaniel)) -- Handle profile updates [\#36](https://github.com/NeonGeckoCom/neon-iris/pull/36) ([NeonDaniel](https://github.com/NeonDaniel)) -- Handle STT Transcripts in Chat UI [\#34](https://github.com/NeonGeckoCom/neon-iris/pull/34) ([NeonDaniel](https://github.com/NeonDaniel)) -- Remove audio resampling and add timing context support [\#33](https://github.com/NeonGeckoCom/neon-iris/pull/33) ([NeonDaniel](https://github.com/NeonDaniel)) -- Fix web\_client language handling to respect configured input language [\#32](https://github.com/NeonGeckoCom/neon-iris/pull/32) ([NeonDaniel](https://github.com/NeonDaniel)) -- Threaded input handling and multi-session support [\#31](https://github.com/NeonGeckoCom/neon-iris/pull/31) ([NeonDaniel](https://github.com/NeonDaniel)) -- Add Gradio web UI with Docker Container [\#24](https://github.com/NeonGeckoCom/neon-iris/pull/24) ([NeonDaniel](https://github.com/NeonDaniel)) -- Add NeonVoiceClient class for minimal remote audio client [\#23](https://github.com/NeonGeckoCom/neon-iris/pull/23) ([NeonDaniel](https://github.com/NeonDaniel)) -- Resolve client compat. bug [\#22](https://github.com/NeonGeckoCom/neon-iris/pull/22) ([NeonDaniel](https://github.com/NeonDaniel)) -- Add CLI endpoints to interact with API and LLM endpoints [\#21](https://github.com/NeonGeckoCom/neon-iris/pull/21) ([NeonDaniel](https://github.com/NeonDaniel)) -- Specify `setup.py` path explicitly [\#20](https://github.com/NeonGeckoCom/neon-iris/pull/20) ([NeonDaniel](https://github.com/NeonDaniel)) -- Update build automation to default branch [\#19](https://github.com/NeonGeckoCom/neon-iris/pull/19) ([NeonDaniel](https://github.com/NeonDaniel)) -- Update to use shared version\_bump.py script [\#18](https://github.com/NeonGeckoCom/neon-iris/pull/18) ([NeonDaniel](https://github.com/NeonDaniel)) -- Remove invalid release Docker action [\#17](https://github.com/NeonGeckoCom/neon-iris/pull/17) ([NeonDaniel](https://github.com/NeonDaniel)) -- Update automation to shared repository [\#15](https://github.com/NeonGeckoCom/neon-iris/pull/15) ([NeonDaniel](https://github.com/NeonDaniel)) -- Add methods with CLI entrypoints to get STT/TTS [\#12](https://github.com/NeonGeckoCom/neon-iris/pull/12) ([NeonDaniel](https://github.com/NeonDaniel)) +- \[BUG\] Upgrade Python version in Docker [\#62](https://github.com/NeonGeckoCom/neon-iris/issues/62) + +**Merged pull requests:** + +- Update Python version and dependencies [\#65](https://github.com/NeonGeckoCom/neon-iris/pull/65) ([NeonDaniel](https://github.com/NeonDaniel)) + +## [0.1.1a7](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a7) (2025-01-11) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a6...0.1.1a7) + +**Merged pull requests:** + +- Update GitHub Actions [\#64](https://github.com/NeonGeckoCom/neon-iris/pull/64) ([NeonDaniel](https://github.com/NeonDaniel)) + +## [0.1.1a6](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a6) (2024-05-14) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a5...0.1.1a6) + +**Fixed bugs:** + +- \[BUG\] Audio input results in errors [\#59](https://github.com/NeonGeckoCom/neon-iris/issues/59) + +**Merged pull requests:** + +- fix: appropriate dependencies [\#61](https://github.com/NeonGeckoCom/neon-iris/pull/61) ([mikejgray](https://github.com/mikejgray)) + +## [0.1.1a5](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a5) (2024-04-10) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a4...0.1.1a5) + +**Merged pull requests:** + +- Implement alert message handler [\#57](https://github.com/NeonGeckoCom/neon-iris/pull/57) ([NeonDaniel](https://github.com/NeonDaniel)) + +## [0.1.1a4](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a4) (2024-01-29) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a2...0.1.1a4) + +**Merged pull requests:** + +- feat: polished UI for websat [\#54](https://github.com/NeonGeckoCom/neon-iris/pull/54) ([mikejgray](https://github.com/mikejgray)) + +## [0.1.1a2](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a2) (2023-12-29) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.1a1...0.1.1a2) + +**Merged pull requests:** + +- Add CLI command to get mapmaker geolocation responses [\#53](https://github.com/NeonGeckoCom/neon-iris/pull/53) ([NeonDaniel](https://github.com/NeonDaniel)) + +## [0.1.1a1](https://github.com/NeonGeckoCom/neon-iris/tree/0.1.1a1) (2023-12-28) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-iris/compare/0.1.0...0.1.1a1) + +**Merged pull requests:** + +- feat: browser sat [\#51](https://github.com/NeonGeckoCom/neon-iris/pull/51) ([mikejgray](https://github.com/mikejgray)) diff --git a/Dockerfile b/Dockerfile index 139a09a..fbf0516 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,51 @@ -FROM python:3.8-slim +FROM python:3.10-slim +# Label for vendor LABEL vendor=neon.ai \ ai.neon.name="neon-iris" -ENV OVOS_CONFIG_BASE_FOLDER neon -ENV OVOS_CONFIG_FILENAME neon.yaml -ENV XDG_CONFIG_HOME /config +# Build argument for specifying extras +ARG EXTRAS -RUN apt update && \ - apt install -y ffmpeg +ENV OVOS_DEFAULT_CONFIG=/opt/neon/neon.yaml \ + OVOS_CONFIG_BASE_FOLDER=neon \ + OVOS_CONFIG_FILENAME=neon.yaml \ + XDG_CONFIG_HOME=/config +# Set the ARG value as an environment variable +ENV EXTRAS=${EXTRAS} -ADD . /neon_iris -WORKDIR /neon_iris +RUN mkdir -p /neon_iris/requirements +COPY ./requirements/* /neon_iris/requirements +RUN apt-get update && apt-get install --no-install-recommends -y curl; rm -rf /var/lib/apt/lists/* +RUN pip install wheel && pip install -r /neon_iris/requirements/requirements.txt +RUN if [ "$EXTRAS" = "gradio" ]; then \ + pip install -r /neon_iris/requirements/gradio.txt; \ + elif [ "$EXTRAS" = "web_sat" ]; then \ + pip install -r /neon_iris/requirements/web_sat.txt; \ + else \ + pip install -r /neon_iris/requirements/requirements.txt; \ + fi -RUN pip install wheel && \ - pip install .[gradio] +WORKDIR /neon_iris +COPY . /neon_iris +RUN pip install . COPY docker_overlay/ / -CMD ["iris", "start-gradio"] \ No newline at end of file +RUN apt-get update \ + && apt-get install -y libsndfile1 libasound2 ffmpeg \ + && apt-get --purge autoremove -y \ + && apt-get clean \ + && rm -rf "${HOME}"/.cache /var/lib/apt /var/log/{apt,dpkg.log} + +# Create a non-root user with a home directory and change ownership of necessary directories + +RUN groupadd -r neon && useradd -r -m -g neon neon \ + && mkdir -p /config/neon \ + && chown -R neon:neon /neon_iris /usr/local/bin /config + +# Use the non-root user to run the container +USER neon + +HEALTHCHECK CMD "/neon_iris/healthcheck.sh" +ENTRYPOINT ["/neon_iris/entrypoint.sh"] diff --git a/LICENSE.md b/LICENSE.md index 525bb37..0fdcf76 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System # All trademark and other rights reserved by their respective owners -# Copyright 2008-2021 Neongecko.com Inc. +# Copyright 2008-2025 Neongecko.com Inc. # BSD-3 License Redistribution and use in source and binary forms, with or without modification, are permitted provided that the diff --git a/README.md b/README.md index 16a537e..f94675e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Neon Iris + Neon Iris (Interactive Relay for Intelligence Systems) provides tools for interacting with Neon systems remotely, via [MQ](https://github.com/NeonGeckoCom/chat_api_mq_proxy). @@ -6,15 +7,18 @@ Install the Iris Python package with: `pip install neon-iris` The `iris` entrypoint is available to interact with a bus via CLI. Help is available via `iris --help`. ## Configuration -Configuration files can be specified via environment variables. By default, -`Iris` will read configuration from `~/.config/neon/diana.yaml` where + +Configuration files can be specified via environment variables. By default, +`Iris` will read configuration from `~/.config/neon/diana.yaml` where `XDG_CONFIG_HOME` is set to the default `~/.config`. -More information about configuration handling can be found +More information about configuration handling can be found [in the docs](https://neongeckocom.github.io/neon-docs/quick_reference/configuration/). -> *Note:* The neon-iris Docker image uses `neon.yaml` by default because the + +> _Note:_ The neon-iris Docker image uses `neon.yaml` by default because the > `iris` web UI is often deployed with neon-core. A default configuration might look like: + ```yaml MQ: server: neonaialpha.com @@ -24,32 +28,133 @@ MQ: user: neon_api_utils password: Klatchat2021 iris: - default_lang: en-us + default_lang: en languages: - - en-us - - uk-ua + - en + - uk webui_chatbot_label: "Neon AI" webui_mic_label: "Speak with Neon" webui_input_placeholder: "Chat with Neon" ``` ### Language Support + For Neon Core deployments that support language support queries via MQ, `languages` may be removed and `enable_lang_api: True` added to configuration. This will use the reported STT/TTS supported languages in place of any `iris` configuration. ## Interfacing with a Diana installation + The `iris` CLI includes utilities for interacting with a `Diana` backend. Use `iris --help` to get a current list of available commands. ### `iris start-listener` -This will start a local wake word recognizer and use a remote Neon + +This will start a local wake word recognizer and use a remote Neon instance connected to MQ for processing audio and providing responses. ### `iris start-gradio` + This will start a local webserver and serve a Gradio UI to interact with a Neon instance connected to MQ. ### `iris start-client` -This starts a CLI client for typing inputs and receiving responses from a Neon + +This starts a CLI client for typing inputs and receiving responses from a Neon instance connected via MQ. + +### `iris start-websat` + +This starts a local webserver and serves a web UI for interacting with a Neon +instance connected to MQ. + +## Docker + +### Building + +To build the Docker image, run: + +```bash +docker build -t ghcr.io/neongeckocom/neon-iris:latest . +``` + +To build the Docker image with gradio extras, run: + +```bash +docker build --build-arg EXTRAS=gradio -t ghcr.io/neongeckocom/neon-iris:latest . +``` + +To build the Docker image with websat extras, run: + +```bash +docker build --build-arg EXTRAS=web_sat -t ghcr.io/neongeckocom/neon-iris:latest . +``` + +### Running + +The Docker image that is built for this service runs the `iris` CLI with the +`-h` argument by default. In order to use the container to run different services, +you must override the entrypoint. For example, to run the `start-websat` service, +you would run: + +```bash +docker run --rm -p 8000:8000 ghcr.io/neongeckocom/neon-iris:latest start-websat +``` + +Running the container without any arguments gives you a list of commands that +can be run. You can choose to run any of these commands by replacing `start-websat` +in the above command with the command you want to run. + +## websat + +### Configuration + +The `websat` web UI is a simple web UI for interacting with a Neon instance. It +accepts special configuration items prefixed with `webui_` to customize the UI. + +| parameter | description | default | +| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ---------------------- | +| webui_description | The header text for the web UI | Chat with Neon | +| webui_title | The title text for the web UI in the browser | Neon AI | +| webui_input_placeholder | The placeholder text for the input box | Ask me something | +| webui_ws_url | The websocket URL to connect to, which must be accessible from the browser you're running in. Note that the default will usually fail. | ws://localhost:8000/ws | + +Iris uses the `Configuration()` class from OVOS to handle configuration. This +means that you can specify configuration in a `neon.yaml` file in the +`~/.config/neon`. When using a container, you can mount a volume to +`/home/neon/.config/neon` to provide a configuration file. + +Example configuration block: + +```yaml +iris: + webui_title: Neon AI + webui_description: Chat with Neon + webui_input_placeholder: Ask me something + webui_ws_url: wss://neonaialpha.com/ws +``` + +### Customization + +The websat web UI reads in the following items from `neon_iris/static/custom`: + +- `error.mp3` - Used for error responses +- `wake.mp3` - Used for wake word responses +- `favicon.ico` - The favicon for the web UI +- `logo.svg` - The logo for the web UI + +To customize these items, you can replace them in the `neon_iris/static/custom` folder and rebuild the image. + +### Websocket endpoint + +The websat web UI uses a websocket to communicate with OpenWakeWord, which can +load `.tflite` or `.onnx` models. The websocket endpoint is `/ws`, but since it +is served with FastAPI, it also supports `wss` for secure connections. To +use `wss`, you must provide a certificate and key file. + +### Chat history + +The websat web UI stores chat history in the browser's [local storage](https://developer.mozilla.org/en-US/docs/Web/API/Window/localStorage). +This allows chat history to persist between browser sessions. However, it also +means that if you clear your browser's local storage, you will lose your chat +history. This is a feature, not a bug. diff --git a/docker_overlay/neon_iris/entrypoint.sh b/docker_overlay/neon_iris/entrypoint.sh new file mode 100755 index 0000000..2a4f022 --- /dev/null +++ b/docker_overlay/neon_iris/entrypoint.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2025 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -e + +if [ "$EXTRAS" = "gradio" ]; then + exec iris start-gradio + elif [ "$EXTRAS" = "web_sat" ]; then + exec iris start-websat +else + echo "No extras specified, showing help. To execute a command, use 'docker run iris '" + exec iris -h +fi diff --git a/docker_overlay/neon_iris/healthcheck.sh b/docker_overlay/neon_iris/healthcheck.sh new file mode 100755 index 0000000..a8b1801 --- /dev/null +++ b/docker_overlay/neon_iris/healthcheck.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2025 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if [ "$EXTRAS" = "gradio" ]; then + port=7860 +else + port=8000 +fi +# Perform the health check using curl +resp=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:${port}/status) + +# Check if the response code is 200 +if [ "$resp" == "200" ]; then + exit 0 # Success +else + echo "Health check failed with HTTP status: $resp" >&2 + exit 1 # Failure +fi diff --git a/docker_overlay/etc/neon/neon.yaml b/docker_overlay/opt/neon/neon.yaml similarity index 86% rename from docker_overlay/etc/neon/neon.yaml rename to docker_overlay/opt/neon/neon.yaml index f3f8e79..8b09f27 100644 --- a/docker_overlay/etc/neon/neon.yaml +++ b/docker_overlay/opt/neon/neon.yaml @@ -12,9 +12,10 @@ iris: webui_chatbot_label: Chat History webui_mic_label: Speak to Neon webui_text_label: Text with Neon + webui_ws_url: ws://localhost:8000/ws # Override, as this needs to be reachable by the browser server_address: "0.0.0.0" server_port: 7860 - default_lang: en-us + default_lang: en enable_lang_api: True location: @@ -43,4 +44,4 @@ logs: error: - pika warning: - - filelock \ No newline at end of file + - filelock diff --git a/neon_iris/__init__.py b/neon_iris/__init__.py index 1efd3a5..ff12569 100644 --- a/neon_iris/__init__.py +++ b/neon_iris/__init__.py @@ -1,6 +1,6 @@ # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System # All trademark and other rights reserved by their respective owners -# Copyright 2008-2021 Neongecko.com Inc. +# Copyright 2008-2025 Neongecko.com Inc. # BSD-3 # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/neon_iris/cli.py b/neon_iris/cli.py index 2539846..480afee 100644 --- a/neon_iris/cli.py +++ b/neon_iris/cli.py @@ -1,6 +1,6 @@ # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System # All trademark and other rights reserved by their respective owners -# Copyright 2008-2021 Neongecko.com Inc. +# Copyright 2008-2025 Neongecko.com Inc. # BSD-3 # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -138,6 +138,19 @@ def start_gradio(): click.echo("Unable to connect to MQ server") +@neon_iris_cli.command(help="Create a Web Voice Satellite session") +@click.option("--port", "-p", default=8000, help="Port to run on, defaults to 8000") +@click.option("--host", default="0.0.0.0", help="Host to run on, defaults to 0.0.0.0") +def start_websat(port, host): + from neon_iris.web_sat_client import app + _print_config() + try: + import uvicorn + uvicorn.run(app, host=host, port=port) + except OSError: + click.echo("Unable to connect to MQ server") + + @neon_iris_cli.command(help="Query Neon Core for supported languages") def get_languages(): from neon_iris.util import query_neon @@ -236,6 +249,23 @@ def get_wolfram_response(api, unit, latitude, longitude, question): click.echo(pformat(resp)) +@neon_iris_cli.command(help="Query Map Maker Geolocation endpoint") +@click.option('--latitude', '--lat', + help="location latitude") +@click.option('--longitude', '--lon', + help="location latitude") +@click.option('--query', '-q', help="Name of location to look up") +def get_mapmaker_response(latitude, longitude, query): + from neon_iris.util import query_api + _print_config() + query = {"lat": latitude, + "lon": longitude, + "address": query, + "service": "map_maker"} + resp = query_api(query) + click.echo(pformat(resp)) + + @neon_iris_cli.command(help="Converse with an LLM") @click.option('--llm', default="chat_gpt", help="LLM Queue to interact with ('chat_gpt' or 'fastchat')") diff --git a/neon_iris/client.py b/neon_iris/client.py index a79984b..57e9b0e 100644 --- a/neon_iris/client.py +++ b/neon_iris/client.py @@ -1,6 +1,6 @@ # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System # All trademark and other rights reserved by their respective owners -# Copyright 2008-2021 Neongecko.com Inc. +# Copyright 2008-2025 Neongecko.com Inc. # BSD-3 # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -39,10 +39,10 @@ from uuid import uuid4 from ovos_bus_client.message import Message from ovos_utils.json_helper import merge_dict -from pika.exceptions import StreamLostError +from pika.exceptions import ConnectionWrongStateError +from neon_iris.mq_connector import IrisConnector from neon_utils.configuration_utils import get_neon_user_config from neon_utils.metrics_utils import Stopwatch -from neon_utils.mq_utils import NeonMQHandler from neon_utils.socket_utils import b64_to_dict from neon_utils.file_utils import decode_base64_string_to_file, \ encode_file_to_base64_string @@ -107,21 +107,10 @@ def user_config(self) -> dict: return json.loads(json.dumps(self._user_config.content)) @property - def connection(self) -> NeonMQHandler: + def connection(self) -> IrisConnector: """ Returns a connected NeonMQHandler object """ - if not self._connection.connection.is_open: - LOG.warning("Connection closed") - self._connection.stop() - self._connection = self._init_mq_connection() - try: - self._connection.connection.channel() - except StreamLostError: - LOG.warning("Connection unexpectedly closed, recreating") - self._connection.stop() - self._connection = self._init_mq_connection() - return self._connection def shutdown(self): @@ -129,7 +118,8 @@ def shutdown(self): Cleanly shuts down the MQ connection associated with this client """ try: - self._connection.stop() + self._connection.shutdown() + LOG.info("Shut down Connection") except Exception as e: LOG.error(e) try: @@ -142,6 +132,7 @@ def shutdown(self): except Exception as x: LOG.exception(x) LOG.error("Consumers not shutdown") + raise e def handle_neon_response(self, channel, method, _, body): """ @@ -152,18 +143,20 @@ def handle_neon_response(self, channel, method, _, body): with _stopwatch: response = b64_to_dict(body) LOG.debug(f"Message deserialized in {_stopwatch.time}s") + # TODO: This should extend NeonApiMessage once that class is extended + # to include all of the below `msg_type`s message = Message(response.get('msg_type'), response.get('data'), response.get('context')) # Get timing data and log message.context.setdefault("timing", {}) - resp_time = message.context['timing'].get('response_sent', recv_time) + resp_time = message.context['timing'].get('response_sent') or recv_time if recv_time != resp_time: transit_time = recv_time - resp_time message.context['timing']['client_from_core'] = transit_time LOG.debug(f"Response MQ transit time={transit_time}") - handling_time = recv_time - message.context['timing'].get('client_sent', - recv_time) + handling_time = recv_time - (message.context['timing'].get( + 'client_sent') or recv_time) LOG.info(f"{message.msg_type} handled in {handling_time}") LOG.debug(f"{pformat(message.context['timing'])}") if message.msg_type == "klat.response": @@ -179,6 +172,8 @@ def handle_neon_response(self, channel, method, _, body): self.handle_error_response(message) elif message.msg_type == "neon.languages.get.response": self._handle_supported_languages(message) + elif message.msg_type == "neon.alert_expired": + self.handle_alert(message) elif message.msg_type.endswith(".response"): self.handle_api_response(message) else: @@ -231,6 +226,12 @@ def clear_media(self, message: Message): Override this method to handle requests to clear media (photos, etc) """ + @abstractmethod + def handle_alert(self, message: Message): + """ + Override this method to handle alerts (timers, alarms, reminders) + """ + def _handle_profile_update(self, message: Message): updated_profile = message.data["profile"] if updated_profile['user']['username'] == \ @@ -267,32 +268,36 @@ def _handle_supported_languages(self, message: Message): self._languages = message.data if not all((x in self._languages for x in ("stt", "tts"))): LOG.warning(f"Language support incomplete response: {self._languages}") + self._languages['stt'] = [l.split('-')[0] + for l in self._languages.get('stt', [])] + self._languages['tts'] = [l.split('-')[0] + for l in self._languages.get('tts', [])] self._languages['stt'].sort() self._languages['tts'].sort() self._language_init.set() - def send_utterance(self, utterance: str, lang: str = "en-us", + def send_utterance(self, utterance: str, lang: str = "en", username: Optional[str] = None, user_profiles: Optional[list] = None, context: Optional[dict] = None): """ Optionally override this to queue text inputs or do any pre-parsing :param utterance: utterance to submit to skills module - :param lang: language code associated with request + :param lang: ISO 639-1 language code associated with request :param username: username associated with request :param user_profiles: user profiles expecting a response :param context: Optional dict context to add to emitted message """ self._send_utterance(utterance, lang, username, user_profiles, context) - def send_audio(self, audio_file: str, lang: str = "en-us", + def send_audio(self, audio_file: str, lang: str = "en", username: Optional[str] = None, user_profiles: Optional[list] = None, context: Optional[dict] = None): """ Optionally override this to queue audio inputs or do any pre-parsing :param audio_file: path to audio file to send to speech module - :param lang: language code associated with request + :param lang: ISO 639-1 language code associated with request :param username: username associated with request :param user_profiles: user profiles expecting a response :param context: Optional dict context to add to emitted message @@ -333,7 +338,7 @@ def _send_utterance(self, utterance: str, lang: str, self._send_serialized_message(serialized) def _send_audio(self, audio_file: str, lang: str, - username: str, user_profiles: list, + username: Optional[str], user_profiles: Optional[list], context: Optional[dict] = None): context = context or dict() audio_data = encode_file_to_base64_string(audio_file) @@ -367,13 +372,21 @@ def _send_serialized_message(self, serialized: dict): queue="neon_chat_api_request", request_data=serialized) LOG.debug(f"emitted {serialized.get('msg_type')}") + except ConnectionWrongStateError: + LOG.error("Restarting RMQ client and retrying") + self.shutdown() + self._connection = self._init_mq_connection() + self.connection.emit_mq_message( + self._connection.connection, + queue="neon_chat_api_request", + request_data=serialized) except Exception as e: - LOG.exception(e) self.shutdown() + raise e def _init_mq_connection(self): mq_config = self._config.get("MQ") or self._config - mq_connection = NeonMQHandler(mq_config, "mq_handler", self._vhost) + mq_connection = IrisConnector(vhost=self._vhost, config=mq_config) mq_connection.register_consumer("neon_response_handler", self._vhost, self.uid, self.handle_neon_response, auto_ack=False) @@ -381,7 +394,8 @@ def _init_mq_connection(self): "neon_chat_api_error", self.handle_neon_error, auto_ack=False) - mq_connection.run(daemon=True) + mq_connection.start() + mq_connection.wait_for_connection() return mq_connection @@ -469,15 +483,18 @@ def handle_api_response(self, message: Message): def clear_caches(self, message: Message): print("Cached Responses Cleared") + def handle_alert(self, message: Message): + print(f"\nAlert Expired: {message.data.get('alert_name')}") + def clear_media(self, message: Message): pass - def send_utterance(self, utterance: str, lang: str = "en-us", + def send_utterance(self, utterance: str, lang: str = "en", _=None, __=None): """ Queue a string request for skills processing :param utterance: User utterance to submit - :param lang: language of utterance + :param lang: ISO 639-1 language of utterance """ self._response_event.clear() self._request_queue.put((utterance, lang)) @@ -486,12 +503,12 @@ def send_utterance(self, utterance: str, lang: str = "en-us", while not self._request_queue.empty(): self._response_event.wait(30) - def send_audio(self, audio_file: str, lang: str = "en-us", + def send_audio(self, audio_file: str, lang: str = "en", _=None, __=None): """ Send an audio file for skills processing :param audio_file: Audio File to submit for STT processing - :param lang: language of audio + :param lang: ISO 639-1 language of audio """ self._response_event.clear() self._send_audio(audio_file, lang, self.username, self.user_profiles) diff --git a/neon_iris/llm.py b/neon_iris/llm.py index 6cb0b65..1cabb6c 100644 --- a/neon_iris/llm.py +++ b/neon_iris/llm.py @@ -1,6 +1,6 @@ # NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework # All trademark and other rights reserved by their respective owners -# Copyright 2008-2022 Neongecko.com Inc. +# Copyright 2008-2025 Neongecko.com Inc. # Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, # Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo # BSD-3 License diff --git a/neon_iris/models/__init__.py b/neon_iris/models/__init__.py new file mode 100644 index 0000000..baee249 --- /dev/null +++ b/neon_iris/models/__init__.py @@ -0,0 +1,27 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2025 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from .web_sat import UserInput, UserInputResponse # noqa \ No newline at end of file diff --git a/neon_iris/models/web_sat.py b/neon_iris/models/web_sat.py new file mode 100644 index 0000000..70b1a34 --- /dev/null +++ b/neon_iris/models/web_sat.py @@ -0,0 +1,42 @@ +"""API data models for the WebSAT API.""" +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2025 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from typing import Optional +from pydantic import BaseModel + +class UserInput(BaseModel): + """UserInput is the input data model for the WebSAT API.""" + utterance: Optional[str] = "" + audio_input: Optional[str] = "" + session_id: str = "websat0000" + +class UserInputResponse(BaseModel): + """UserInputResponse is the response data model for the WebSAT API.""" + utterance: Optional[str] = "" + audio_output: Optional[str] = "" + session_id: str = "websat0000" + transcription: str diff --git a/neon_iris/mq_connector.py b/neon_iris/mq_connector.py new file mode 100644 index 0000000..1f7e377 --- /dev/null +++ b/neon_iris/mq_connector.py @@ -0,0 +1,152 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2025 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import pika.exceptions + +from time import sleep, time +from threading import Event, Thread +from neon_mq_connector.utils.client_utils import MQConnector +from ovos_utils.log import LOG +from ovos_utils.process_utils import ProcessStatus +from pika.adapters.select_connection import SelectConnection +from pika.channel import Channel + + +class IrisConnector(MQConnector, Thread): + async_consumers_enabled = True + + def __init__(self, *args, **kwargs): + vhost = kwargs.pop('vhost') + # TODO: service_name chosen for backwards-compat. and should be updated + # to something more descriptive + kwargs['service_name'] = 'mq_handler' + Thread.__init__(self, daemon=True) + MQConnector.__init__(self, *args, **kwargs) + self.status = ProcessStatus(name="iris") + self.vhost = vhost + self._ready = Event() + self._channel_closed = Event() + self._stopping = False + + self.status.set_alive() + self._connection = self.init_connection() + + def wait_for_connection(self): + LOG.info("Waiting for connection") + self._ready.wait() + self.status.set_ready() + LOG.info("Connected!") + + @property + def connection(self) -> SelectConnection: + if self._connection is None: + self._connection = self.init_connection() + return self._connection + + @property + def ready(self) -> bool: + return self._ready.is_set() + + def run(self, *args, **kwargs): + MQConnector.run(self) + self._connection.ioloop.start() + + def init_connection(self) -> SelectConnection: + return SelectConnection( + parameters=self.get_connection_params(self.vhost), + on_open_callback=self.on_connected, + on_open_error_callback=self.on_connection_fail, + on_close_callback=self.on_close) + + def on_connected(self, _: pika.SelectConnection): + """Called when we are fully connected to RabbitMQ""" + LOG.info("MQ Connected") + self.status.set_ready() + self.connection.channel(on_open_callback=self.on_channel_open) + + def on_connection_fail(self, *_, **__): + """ Called when connection to RabbitMQ fails""" + LOG.error("Failed to connect to MQ") + self.status.set_error("MQ Connection failed") + self._connection = None + + def on_channel_open(self, new_channel: Channel): + """Called when our channel has opened""" + LOG.info("MQ Channel opened.") + new_channel.add_on_close_callback(self.on_channel_close) + self._ready.set() + + def on_channel_close(self, *_, **__): + LOG.info("Channel closed.") + self.status.set_stopping() + self._channel_closed.set() + + def on_close(self, _: pika.SelectConnection, e: Exception): + if isinstance(e, pika.exceptions.ConnectionClosed): + LOG.info(f"Connection closed normally: {e}") + elif isinstance(e, pika.exceptions.StreamLostError): + LOG.warning("MQ connection lost; " + "RabbitMQ is likely temporarily unavailable.") + else: + LOG.error(f"MQ connection closed due to exception: {e}") + self.status.set_stopping() + if not self._stopping: + # Connection was gracefully closed by the server. Try to re-connect + LOG.info("Trying to reconnect after server closed connection") + self._connection = self.init_connection() + + def shutdown(self): + """ + Clean up this object. Closes all connections and stops any processing. + """ + try: + self.status.set_stopping() + self._stopping = True + if self.connection and not (self.connection.is_closed or + self.connection.is_closing): + self.connection.close() + LOG.info("Waiting for channel close") + if not self._channel_closed.wait(15): + raise TimeoutError("Timeout waiting for channel close.") + + # Wait for the connection to close + waiter = Event() + timeout = time() + 15 + while not self.connection.is_closed and time() < timeout: + waiter.wait(1) + if not self.connection.is_closed: + raise TimeoutError("Timeout waiting for connection close.") + LOG.info("Connection closed") + + if self.connection: + LOG.info("Stopping IOLoop") + self.connection.ioloop.stop() + MQConnector.stop(self) + self._ready.clear() + LOG.info("MQ Connector shutdown complete") + + except Exception as e: + LOG.error(f"Failed to close connection: {e}") diff --git a/neon_iris/static/custom/error.mp3 b/neon_iris/static/custom/error.mp3 new file mode 100644 index 0000000..c6404ce Binary files /dev/null and b/neon_iris/static/custom/error.mp3 differ diff --git a/neon_iris/static/custom/favicon.ico b/neon_iris/static/custom/favicon.ico new file mode 100644 index 0000000..b49b0fb Binary files /dev/null and b/neon_iris/static/custom/favicon.ico differ diff --git a/neon_iris/static/custom/logo.svg b/neon_iris/static/custom/logo.svg new file mode 100644 index 0000000..1b21556 --- /dev/null +++ b/neon_iris/static/custom/logo.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/neon_iris/static/custom/wake.mp3 b/neon_iris/static/custom/wake.mp3 new file mode 100644 index 0000000..efb06b4 Binary files /dev/null and b/neon_iris/static/custom/wake.mp3 differ diff --git a/neon_iris/static/scripts/audio.js b/neon_iris/static/scripts/audio.js new file mode 100644 index 0000000..4b71ee3 --- /dev/null +++ b/neon_iris/static/scripts/audio.js @@ -0,0 +1,112 @@ +// Manages audio capture and processing +const AudioHandler = (() => { + let audioStream; + let audioContext; + let recorder; + let volume; + let sampleRate; + let isRecording = false; + + // Ensure the getUserMedia is correctly referenced + const getUserMedia = + navigator.getUserMedia || + navigator.webkitGetUserMedia || + navigator.mozGetUserMedia || + navigator.msGetUserMedia; + + const startAudio = () => { + if (getUserMedia) { + getUserMedia.call( + navigator, + { audio: true }, + (stream) => { + audioStream = stream; + const AudioContext = window.AudioContext || window.webkitAudioContext; + audioContext = new AudioContext(); + sampleRate = audioContext.sampleRate; + volume = audioContext.createGain(); + const audioInput = audioContext.createMediaStreamSource(audioStream); + audioInput.connect(volume); + + const bufferSize = 4096; + // Use the audio context to create the script processor + recorder = audioContext.createScriptProcessor(bufferSize, 1, 1); + + recorder.onaudioprocess = (event) => { + const samples = event.inputBuffer.getChannelData(0); + const PCM16iSamples = convertFloat32ToInt16(samples); + WebSocketHandler.send( + new Blob([PCM16iSamples], { type: "application/octet-stream" }) + ); + }; + + volume.connect(recorder); + recorder.connect(audioContext.destination); + WebSocketHandler.setSampleRate(sampleRate); + isRecording = true; + }, + (error) => { + console.error("Error capturing audio.", error); + } + ); + } else { + console.error("getUserMedia not supported in this browser."); + } + }; + + const stopAudio = () => { + if (isRecording) { + if (recorder) { + recorder.disconnect(); + volume.disconnect(); + // Disconnecting the audio context might not be necessary + // audioContext.close(); + } + if (audioStream) { + const tracks = audioStream.getTracks(); + tracks.forEach((track) => track.stop()); + } + } + }; + + const toggle = () => { + if (!isRecording) { + startAudio(); + } else { + stopAudio(); + } + isRecording = !isRecording; // Toggle the recording state + }; + + const isCurrentlyRecording = () => isRecording; + + const convertFloat32ToInt16 = (buffer) => { + let l = buffer.length; + let buf = new Int16Array(l); + while (l--) { + buf[l] = Math.min(1, buffer[l]) * 0x7fff; + } + return buf.buffer; + }; + + return { + toggle, + isRecording: isCurrentlyRecording, + }; +})(); + +const startButton = document.getElementById("startButton"); +startButton.addEventListener("click", function () { + AudioHandler.toggle(); + + // Update the button's text and class based on the recording state + if (AudioHandler.isRecording()) { + startButton.classList.add("listening"); + startButton.classList.toggle("bg-blue-500"); + startButton.textContent = "Listening..."; + } else { + startButton.classList.toggle("bg-blue-500"); + startButton.classList.remove("listening"); + startButton.textContent = "Start Listening"; + } +}); diff --git a/neon_iris/static/scripts/sprite.js b/neon_iris/static/scripts/sprite.js new file mode 100644 index 0000000..303e72b --- /dev/null +++ b/neon_iris/static/scripts/sprite.js @@ -0,0 +1,20 @@ +const sprite = document.querySelector(".sprite"); + +function triggerWake() { + sprite.classList.remove("record", "waiting"); + sprite.classList.add("wake"); +} + +function triggerRecord() { + sprite.classList.remove("wake", "waiting"); + sprite.classList.add("record"); +} + +function triggerWaiting() { + sprite.classList.remove("wake", "record"); + sprite.classList.add("waiting"); +} + +function triggerDone() { + sprite.classList.remove("wake", "record", "waiting"); +} diff --git a/neon_iris/static/scripts/ui.js b/neon_iris/static/scripts/ui.js new file mode 100644 index 0000000..7e55d77 --- /dev/null +++ b/neon_iris/static/scripts/ui.js @@ -0,0 +1,150 @@ +function submitMessage() { + const inputElement = document.getElementById("chatInput"); + const userMessage = inputElement.value.trim(); + + if (userMessage !== "") { + const userMessageDiv = createMessageDiv("user", userMessage); + appendMessageToHistory(userMessageDiv); + + // Save the message to localStorage + saveMessageToLocalStorage("user", userMessage); + + inputElement.value = ""; + + // Get AI response and update the chat history + getAIResponse(userMessage); // Pass the user message to the function + } +} + +async function getAIResponse(text = "", recording = "") { + try { + triggerWaiting(); // Trigger waiting animation + const payload = + text !== "" && recording === "" + ? { utterance: text } + : { audio_input: recording }; + // Make the POST request to the server + const response = await fetch("/user_input", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), // Send the user message in the body + }); + + // Check if the response is okay + if (!response.ok) { + throw new Error("Network response was not ok: " + response.statusText); + } + + // Convert the response payload into JSON + const data = await response.json(); + console.debug(data, null, 4); + + // Assuming 'data' contains the AI response in a property named 'reply' + const aiMessage = data.transcription; + + triggerDone(); // Trigger done animation + // Add in the user's transcription if STT + if (text === "" && recording !== "") { + const userMessage = createMessageDiv("user", data.utterance); + appendMessageToHistory(userMessage); + saveMessageToLocalStorage("user", data.utterance); + } + + // Create the AI message div and append it to the history + const aiMessageDiv = createMessageDiv("ai", aiMessage); + appendMessageToHistory(aiMessageDiv); + + // Save the AI message to localStorage + saveMessageToLocalStorage("ai", aiMessage); + + // Play the TTS audio + const audioBlob = base64ToBlob(data.audio_output, "audio/wav"); + const audioUrl = URL.createObjectURL(audioBlob); + const audio = new Audio(audioUrl); + audio.type = "audio/wav"; + await audio.play(); + audio.onended = () => { + if (shouldListen && myVad) { + myVad.start(); + } else { + myVad.pause(); + } + }; + } catch (error) { + console.error("Error fetching AI response:", error); + // Handle the error, such as showing a message to the user + } +} + +function simulateAIResponse() { + setTimeout(() => { + const aiMessage = "This is a sample AI response."; + const aiMessageDiv = createMessageDiv("ai", aiMessage); + appendMessageToHistory(aiMessageDiv); + + // Save the AI response to localStorage + saveMessageToLocalStorage("ai", aiMessage); + }, 1000); // Simulated delay of 1 second +} + +function createMessageDiv(sender, message) { + const messageDiv = document.createElement("div"); + messageDiv.className = `${sender}-message`; + messageDiv.textContent = message; + return messageDiv; +} + +function appendMessageToHistory(messageDiv) { + const messageContainer = document.getElementById("chatHistory"); + messageContainer.appendChild(messageDiv); + setTimeout(() => { + messageContainer.scrollTop = messageContainer.scrollHeight; + }, 0); +} + +function saveMessageToLocalStorage(sender, message) { + // Retrieve existing chat history from localStorage + const chatHistory = JSON.parse(localStorage.getItem("chatHistory")) || []; + + // Add the new message to the chat history + chatHistory.push({ sender, message }); + + // Store the updated chat history back in localStorage + localStorage.setItem("chatHistory", JSON.stringify(chatHistory)); +} + +function base64ToBlob(base64, mimeType) { + const byteCharacters = atob(base64.replace(/^data:audio\/wav;base64,/, "")); + const byteNumbers = new Array(byteCharacters.length); + for (let i = 0; i < byteCharacters.length; i++) { + byteNumbers[i] = byteCharacters.charCodeAt(i); + } + const byteArray = new Uint8Array(byteNumbers); + return new Blob([byteArray], { type: mimeType }); +} + +// Load chat history from localStorage when the page loads +window.addEventListener("load", () => { + const chatHistory = JSON.parse(localStorage.getItem("chatHistory")) || []; + + for (const { sender, message } of chatHistory) { + const messageDiv = createMessageDiv(sender, message); + appendMessageToHistory(messageDiv); + } +}); + +document.addEventListener("DOMContentLoaded", function () { + // Get the input element + const inputElement = document.getElementById("chatInput"); + + // Add the keydown event listener to the input element + inputElement.addEventListener("keydown", function (event) { + // Check if Enter was pressed, or Ctrl+Enter + if (event.key === "Enter" && (event.ctrlKey || !event.shiftKey)) { + event.preventDefault(); + submitMessage(); + } + }); +}); diff --git a/neon_iris/static/scripts/websocket.js b/neon_iris/static/scripts/websocket.js new file mode 100644 index 0000000..0415a36 --- /dev/null +++ b/neon_iris/static/scripts/websocket.js @@ -0,0 +1,162 @@ +function float32ArrayToWavBlob(float32Array, sampleRate = 16000) { + const buffer = new ArrayBuffer(44 + float32Array.length * 2); + const view = new DataView(buffer); + + // Write WAV header to the buffer + // RIFF chunk descriptor + writeString(view, 0, "RIFF"); + view.setUint32(4, 36 + float32Array.length * 2, true); + writeString(view, 8, "WAVE"); + // FMT sub-chunk + writeString(view, 12, "fmt "); + view.setUint32(16, 16, true); // Subchunk1Size (16 for PCM) + view.setUint16(20, 1, true); // AudioFormat (PCM = 1) + view.setUint16(22, 1, true); // NumChannels (Mono = 1, Stereo = 2) + view.setUint32(24, sampleRate, true); // SampleRate + view.setUint32(28, sampleRate * 2, true); // ByteRate (SampleRate * NumChannels * BitsPerSample/8) + view.setUint16(32, 2, true); // BlockAlign (NumChannels * BitsPerSample/8) + view.setUint16(34, 16, true); // BitsPerSample + // Data sub-chunk + writeString(view, 36, "data"); + view.setUint32(40, float32Array.length * 2, true); + + // Write the audio data + float32To16BitPCM(view, 44, float32Array); + + return new Blob([view], { type: "audio/wav" }); +} + +function writeString(view, offset, string) { + for (let i = 0; i < string.length; i++) { + view.setUint8(offset + i, string.charCodeAt(i)); + } +} + +function float32To16BitPCM(output, offset, input) { + for (let i = 0; i < input.length; i++, offset += 2) { + const s = Math.max(-1, Math.min(1, input[i])); + output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true); + } +} + +function wavBlobToBase64(blob) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onloadend = () => { + const base64data = reader.result; + // Extract the base64 part + const base64String = base64data.split(",")[1]; + resolve(base64String); + }; + reader.onerror = (error) => { + reject(error); + }; + }); +} + +let shouldListen = false; // Global state flag for controlling VAD listening state +let myVad; // VAD instance +let isVadRunning = false; + +async function initializeVad() { + myVad = await vad.MicVAD.new({ + onSpeechEnd: handleSpeechEnd, + }); + if (shouldListen && !isVadRunning) { + myVad.start(); + isVadRunning = true; + } +} + +async function handleSpeechEnd(audio) { + const wavBlob = float32ArrayToWavBlob(audio); + const audioUrl = URL.createObjectURL(wavBlob); + const audioOutput = await wavBlobToBase64(wavBlob); + + // Save the spoken audio as a downloadable file + const downloadArea = document.getElementById("download-area"); + if (downloadArea) { + downloadArea.innerHTML = ""; // Clear the download area + const downloadButton = document.createElement("a"); + downloadButton.href = audioUrl; + downloadButton.download = "recorded_audio.wav"; + downloadButton.textContent = "Download Recorded Audio"; + downloadButton.className = "download-button"; // Add a class for styling + downloadButton.setAttribute("role", "button"); // Accessibility improvement + downloadArea.appendChild(downloadButton); + triggerWaiting(); // Trigger waiting animation + } else { + console.error("Download area not found"); + } + if (myVad && isVadRunning) { + myVad.pause(); + isVadRunning = false; + shouldListen = false; + } + + // Send audio to STT + getAIResponse("", audioOutput); +} + +function toggleListeningState() { + shouldListen = !shouldListen; + if (shouldListen && !isVadRunning) { + startVad(); + isVadRunning = true; + } else { + stopVad(); + isVadRunning = false; + } +} + +// Handles WebSocket connection and message events +const WebSocketHandler = (() => { + let lastActivationTime = 0; + const activationCooldown = 3000; // 3 seconds cooldown + const ws = new WebSocket(WS_URL); + const audio = new Audio("/static/custom/wake.mp3"); // Wakeword acknowledgment sound + + ws.onopen = () => { + console.info("WebSocket connection is open"); + }; + + ws.onmessage = async (event) => { + console.log(event.data); + const model_payload = JSON.parse(event.data); + const currentTime = Date.now(); + if ("activations" in model_payload) { + if ( + model_payload.activations.includes("hey_neon_high") && + currentTime - lastActivationTime > activationCooldown + ) { + shouldListen = true; + audio.onended = () => { + console.log("Activation sound is done playing"); + if (myVad && !isVadRunning) { + triggerRecord(); // Trigger recording animation + myVad.start(); + isVadRunning = true; + } else if (!shouldListen && isVadRunning) { + myVad.pause(); + isVadRunning = false; + } + }; + triggerWake(); // Trigger wake animation + audio.play(); + lastActivationTime = currentTime; + } + } + }; + + return { + send: (data) => ws.send(data), + setSampleRate: (rate) => ws.send(rate), + }; +})(); + +// Initialize VAD when the page is ready +window.addEventListener("DOMContentLoaded", (event) => { + initializeVad(); +}); +WebSocketHandler; diff --git a/neon_iris/static/sprite.css b/neon_iris/static/sprite.css new file mode 100644 index 0000000..73d5d1e --- /dev/null +++ b/neon_iris/static/sprite.css @@ -0,0 +1,61 @@ +/* Base styles for the split sprite */ +.sprite { + width: 40px; /* sprite width adjusted to 40px */ + height: 40px; /* sprite height adjusted to 40px */ + border-radius: 50%; + background-image: linear-gradient(to right, #000 50%, #fff 50%); + box-shadow: 0 0 0 2px #000; /* Adjusted border thickness for smaller size */ + transition: transform 0.3s ease, opacity 0.3s ease; + opacity: 0; /* sprite is invisible by default */ + position: relative; /* Required for absolute positioning of pseudo-elements */ + display: flex; /* Center content */ + justify-content: center; + align-items: center; + margin: 20px; +} + +/* Pulse animation while recording */ +@keyframes pulse { + 0%, + 100% { + transform: scale(1); + } + 50% { + transform: scale(1.1); + } +} + +.sprite.record { + animation: pulse 1s infinite ease-in-out; + opacity: 1; /* sprite is visible while recording */ +} + +/* Spin animation while waiting for a response */ +@keyframes spin { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } +} + +.sprite.waiting { + animation: spin 2s infinite linear; + opacity: 1; /* sprite is visible while waiting */ +} + +/* Appear animation for wake word activation */ +@keyframes appear { + 0% { + opacity: 0; + } + 100% { + opacity: 1; + } +} + +.sprite.wake { + animation: appear 1s forwards; + opacity: 1; /* sprite is visible when awake */ +} diff --git a/neon_iris/static/styles.css b/neon_iris/static/styles.css new file mode 100644 index 0000000..802a215 --- /dev/null +++ b/neon_iris/static/styles.css @@ -0,0 +1,36 @@ +#startButton.listening { + background-color: var(--color-neon-green); +} + +/* Style for user messages */ +.user-message { + background-color: #007bff; /* Blue background for user messages */ + color: #fff; /* White text color for user messages */ + padding: 5px 10px; + margin: 5px 0; + border-radius: 10px; + align-self: flex-end; /* Right-align user messages */ + max-width: 60%; + word-wrap: break-word; /* Wrap long words if needed */ +} + +/* Style for AI messages */ +.ai-message { + background-color: #e0e0e0; /* Gray background for AI messages */ + color: #000; /* Black text color for AI messages */ + padding: 5px 10px; + margin: 5px 0; + border-radius: 10px; + align-self: flex-start; /* Left-align AI messages */ + max-width: 60%; + word-wrap: break-word; /* Wrap long words if needed */ +} + +.chat-container { + display: flex; + flex-direction: column; + height: 100%; + background-color: #1a1a1a; + overflow: auto; + max-height: calc(100vh - 20vh); +} diff --git a/neon_iris/templates/index.html b/neon_iris/templates/index.html new file mode 100644 index 0000000..de9fee5 --- /dev/null +++ b/neon_iris/templates/index.html @@ -0,0 +1,84 @@ + + + + + + + {{ title }} + + + + + + + +
+
+ {{ description }} + +
+ +
+
+ +
+
+ +
+ + +
+
+
+ + + + + + + + + + + + diff --git a/neon_iris/util.py b/neon_iris/util.py index 95f8201..89e135f 100644 --- a/neon_iris/util.py +++ b/neon_iris/util.py @@ -1,6 +1,6 @@ # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System # All trademark and other rights reserved by their respective owners -# Copyright 2008-2021 Neongecko.com Inc. +# Copyright 2008-2025 Neongecko.com Inc. # BSD-3 # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/neon_iris/version.py b/neon_iris/version.py index 4488dcb..80e1bb7 100644 --- a/neon_iris/version.py +++ b/neon_iris/version.py @@ -1,8 +1,9 @@ # NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework # All trademark and other rights reserved by their respective owners -# Copyright 2008-2022 Neongecko.com Inc. +# Copyright 2008-2025 Neongecko.com Inc. # Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, # Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# Mike Gray, David Scripka # BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -26,4 +27,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/neon_iris/voice_client.py b/neon_iris/voice_client.py index d8efd90..c9bf666 100644 --- a/neon_iris/voice_client.py +++ b/neon_iris/voice_client.py @@ -1,6 +1,6 @@ # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System # All trademark and other rights reserved by their respective owners -# Copyright 2008-2021 Neongecko.com Inc. +# Copyright 2008-2025 Neongecko.com Inc. # BSD-3 # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -43,6 +43,7 @@ from ovos_utils.sound import play_wav from ovos_bus_client.message import Message from neon_utils.file_utils import decode_base64_string_to_file +from neon_data_models.models.api.messagebus import NeonTtsResponse from neon_iris.client import NeonAIClient @@ -111,14 +112,17 @@ def on_hotword_audio(self, audio: bytes, context: dict): self.bus.emit(Message(msg_type, payload, context)) def handle_klat_response(self, message: Message): - responses = message.data.get('responses') + response = NeonTtsResponse(msg_type=message.msg_type, + data=message.data, + context=message.context) + responses = response.data.responses for lang, data in responses.items(): - text = data.get('sentence') + text = data.sentence LOG.info(text) file_basename = f"{hash(text)}.wav" - genders = data.get('genders', []) + genders = data.genders for gender in genders: - audio_data = data["audio"].get(gender) + audio_data = data.audio.get(gender) audio_file = join(self._tts_audio_path, lang, gender, file_basename) try: diff --git a/neon_iris/wakeword_models/hey_neon/hey_neon.onnx b/neon_iris/wakeword_models/hey_neon/hey_neon.onnx new file mode 100644 index 0000000..c66a46d Binary files /dev/null and b/neon_iris/wakeword_models/hey_neon/hey_neon.onnx differ diff --git a/neon_iris/wakeword_models/hey_neon/hey_neon.tflite b/neon_iris/wakeword_models/hey_neon/hey_neon.tflite new file mode 100644 index 0000000..048ab6c Binary files /dev/null and b/neon_iris/wakeword_models/hey_neon/hey_neon.tflite differ diff --git a/neon_iris/wakeword_models/hey_neon/hey_neon_high.onnx b/neon_iris/wakeword_models/hey_neon/hey_neon_high.onnx new file mode 100644 index 0000000..3dd5d65 Binary files /dev/null and b/neon_iris/wakeword_models/hey_neon/hey_neon_high.onnx differ diff --git a/neon_iris/wakeword_models/hey_neon/hey_neon_high.tflite b/neon_iris/wakeword_models/hey_neon/hey_neon_high.tflite new file mode 100644 index 0000000..790eca4 Binary files /dev/null and b/neon_iris/wakeword_models/hey_neon/hey_neon_high.tflite differ diff --git a/neon_iris/web_client.py b/neon_iris/web_client.py index b33fd66..a49bbf1 100644 --- a/neon_iris/web_client.py +++ b/neon_iris/web_client.py @@ -1,6 +1,6 @@ # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System # All trademark and other rights reserved by their respective owners -# Copyright 2008-2021 Neongecko.com Inc. +# Copyright 2008-2025 Neongecko.com Inc. # BSD-3 # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -27,9 +27,11 @@ from os import makedirs from os.path import isfile, join, isdir from time import time -from typing import List, Dict, Tuple -from uuid import uuid4 +from typing import List, Dict +from fastapi import FastAPI, Response +from ovos_utils.process_utils import ProcessState +import uvicorn import gradio from threading import Event @@ -37,9 +39,9 @@ from ovos_config import Configuration from ovos_utils import LOG from ovos_utils.json_helper import merge_dict - from neon_utils.file_utils import decode_base64_string_to_file from ovos_utils.xdg_utils import xdg_data_home +from neon_data_models.models.api.messagebus import NeonTtsResponse from neon_iris.client import NeonAIClient @@ -50,6 +52,7 @@ def __init__(self, lang: str = None): self.config = config.get('iris') or dict() NeonAIClient.__init__(self, config.get("MQ")) self._await_response = Event() + self._alerts = dict() self._response = None self._transcribed = None self._current_tts = dict() @@ -57,29 +60,45 @@ def __init__(self, lang: str = None): self._audio_path = join(xdg_data_home(), "iris", "stt") if not isdir(self._audio_path): makedirs(self._audio_path) - self.default_lang = lang or self.config.get('default_lang') + self.default_lang = (lang or + self.config.get('default_lang')).split('-')[0] self.chat_ui = gradio.Blocks() - def get_lang(self, session_id: str): + def get_lang(self, session_id: str) -> str: + """ + Get the ISO 639-1 language code for the specified session + @param session_id: Gradio session ID + @returns: ISO 639-1 language code + """ if session_id and session_id in self._profiles: - return self._profiles[session_id]['speech']['stt_language'] - return self.user_config['speech']['stt_language'] or self.default_lang + return self._profiles[session_id]['speech']['stt_language'].split('-')[0] + return (self.user_config['speech']['stt_language'] or + self.default_lang).split('-')[0] @property def supported_languages(self) -> List[str]: """ Get a list of supported languages from configuration - @returns: list of BCP-47 language codes + @returns: list of ISO 639-1 language codes """ - return self.config.get('languages') or [self.default_lang] + return [lang.split('-')[0] for lang in + self.config.get('languages') or [self.default_lang]] - def _start_session(self): - sid = uuid4().hex + def _start_session(self, request: gradio.Request): + sid = request.session_hash self._current_tts[sid] = None self._profiles[sid] = self.user_config self._profiles[sid]['user']['username'] = sid return sid + def check_alerts(self, session_id: str): + if not self._alerts.get(session_id): + gradio.Info("No Alerts") + return session_id + while self._alerts.get(session_id): + gradio.Info(self._alerts[session_id].pop()) + return session_id + def update_profile(self, stt_lang: str, tts_lang: str, tts_lang_2: str, time: int, date: str, uom: str, city: str, state: str, country: str, first: str, middle: str, last: str, @@ -116,16 +135,20 @@ def update_profile(self, stt_lang: str, tts_lang: str, tts_lang_2: str, return session_id def on_user_input(self, utterance: str, - chat_history: List[Tuple[str, str]], - audio_input: str, - client_session: str) -> (List[Tuple[str, str]], str, str, None, str): + chat_history: List[gradio.ChatMessage], + audio_input: str, # gr.Audio object input + client_session: str):# -> tuple[List[Tuple[str, str]], str, Literal[''], None, Any]: """ Callback to handle textual user input @param utterance: String utterance submitted by the user @returns: Input box contents, Updated chat history, Gradio session ID, audio input, audio output """ + if not any((utterance, audio_input)): + # Empty input + return chat_history, client_session, "", None, None + input_time = time() - LOG.debug(f"Input received") + LOG.debug("Input received") if not self._await_response.wait(30): LOG.error("Previous response not completed after 30 seconds") in_queue = time() - input_time @@ -148,20 +171,34 @@ def on_user_input(self, utterance: str, context={"gradio": {"session": gradio_id}, "timing": {"wait_in_queue": in_queue, "gradio_sent": time()}}) - chat_history.append(((audio_input, None), None)) + # Add the audio input to the chat history + chat_message = gradio.ChatMessage(role="user", content=audio_input) + chat_history.append(chat_message) if not self._await_response.wait(30): LOG.error("No response received after 30s") self._await_response.set() self._response = self._response or "ERROR" LOG.info(f"Got response={self._response}") if utterance: - chat_history.append((utterance, self._response)) + chat_history.append(gradio.ChatMessage(role="user", content=utterance)) elif isinstance(self._transcribed, str): LOG.info(f"Got transcript: {self._transcribed}") - chat_history.append((self._transcribed, self._response)) - chat_history.append((None, (self._current_tts[gradio_id], None))) + chat_history.append(gradio.ChatMessage(role="user", content=self._transcribed)) + chat_history.append(gradio.ChatMessage(role="assistant", content=self._response)) + # Add audio response to chat + chat_message = gradio.ChatMessage(role="assistant", content={"path": self._current_tts[gradio_id]}) + chat_history.append(chat_message) + LOG.info(f"Chat history now has {len(chat_history)} messages") return chat_history, gradio_id, "", None, self._current_tts[gradio_id] + def play_tts_audio(self, session_id: str): + """ + Helper method to return TTS audio for playback + """ + audio_file = self._current_tts.get(session_id) + LOG.info(f"Playing TTS audio: {audio_file}") + return audio_file + # def play_tts(self, session_id: str): # LOG.info(f"Playing most recent TTS file {self._current_tts}") # return self._current_tts.get(session_id), session_id @@ -182,22 +219,68 @@ def run(self): port = self.config.get("server_port") or 7860 with self.chat_ui as blocks: - client_session = gradio.State(self._start_session()) - client_session.attach_load_event(self._start_session, None) + # Add custom CSS to hide audio waveforms + blocks.css = """ + /* Ensure elements in a row share the same height */ + #custom-submit { + height: 100%; /* Match the height of other elements */ + display: flex; + align-items: center; + } + + /* Hide the waveform container and visualization */ + .waveform-container, #waveform, .svelte-19usgod .waveform-container { + display: none !important; + } + + /* Hide timestamps */ + .timestamps { + display: none !important; + } + + /* Keep only essential playback controls and make them compact */ + [data-testid="unlabelled-audio"] { + max-height: 60px !important; + overflow: hidden !important; + } + + /* Make the controls more compact */ + .controls.svelte-ije4bl { + padding: 5px !important; + margin: 0 !important; + } + + /* Hide volume and playback speed controls to make it even more minimal */ + .control-wrapper .volume, .control-wrapper .playback { + display: none !important; + } + + /* Hide rewind/skip buttons for minimal interface */ + .rewind, .skip { + display: none !important; + } + """ + # Do session-specific initialization + client_session = gradio.State() + blocks.load(self._start_session, None, outputs=[client_session]) + # Define primary UI blocks.title = title - chatbot = gradio.Chatbot(label=chatbot_label) - with gradio.Row(): + chatbot = gradio.Chatbot(type='messages', label=chatbot_label) + with gradio.Row(height='120px'): textbox = gradio.Textbox(label=text_label, - placeholder=placeholder, + placeholder=placeholder, lines=2, scale=8) - audio_input = gradio.Audio(source="microphone", + audio_input = gradio.Audio(sources=["microphone"], type="filepath", label=speech, scale=2) - submit = gradio.Button(value="Submit", + submit = gradio.Button(value="Submit", elem_id="custom-submit", variant="primary") - tts_audio = gradio.Audio(autoplay=True, visible=False) + tts_audio = gradio.Audio(autoplay=True, visible=True, + label="Response Audio", + interactive=False, + type="filepath", scale=1) submit.click(self.on_user_input, inputs=[textbox, chatbot, audio_input, client_session], @@ -214,9 +297,12 @@ def run(self): # inputs=[client_session], # outputs=[tts_audio, client_session]) # Define settings UI + with gradio.Row(): + submit = gradio.Button("Update User Settings") + check_alerts = gradio.Button("Check for Alerts") with gradio.Row(): with gradio.Column(): - lang = self.get_lang(client_session.value).split('-')[0] + lang = self.get_lang(client_session.value) stt_lang = gradio.Radio(label="Input Language", choices=self._languages.get("stt") or self.supported_languages, @@ -253,14 +339,38 @@ def run(self): pref_name = gradio.Textbox(label="Preferred Name") email_addr = gradio.Textbox(label="Email Address") # TODO: DoB, pic, about, phone? - submit = gradio.Button("Update User Settings") submit.click(self.update_profile, inputs=[stt_lang, tts_lang, tts_lang_2, time_format, date_format, unit_of_measure, city, state, country, first_name, middle_name, last_name, pref_name, email_addr, client_session], outputs=[client_session]) - blocks.launch(server_name=address, server_port=port) + check_alerts.click(self.check_alerts, inputs=[client_session], + outputs=[client_session]) + app = FastAPI() + + def _health_check(*_): + if not self.connection.check_health(): + return Response(status_code=500, content="MQ Error") + + if self.connection.status.state == ProcessState.ALIVE: + # Startup + return Response(status_code=503, content="Starting") + if self.connection.status.state == ProcessState.STOPPING: + # Shutdown + return Response(status_code=503, content="Stopping") + if self.connection.status.state == ProcessState.ERROR: + # Error + return Response(status_code=500, content="Error") + if self.connection.status.state == ProcessState.READY: + return Response(status_code=200, content="Ready") + + # Catch-all exception case + return Response(status_code=500, content="Error") + + app.add_route("/status", _health_check, methods=["GET"]) + gr_app = gradio.mount_gradio_app(app, blocks, '') + uvicorn.run(gr_app, host=address, port=port) def handle_klat_response(self, message: Message): """ @@ -268,17 +378,24 @@ def handle_klat_response(self, message: Message): audio in all requested languages. @param message: Neon response message """ - LOG.debug(f"gradio context={message.context['gradio']}") - resp_data = message.data["responses"] + response = NeonTtsResponse(msg_type=message.msg_type, + data=message.data, + context=message.context) + LOG.debug(f"gradio context={response.context.gradio}") + resp_data = response.data.responses files = [] sentences = [] - session = message.context['gradio']['session'] + session = response.context.gradio.session + LOG.info(f"Got response for session {session}") for lang, response in resp_data.items(): - sentences.append(response.get("sentence")) - if response.get("audio"): - for gender, data in response["audio"].items(): + LOG.debug(f"Response for {lang}: {response}") + sentences.append(response.sentence) + if response.audio: + for gender, data in response.audio.items(): + audio_path = getattr(response, gender) + LOG.debug(f"Got audio file: {audio_path}") filepath = "/".join([self.audio_cache_dir] + - response[gender].split('/')[-4:]) + audio_path.split('/')[-4:]) # TODO: This only plays the most recent, so it doesn't # support multiple languages or multi-utterance responses self._current_tts[session] = filepath @@ -307,6 +424,17 @@ def handle_api_response(self, message: Message): if message.msg_type == "neon.audio_input.response": self._transcribed = message.data.get("transcripts", [""])[0] + def handle_alert(self, message: Message): + """ + Handle an expired alert that was previously set by this session. + @param message: neon.alert_expired Message + """ + user = message.context['username'] + LOG.info(f"Alert expired for user: {user}") + alert = f"Alert Expired: {message.data.get('alert_name')}" + self._alerts.setdefault(user, list()) + self._alerts[user].append(alert) + def _handle_profile_update(self, message: Message): updated_profile = message.data["profile"] session_id = updated_profile['user']['username'] diff --git a/neon_iris/web_sat_client.py b/neon_iris/web_sat_client.py new file mode 100644 index 0000000..6013592 --- /dev/null +++ b/neon_iris/web_sat_client.py @@ -0,0 +1,340 @@ +"""Runs a web server that serves the Neon AI Web UI and Voice Satellite.""" +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2025 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +from os import makedirs +from os.path import isdir, join +from threading import Event +from time import time +from typing import Dict, Optional, Sequence +from uuid import uuid4 + +import numpy as np +import resampy +from fastapi import APIRouter, FastAPI, Request, WebSocket, Response +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +from neon_utils.file_utils import decode_base64_string_to_file +from openwakeword import Model +from ovos_bus_client import Message +from ovos_config import Configuration +from ovos_utils import LOG +from ovos_utils.xdg_utils import xdg_data_home +from neon_data_models.models.api.messagebus import NeonTtsResponse + +from neon_iris.client import NeonAIClient +from neon_iris.models.web_sat import UserInput, UserInputResponse + + +class WebSatNeonClient(NeonAIClient): + """Neon AI Web UI and Voice Satellite client.""" + + def __init__(self, lang: str = ""): + config = Configuration() + self.config = config.get("iris") or dict() + self.mq_config = config.get("MQ") + if not self.mq_config: + raise ValueError( + "Missing MQ configuration, please set it in ~/.config/neon/neon.yaml" + ) + NeonAIClient.__init__(self, self.mq_config) + self.router = APIRouter() + self._await_response = Event() + self._response = None + self._transcribed = None + self._current_tts = dict() + self._profiles: Dict[str, dict] = dict() + self._audio_path = join( + xdg_data_home(), "iris", "stt" + ) # TODO: Clear periodically, or have persistent storage + if not isdir(self._audio_path): + makedirs(self._audio_path) + self.default_lang = (lang or + self.config.get("default_lang", "")).split('-')[0] + LOG.name = "iris" + LOG.init(self.config.get("logs")) + # OpenWW + # TODO: Allow for arbitrary models, or pre-existing OpenWW models + self.oww_model = Model( + wakeword_models=["neon_iris/wakeword_models/hey_neon/hey_neon_high.tflite"], + inference_framework="tflite", + ) + # FastAPI + self.templates = Jinja2Templates(directory="neon_iris/templates") + self.build_routes() + + def get_lang(self, session_id: str): + """Get the ISO 693-1 language code for a session.""" + if session_id and session_id in self._profiles: + return self._profiles[session_id]["speech"]["stt_language"] + return (self.user_config["speech"]["stt_language"] or + self.default_lang).split('-')[0] + + def handle_api_response(self, message: Message): + """ + Catch-all handler for `.response` messages routed to this client that + are not explicitly handled (i.e. get_stt, get_tts) + @param message: Response message to something emitted by this client + """ + LOG.debug(f"Got {message.msg_type}: {message.data}") + if message.msg_type == "neon.audio_input.response": + self._transcribed = message.data.get("transcripts", [""])[0] + + def handle_klat_response(self, message: Message): + """ + Handle a valid response from Neon. This includes text and base64-encoded + audio in all requested languages. + @param message: Neon response message + """ + response = NeonTtsResponse(msg_type=message.msg_type, + data=message.data, + context=message.context) + LOG.debug(f"gradio context={response.context.gradio}") + resp_data = response.data.responses + sentences = [] + session = response.context.gradio.session + for _, response in resp_data.items(): # lang, response + sentences.append(response.sentence) + if response.audio: + for _, data in response.audio.items(): + self._current_tts[session] = data + self._response = "\n".join(sentences) + self._await_response.set() + + def send_audio( # pylint: disable=arguments-renamed + self, + audio_b64_string: str, + lang: str = "en", + username: Optional[str] = None, + user_profiles: Optional[list] = None, + context: Optional[dict] = None, + ): + """ + Optionally override this to queue audio inputs or do any pre-parsing + :param audio_file: path to audio file to send to speech module + :param lang: language code associated with request + :param username: username associated with request + :param user_profiles: user profiles expecting a response + :param context: Optional dict context to add to emitted message + """ + audio_path = decode_base64_string_to_file( + audio_b64_string, + join(f"{self._audio_path}/{time()}.wav"), + ) + self._send_audio( + audio_file=audio_path, + lang=lang, + username=username, + user_profiles=user_profiles, + context=context, + ) + + @property + def supported_languages(self) -> Sequence[str]: + """ + Get a list of supported languages from configuration + @returns: list of ISO 639-1 language codes + """ + languages = [lang.split('-')[0] for lang in self.config.get("languages")] + if languages is None: + return [self.default_lang] + if not isinstance(languages, list): + raise TypeError("Expected a list of languages in the configuration") + return languages + + def _start_session(self): + sid = uuid4().hex + self._current_tts[sid] = None + self._profiles[sid] = self.user_config + self._profiles[sid]["user"]["username"] = sid + return sid + + def build_routes(self): + """Build the FastAPI routes.""" + + @self.router.get("/") + async def read_root(request: Request): + """Render the Neon AI Web UI and Voice Satellite.""" + description = self.config.get("webui_description", "Chat With Neon") + title = self.config.get("webui_title", "Neon AI") + placeholder = self.config.get("webui_input_placeholder", "Ask me something") + ws_url = self.config.get("webui_ws_url", "ws://localhost:8000/ws") + + context = { + "request": request, + "title": title, + "description": description, + "placeholder": placeholder, + "ws_url": ws_url + } + return self.templates.TemplateResponse("index.html", context) + + @self.router.websocket("/ws") + async def websocket_endpoint(websocket: WebSocket): + """Handles websocket connections to OpenWakeWord, which runs as part of this service.""" + await websocket.accept() + # Send loaded models + await websocket.send_text( + json.dumps({"loaded_models": list(self.oww_model.models.keys())}) + ) + sample_rate = None + + while True: + message = await websocket.receive() + + if message["type"] == "websocket.disconnect": + break + + if message["type"] == "websocket.receive": + if "text" in message: + # Process text message + sample_rate = int(message["text"]) + elif "bytes" in message: + # Process bytes message + audio_bytes = message["bytes"] + + # Add extra bytes of silence if needed + if len(audio_bytes) % 2 == 1: + audio_bytes += b"\x00" + + # Convert audio to correct format and sample rate + audio_data = np.frombuffer(audio_bytes, dtype=np.int16) + if sample_rate and sample_rate != 16000: + audio_data = resampy.resample( + audio_data, sample_rate, 16000 + ) + + # Get openWakeWord predictions and send to browser client + predictions = self.oww_model.predict(audio_data) + + activations = [ + key for key, value in predictions.items() if value >= 0.5 + ] + + if activations: + await websocket.send_text( + json.dumps({"activations": activations}) + ) + + @self.router.post("/user_input") + async def on_user_input_worker( + req: UserInput, + ): + """ + Callback to handle textual user input + @param utterance: String utterance submitted by the user + @returns: Session ID, audio input, audio output + """ + utterance = req.utterance or "" + audio_input = req.audio_input or "" + session_id = req.session_id or "websat0000" + + chat_history = [] + input_time = time() + LOG.debug("Input received") + if not self._profiles.get("session_id"): + self._profiles[session_id] = { + "speech": {"stt_language": self.default_lang} + } + self._current_tts[session_id] = None + if not self._await_response.wait(30): + LOG.error("Previous response not completed after 30 seconds") + in_queue = time() - input_time + self._await_response.clear() + self._response = None + self._transcribed = None + lang = self.get_lang(session_id) + if utterance: + LOG.info(f"Sending utterance: {utterance} with lang: {lang}") + self.send_utterance( + utterance, + lang or "en", + username=session_id, + user_profiles=[self._profiles[session_id]], + context={ + "gradio": {"session": session_id}, + "timing": {"wait_in_queue": in_queue, "gradio_sent": time()}, + }, + ) + else: + LOG.info(f"Sending audio with length of {len(audio_input)} with lang: {lang}") + self.send_audio( + audio_input, + lang or "en", + username=session_id, + user_profiles=[self._profiles[session_id]], + context={ + "gradio": {"session": session_id}, + "timing": {"wait_in_queue": in_queue, "gradio_sent": time()}, + }, + ) + chat_history.append(((audio_input, None), None)) + if not self._await_response.wait(30): + LOG.error("No response received after 30s") + self._await_response.set() + self._response = self._response or "ERROR" + LOG.info(f"Got response={self._response}") + if utterance: + chat_history.append((utterance, self._response)) + elif isinstance(self._transcribed, str): + LOG.info(f"Got transcript: {self._transcribed}") + chat_history.append((self._transcribed, self._response)) + utterance = self._transcribed + resp = UserInputResponse( + **{ + "utterance": utterance, + "audio_output": self._current_tts[session_id], + "session_id": session_id, + "transcription": self._response, + } + ) + return resp + + def health_check(self, *_): + if self.connection.check_health(): + return Response(status_code=200, content="Ready") + elif not self.connection.ready: + return Response(status_code=200, content="Starting") + else: + return Response(status_code=500, content="Error") + + + +app = FastAPI() +neon_client = WebSatNeonClient() +app.mount( + "/static", + StaticFiles(directory="neon_iris/static"), + name="Neon Web Voice Satellite", +) +app.include_router(neon_client.router) +app.add_route("/status", neon_client.health_check, methods=["GET"]) + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/requirements/gradio.txt b/requirements/gradio.txt index fb2c552..3653c07 100644 --- a/requirements/gradio.txt +++ b/requirements/gradio.txt @@ -1,3 +1,3 @@ -gradio~=3.28 +gradio~=5.0 librosa~=0.9 -soundfile~=0.12 \ No newline at end of file +soundfile~=0.12 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index c1de4b9..aba1426 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,6 +1,9 @@ click~=8.0 click-default-group~=1.2 -neon-utils~=1.0 +neon-utils[sentry]~=1.14,>=1.14.1 pyyaml>=5.4,<7.0.0 -neon-mq-connector~=0.7,>=0.7.1 -ovos-bus-client~=0.0.3 \ No newline at end of file +neon-mq-connector~=0.10 +ovos-bus-client~=0.0 +ovos-config~=0.1 +ovos-utils~=0.8 +neon-data-models~=0.0,>=0.0.2 diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt index 55b033e..6ecc3f1 100644 --- a/requirements/test_requirements.txt +++ b/requirements/test_requirements.txt @@ -1 +1,2 @@ -pytest \ No newline at end of file +pytest~=8.0 +neon-minerva[rmq]~=0.4,>=0.4.1 diff --git a/requirements/web_sat.txt b/requirements/web_sat.txt new file mode 100644 index 0000000..6fa3a9f --- /dev/null +++ b/requirements/web_sat.txt @@ -0,0 +1,9 @@ +fastapi~=0.104.1 +uvicorn[standard]~=0.24.0.post1 +aiohttp~=3.8.6 +resampy~=0.4.2 +openwakeword~=0.5.1 +tflite~=2.10.0 +onnxruntime~=1.17 +jinja2~=3.1.2 +numpy~=1.0 diff --git a/setup.py b/setup.py index 99fc4f4..b7f644b 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,9 @@ # NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework # All trademark and other rights reserved by their respective owners -# Copyright 2008-2022 Neongecko.com Inc. +# Copyright 2008-2025 Neongecko.com Inc. # Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, # Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# Mike Gray, David Scripka # BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -35,17 +36,25 @@ def get_requirements(requirements_filename: str): - requirements_file = path.join(BASE_PATH, "requirements", requirements_filename) - with open(requirements_file, 'r', encoding='utf-8') as r: + requirements_file = path.join( + BASE_PATH, "requirements", requirements_filename + ) + with open(requirements_file, "r", encoding="utf-8") as r: requirements = r.readlines() - requirements = [r.strip() for r in requirements if r.strip() and not r.strip().startswith("#")] + requirements = [ + r.strip() + for r in requirements + if r.strip() and not r.strip().startswith("#") + ] return requirements with open(path.join(BASE_PATH, "README.md"), "r") as f: long_description = f.read() -with open(path.join(BASE_PATH, "neon_iris", "version.py"), "r", encoding="utf-8") as v: +with open( + path.join(BASE_PATH, "neon_iris", "version.py"), "r", encoding="utf-8" +) as v: for line in v.readlines(): if line.startswith("__version__"): if '"' in line: @@ -56,9 +65,9 @@ def get_requirements(requirements_filename: str): setuptools.setup( name="neon-iris", version=version, - author='Neongecko', - author_email='developers@neon.ai', - license='BSD-3-Clause', + author="Neongecko", + author_email="developers@neon.ai", + license="BSD-3-Clause", description="Interactive Relay for Intelligence Systems", long_description=long_description, long_description_content_type="text/markdown", @@ -67,12 +76,17 @@ def get_requirements(requirements_filename: str): include_package_data=True, classifiers=[ "Programming Language :: Python :: 3", - "Operating System :: OS Independent" + "Operating System :: OS Independent", ], - python_requires='>=3.6', + python_requires=">=3.7", install_requires=get_requirements("requirements.txt"), - extras_require={"gradio": get_requirements("gradio.txt")}, - entry_points={ - 'console_scripts': ['iris=neon_iris.cli:neon_iris_cli'] - } + extras_require={ + "gradio": get_requirements("gradio.txt"), + "web_sat": get_requirements("web_sat.txt"), + "test": get_requirements("test_requirements.txt"), + }, + entry_points={"console_scripts": ["iris=neon_iris.cli:neon_iris_cli"]}, + package_data={ + "neon_iris": ["static/*", "templates/*", "res/*", "wakeword_models/*"] + }, ) diff --git a/tests/test_client.py b/tests/test_client.py index 183c4c9..5f27bc2 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,6 +1,6 @@ # NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework # All trademark and other rights reserved by their respective owners -# Copyright 2008-2022 Neongecko.com Inc. +# Copyright 2008-2025 Neongecko.com Inc. # Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, # Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo # BSD-3 License @@ -27,17 +27,14 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os -import sys import unittest -from neon_utils.mq_utils import NeonMQHandler - -sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) +from neon_iris.mq_connector import IrisConnector from neon_iris.client import NeonAIClient _test_config = { "MQ": { - "server": "mq.2022.us", + "server": "mq.neonaialpha.com", "port": 25672, "users": { "mq_handler": { @@ -57,10 +54,7 @@ def test_client_create(self): self.assertEqual(client._connection.config, _test_config["MQ"]) self.assertTrue(os.path.isdir(client.audio_cache_dir)) self.assertIsInstance(client.client_name, str) - self.assertIsInstance(client.connection, NeonMQHandler) + self.assertIsInstance(client.connection, IrisConnector) self.assertEqual(client.connection.vhost, "/neon_chat_api") client.shutdown() - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_mq_connector.py b/tests/test_mq_connector.py new file mode 100644 index 0000000..48fba7d --- /dev/null +++ b/tests/test_mq_connector.py @@ -0,0 +1,74 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2025 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import unittest +from threading import Thread + +import pytest + +from os import environ +from neon_mq_connector import MQConnector +from pika.adapters.select_connection import SelectConnection + +environ['TEST_RMQ_USERNAME'] = "test_user" +environ['TEST_RMQ_PASSWORD'] = "test_password" +environ['TEST_RMQ_VHOSTS'] = "/neon_chat_api" + + +# Fixture defined in neon_minerva.integration.rabbit_mq +@pytest.mark.usefixtures("rmq_instance") +class TestClient(unittest.TestCase): + mq_config = {"server": "localhost", + "port": None, + "users": {"mq_handler": {"user": "test_user", + "password": "test_password"}}} + + def setUp(self): + if self.mq_config["port"] is None: + self.mq_config["port"] = self.rmq_instance.port + + def test_lifecycle(self): + from neon_iris.mq_connector import IrisConnector + connector = IrisConnector(vhost="/neon_chat_api", config=self.mq_config) + self.assertIsInstance(connector, MQConnector) + self.assertIsInstance(connector.connection, SelectConnection) + self.assertFalse(connector.ready) + + # Start the connector + thread = Thread(target=connector.run) + thread.start() + connector.wait_for_connection() + self.assertTrue(connector.ready) + + # Stop the connector + connector.shutdown() + self.assertFalse(connector.ready) + thread.join(timeout=5) + self.assertFalse(thread.is_alive()) +